1 | /* |
2 | * device quirks for PCI devices |
3 | * |
4 | * Copyright Red Hat, Inc. 2012-2015 |
5 | * |
6 | * Authors: |
7 | * Alex Williamson <alex.williamson@redhat.com> |
8 | * |
9 | * This work is licensed under the terms of the GNU GPL, version 2. See |
10 | * the COPYING file in the top-level directory. |
11 | */ |
12 | |
13 | #include "qemu/osdep.h" |
14 | #include "exec/memop.h" |
15 | #include "qemu/units.h" |
16 | #include "qemu/error-report.h" |
17 | #include "qemu/main-loop.h" |
18 | #include "qemu/module.h" |
19 | #include "qemu/range.h" |
20 | #include "qapi/error.h" |
21 | #include "qapi/visitor.h" |
22 | #include <sys/ioctl.h> |
23 | #include "hw/hw.h" |
24 | #include "hw/nvram/fw_cfg.h" |
25 | #include "hw/qdev-properties.h" |
26 | #include "pci.h" |
27 | #include "trace.h" |
28 | |
29 | /* Use uin32_t for vendor & device so PCI_ANY_ID expands and cannot match hw */ |
30 | static bool vfio_pci_is(VFIOPCIDevice *vdev, uint32_t vendor, uint32_t device) |
31 | { |
32 | return (vendor == PCI_ANY_ID || vendor == vdev->vendor_id) && |
33 | (device == PCI_ANY_ID || device == vdev->device_id); |
34 | } |
35 | |
36 | static bool vfio_is_vga(VFIOPCIDevice *vdev) |
37 | { |
38 | PCIDevice *pdev = &vdev->pdev; |
39 | uint16_t class = pci_get_word(pdev->config + PCI_CLASS_DEVICE); |
40 | |
41 | return class == PCI_CLASS_DISPLAY_VGA; |
42 | } |
43 | |
44 | /* |
45 | * List of device ids/vendor ids for which to disable |
46 | * option rom loading. This avoids the guest hangs during rom |
47 | * execution as noticed with the BCM 57810 card for lack of a |
48 | * more better way to handle such issues. |
49 | * The user can still override by specifying a romfile or |
50 | * rombar=1. |
51 | * Please see https://bugs.launchpad.net/qemu/+bug/1284874 |
52 | * for an analysis of the 57810 card hang. When adding |
53 | * a new vendor id/device id combination below, please also add |
54 | * your card/environment details and information that could |
55 | * help in debugging to the bug tracking this issue |
56 | */ |
57 | static const struct { |
58 | uint32_t vendor; |
59 | uint32_t device; |
60 | } romblacklist[] = { |
61 | { 0x14e4, 0x168e }, /* Broadcom BCM 57810 */ |
62 | }; |
63 | |
64 | bool vfio_blacklist_opt_rom(VFIOPCIDevice *vdev) |
65 | { |
66 | int i; |
67 | |
68 | for (i = 0 ; i < ARRAY_SIZE(romblacklist); i++) { |
69 | if (vfio_pci_is(vdev, romblacklist[i].vendor, romblacklist[i].device)) { |
70 | trace_vfio_quirk_rom_blacklisted(vdev->vbasedev.name, |
71 | romblacklist[i].vendor, |
72 | romblacklist[i].device); |
73 | return true; |
74 | } |
75 | } |
76 | return false; |
77 | } |
78 | |
79 | /* |
80 | * Device specific region quirks (mostly backdoors to PCI config space) |
81 | */ |
82 | |
83 | /* |
84 | * The generic window quirks operate on an address and data register, |
85 | * vfio_generic_window_address_quirk handles the address register and |
86 | * vfio_generic_window_data_quirk handles the data register. These ops |
87 | * pass reads and writes through to hardware until a value matching the |
88 | * stored address match/mask is written. When this occurs, the data |
89 | * register access emulated PCI config space for the device rather than |
90 | * passing through accesses. This enables devices where PCI config space |
91 | * is accessible behind a window register to maintain the virtualization |
92 | * provided through vfio. |
93 | */ |
94 | typedef struct VFIOConfigWindowMatch { |
95 | uint32_t match; |
96 | uint32_t mask; |
97 | } VFIOConfigWindowMatch; |
98 | |
99 | typedef struct VFIOConfigWindowQuirk { |
100 | struct VFIOPCIDevice *vdev; |
101 | |
102 | uint32_t address_val; |
103 | |
104 | uint32_t address_offset; |
105 | uint32_t data_offset; |
106 | |
107 | bool window_enabled; |
108 | uint8_t bar; |
109 | |
110 | MemoryRegion *addr_mem; |
111 | MemoryRegion *data_mem; |
112 | |
113 | uint32_t nr_matches; |
114 | VFIOConfigWindowMatch matches[]; |
115 | } VFIOConfigWindowQuirk; |
116 | |
117 | static uint64_t vfio_generic_window_quirk_address_read(void *opaque, |
118 | hwaddr addr, |
119 | unsigned size) |
120 | { |
121 | VFIOConfigWindowQuirk *window = opaque; |
122 | VFIOPCIDevice *vdev = window->vdev; |
123 | |
124 | return vfio_region_read(&vdev->bars[window->bar].region, |
125 | addr + window->address_offset, size); |
126 | } |
127 | |
128 | static void vfio_generic_window_quirk_address_write(void *opaque, hwaddr addr, |
129 | uint64_t data, |
130 | unsigned size) |
131 | { |
132 | VFIOConfigWindowQuirk *window = opaque; |
133 | VFIOPCIDevice *vdev = window->vdev; |
134 | int i; |
135 | |
136 | window->window_enabled = false; |
137 | |
138 | vfio_region_write(&vdev->bars[window->bar].region, |
139 | addr + window->address_offset, data, size); |
140 | |
141 | for (i = 0; i < window->nr_matches; i++) { |
142 | if ((data & ~window->matches[i].mask) == window->matches[i].match) { |
143 | window->window_enabled = true; |
144 | window->address_val = data & window->matches[i].mask; |
145 | trace_vfio_quirk_generic_window_address_write(vdev->vbasedev.name, |
146 | memory_region_name(window->addr_mem), data); |
147 | break; |
148 | } |
149 | } |
150 | } |
151 | |
152 | static const MemoryRegionOps vfio_generic_window_address_quirk = { |
153 | .read = vfio_generic_window_quirk_address_read, |
154 | .write = vfio_generic_window_quirk_address_write, |
155 | .endianness = DEVICE_LITTLE_ENDIAN, |
156 | }; |
157 | |
158 | static uint64_t vfio_generic_window_quirk_data_read(void *opaque, |
159 | hwaddr addr, unsigned size) |
160 | { |
161 | VFIOConfigWindowQuirk *window = opaque; |
162 | VFIOPCIDevice *vdev = window->vdev; |
163 | uint64_t data; |
164 | |
165 | /* Always read data reg, discard if window enabled */ |
166 | data = vfio_region_read(&vdev->bars[window->bar].region, |
167 | addr + window->data_offset, size); |
168 | |
169 | if (window->window_enabled) { |
170 | data = vfio_pci_read_config(&vdev->pdev, window->address_val, size); |
171 | trace_vfio_quirk_generic_window_data_read(vdev->vbasedev.name, |
172 | memory_region_name(window->data_mem), data); |
173 | } |
174 | |
175 | return data; |
176 | } |
177 | |
178 | static void vfio_generic_window_quirk_data_write(void *opaque, hwaddr addr, |
179 | uint64_t data, unsigned size) |
180 | { |
181 | VFIOConfigWindowQuirk *window = opaque; |
182 | VFIOPCIDevice *vdev = window->vdev; |
183 | |
184 | if (window->window_enabled) { |
185 | vfio_pci_write_config(&vdev->pdev, window->address_val, data, size); |
186 | trace_vfio_quirk_generic_window_data_write(vdev->vbasedev.name, |
187 | memory_region_name(window->data_mem), data); |
188 | return; |
189 | } |
190 | |
191 | vfio_region_write(&vdev->bars[window->bar].region, |
192 | addr + window->data_offset, data, size); |
193 | } |
194 | |
195 | static const MemoryRegionOps vfio_generic_window_data_quirk = { |
196 | .read = vfio_generic_window_quirk_data_read, |
197 | .write = vfio_generic_window_quirk_data_write, |
198 | .endianness = DEVICE_LITTLE_ENDIAN, |
199 | }; |
200 | |
201 | /* |
202 | * The generic mirror quirk handles devices which expose PCI config space |
203 | * through a region within a BAR. When enabled, reads and writes are |
204 | * redirected through to emulated PCI config space. XXX if PCI config space |
205 | * used memory regions, this could just be an alias. |
206 | */ |
207 | typedef struct VFIOConfigMirrorQuirk { |
208 | struct VFIOPCIDevice *vdev; |
209 | uint32_t offset; |
210 | uint8_t bar; |
211 | MemoryRegion *mem; |
212 | uint8_t data[]; |
213 | } VFIOConfigMirrorQuirk; |
214 | |
215 | static uint64_t vfio_generic_quirk_mirror_read(void *opaque, |
216 | hwaddr addr, unsigned size) |
217 | { |
218 | VFIOConfigMirrorQuirk *mirror = opaque; |
219 | VFIOPCIDevice *vdev = mirror->vdev; |
220 | uint64_t data; |
221 | |
222 | /* Read and discard in case the hardware cares */ |
223 | (void)vfio_region_read(&vdev->bars[mirror->bar].region, |
224 | addr + mirror->offset, size); |
225 | |
226 | data = vfio_pci_read_config(&vdev->pdev, addr, size); |
227 | trace_vfio_quirk_generic_mirror_read(vdev->vbasedev.name, |
228 | memory_region_name(mirror->mem), |
229 | addr, data); |
230 | return data; |
231 | } |
232 | |
233 | static void vfio_generic_quirk_mirror_write(void *opaque, hwaddr addr, |
234 | uint64_t data, unsigned size) |
235 | { |
236 | VFIOConfigMirrorQuirk *mirror = opaque; |
237 | VFIOPCIDevice *vdev = mirror->vdev; |
238 | |
239 | vfio_pci_write_config(&vdev->pdev, addr, data, size); |
240 | trace_vfio_quirk_generic_mirror_write(vdev->vbasedev.name, |
241 | memory_region_name(mirror->mem), |
242 | addr, data); |
243 | } |
244 | |
245 | static const MemoryRegionOps vfio_generic_mirror_quirk = { |
246 | .read = vfio_generic_quirk_mirror_read, |
247 | .write = vfio_generic_quirk_mirror_write, |
248 | .endianness = DEVICE_LITTLE_ENDIAN, |
249 | }; |
250 | |
251 | /* Is range1 fully contained within range2? */ |
252 | static bool vfio_range_contained(uint64_t first1, uint64_t len1, |
253 | uint64_t first2, uint64_t len2) { |
254 | return (first1 >= first2 && first1 + len1 <= first2 + len2); |
255 | } |
256 | |
257 | #define PCI_VENDOR_ID_ATI 0x1002 |
258 | |
259 | /* |
260 | * Radeon HD cards (HD5450 & HD7850) report the upper byte of the I/O port BAR |
261 | * through VGA register 0x3c3. On newer cards, the I/O port BAR is always |
262 | * BAR4 (older cards like the X550 used BAR1, but we don't care to support |
263 | * those). Note that on bare metal, a read of 0x3c3 doesn't always return the |
264 | * I/O port BAR address. Originally this was coded to return the virtual BAR |
265 | * address only if the physical register read returns the actual BAR address, |
266 | * but users have reported greater success if we return the virtual address |
267 | * unconditionally. |
268 | */ |
269 | static uint64_t vfio_ati_3c3_quirk_read(void *opaque, |
270 | hwaddr addr, unsigned size) |
271 | { |
272 | VFIOPCIDevice *vdev = opaque; |
273 | uint64_t data = vfio_pci_read_config(&vdev->pdev, |
274 | PCI_BASE_ADDRESS_4 + 1, size); |
275 | |
276 | trace_vfio_quirk_ati_3c3_read(vdev->vbasedev.name, data); |
277 | |
278 | return data; |
279 | } |
280 | |
281 | static const MemoryRegionOps vfio_ati_3c3_quirk = { |
282 | .read = vfio_ati_3c3_quirk_read, |
283 | .endianness = DEVICE_LITTLE_ENDIAN, |
284 | }; |
285 | |
286 | static VFIOQuirk *vfio_quirk_alloc(int nr_mem) |
287 | { |
288 | VFIOQuirk *quirk = g_new0(VFIOQuirk, 1); |
289 | QLIST_INIT(&quirk->ioeventfds); |
290 | quirk->mem = g_new0(MemoryRegion, nr_mem); |
291 | quirk->nr_mem = nr_mem; |
292 | |
293 | return quirk; |
294 | } |
295 | |
296 | static void vfio_ioeventfd_exit(VFIOPCIDevice *vdev, VFIOIOEventFD *ioeventfd) |
297 | { |
298 | QLIST_REMOVE(ioeventfd, next); |
299 | memory_region_del_eventfd(ioeventfd->mr, ioeventfd->addr, ioeventfd->size, |
300 | true, ioeventfd->data, &ioeventfd->e); |
301 | |
302 | if (ioeventfd->vfio) { |
303 | struct vfio_device_ioeventfd vfio_ioeventfd; |
304 | |
305 | vfio_ioeventfd.argsz = sizeof(vfio_ioeventfd); |
306 | vfio_ioeventfd.flags = ioeventfd->size; |
307 | vfio_ioeventfd.data = ioeventfd->data; |
308 | vfio_ioeventfd.offset = ioeventfd->region->fd_offset + |
309 | ioeventfd->region_addr; |
310 | vfio_ioeventfd.fd = -1; |
311 | |
312 | if (ioctl(vdev->vbasedev.fd, VFIO_DEVICE_IOEVENTFD, &vfio_ioeventfd)) { |
313 | error_report("Failed to remove vfio ioeventfd for %s+0x%" |
314 | HWADDR_PRIx"[%d]:0x%" PRIx64" (%m)" , |
315 | memory_region_name(ioeventfd->mr), ioeventfd->addr, |
316 | ioeventfd->size, ioeventfd->data); |
317 | } |
318 | } else { |
319 | qemu_set_fd_handler(event_notifier_get_fd(&ioeventfd->e), |
320 | NULL, NULL, NULL); |
321 | } |
322 | |
323 | event_notifier_cleanup(&ioeventfd->e); |
324 | trace_vfio_ioeventfd_exit(memory_region_name(ioeventfd->mr), |
325 | (uint64_t)ioeventfd->addr, ioeventfd->size, |
326 | ioeventfd->data); |
327 | g_free(ioeventfd); |
328 | } |
329 | |
330 | static void vfio_drop_dynamic_eventfds(VFIOPCIDevice *vdev, VFIOQuirk *quirk) |
331 | { |
332 | VFIOIOEventFD *ioeventfd, *tmp; |
333 | |
334 | QLIST_FOREACH_SAFE(ioeventfd, &quirk->ioeventfds, next, tmp) { |
335 | if (ioeventfd->dynamic) { |
336 | vfio_ioeventfd_exit(vdev, ioeventfd); |
337 | } |
338 | } |
339 | } |
340 | |
341 | static void vfio_ioeventfd_handler(void *opaque) |
342 | { |
343 | VFIOIOEventFD *ioeventfd = opaque; |
344 | |
345 | if (event_notifier_test_and_clear(&ioeventfd->e)) { |
346 | vfio_region_write(ioeventfd->region, ioeventfd->region_addr, |
347 | ioeventfd->data, ioeventfd->size); |
348 | trace_vfio_ioeventfd_handler(memory_region_name(ioeventfd->mr), |
349 | (uint64_t)ioeventfd->addr, ioeventfd->size, |
350 | ioeventfd->data); |
351 | } |
352 | } |
353 | |
354 | static VFIOIOEventFD *vfio_ioeventfd_init(VFIOPCIDevice *vdev, |
355 | MemoryRegion *mr, hwaddr addr, |
356 | unsigned size, uint64_t data, |
357 | VFIORegion *region, |
358 | hwaddr region_addr, bool dynamic) |
359 | { |
360 | VFIOIOEventFD *ioeventfd; |
361 | |
362 | if (vdev->no_kvm_ioeventfd) { |
363 | return NULL; |
364 | } |
365 | |
366 | ioeventfd = g_malloc0(sizeof(*ioeventfd)); |
367 | |
368 | if (event_notifier_init(&ioeventfd->e, 0)) { |
369 | g_free(ioeventfd); |
370 | return NULL; |
371 | } |
372 | |
373 | /* |
374 | * MemoryRegion and relative offset, plus additional ioeventfd setup |
375 | * parameters for configuring and later tearing down KVM ioeventfd. |
376 | */ |
377 | ioeventfd->mr = mr; |
378 | ioeventfd->addr = addr; |
379 | ioeventfd->size = size; |
380 | ioeventfd->data = data; |
381 | ioeventfd->dynamic = dynamic; |
382 | /* |
383 | * VFIORegion and relative offset for implementing the userspace |
384 | * handler. data & size fields shared for both uses. |
385 | */ |
386 | ioeventfd->region = region; |
387 | ioeventfd->region_addr = region_addr; |
388 | |
389 | if (!vdev->no_vfio_ioeventfd) { |
390 | struct vfio_device_ioeventfd vfio_ioeventfd; |
391 | |
392 | vfio_ioeventfd.argsz = sizeof(vfio_ioeventfd); |
393 | vfio_ioeventfd.flags = ioeventfd->size; |
394 | vfio_ioeventfd.data = ioeventfd->data; |
395 | vfio_ioeventfd.offset = ioeventfd->region->fd_offset + |
396 | ioeventfd->region_addr; |
397 | vfio_ioeventfd.fd = event_notifier_get_fd(&ioeventfd->e); |
398 | |
399 | ioeventfd->vfio = !ioctl(vdev->vbasedev.fd, |
400 | VFIO_DEVICE_IOEVENTFD, &vfio_ioeventfd); |
401 | } |
402 | |
403 | if (!ioeventfd->vfio) { |
404 | qemu_set_fd_handler(event_notifier_get_fd(&ioeventfd->e), |
405 | vfio_ioeventfd_handler, NULL, ioeventfd); |
406 | } |
407 | |
408 | memory_region_add_eventfd(ioeventfd->mr, ioeventfd->addr, ioeventfd->size, |
409 | true, ioeventfd->data, &ioeventfd->e); |
410 | trace_vfio_ioeventfd_init(memory_region_name(mr), (uint64_t)addr, |
411 | size, data, ioeventfd->vfio); |
412 | |
413 | return ioeventfd; |
414 | } |
415 | |
416 | static void vfio_vga_probe_ati_3c3_quirk(VFIOPCIDevice *vdev) |
417 | { |
418 | VFIOQuirk *quirk; |
419 | |
420 | /* |
421 | * As long as the BAR is >= 256 bytes it will be aligned such that the |
422 | * lower byte is always zero. Filter out anything else, if it exists. |
423 | */ |
424 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || |
425 | !vdev->bars[4].ioport || vdev->bars[4].region.size < 256) { |
426 | return; |
427 | } |
428 | |
429 | quirk = vfio_quirk_alloc(1); |
430 | |
431 | memory_region_init_io(quirk->mem, OBJECT(vdev), &vfio_ati_3c3_quirk, vdev, |
432 | "vfio-ati-3c3-quirk" , 1); |
433 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
434 | 3 /* offset 3 bytes from 0x3c0 */, quirk->mem); |
435 | |
436 | QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks, |
437 | quirk, next); |
438 | |
439 | trace_vfio_quirk_ati_3c3_probe(vdev->vbasedev.name); |
440 | } |
441 | |
442 | /* |
443 | * Newer ATI/AMD devices, including HD5450 and HD7850, have a mirror to PCI |
444 | * config space through MMIO BAR2 at offset 0x4000. Nothing seems to access |
445 | * the MMIO space directly, but a window to this space is provided through |
446 | * I/O port BAR4. Offset 0x0 is the address register and offset 0x4 is the |
447 | * data register. When the address is programmed to a range of 0x4000-0x4fff |
448 | * PCI configuration space is available. Experimentation seems to indicate |
449 | * that read-only may be provided by hardware. |
450 | */ |
451 | static void vfio_probe_ati_bar4_quirk(VFIOPCIDevice *vdev, int nr) |
452 | { |
453 | VFIOQuirk *quirk; |
454 | VFIOConfigWindowQuirk *window; |
455 | |
456 | /* This windows doesn't seem to be used except by legacy VGA code */ |
457 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || |
458 | !vdev->vga || nr != 4) { |
459 | return; |
460 | } |
461 | |
462 | quirk = vfio_quirk_alloc(2); |
463 | window = quirk->data = g_malloc0(sizeof(*window) + |
464 | sizeof(VFIOConfigWindowMatch)); |
465 | window->vdev = vdev; |
466 | window->address_offset = 0; |
467 | window->data_offset = 4; |
468 | window->nr_matches = 1; |
469 | window->matches[0].match = 0x4000; |
470 | window->matches[0].mask = vdev->config_size - 1; |
471 | window->bar = nr; |
472 | window->addr_mem = &quirk->mem[0]; |
473 | window->data_mem = &quirk->mem[1]; |
474 | |
475 | memory_region_init_io(window->addr_mem, OBJECT(vdev), |
476 | &vfio_generic_window_address_quirk, window, |
477 | "vfio-ati-bar4-window-address-quirk" , 4); |
478 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
479 | window->address_offset, |
480 | window->addr_mem, 1); |
481 | |
482 | memory_region_init_io(window->data_mem, OBJECT(vdev), |
483 | &vfio_generic_window_data_quirk, window, |
484 | "vfio-ati-bar4-window-data-quirk" , 4); |
485 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
486 | window->data_offset, |
487 | window->data_mem, 1); |
488 | |
489 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
490 | |
491 | trace_vfio_quirk_ati_bar4_probe(vdev->vbasedev.name); |
492 | } |
493 | |
494 | /* |
495 | * Trap the BAR2 MMIO mirror to config space as well. |
496 | */ |
497 | static void vfio_probe_ati_bar2_quirk(VFIOPCIDevice *vdev, int nr) |
498 | { |
499 | VFIOQuirk *quirk; |
500 | VFIOConfigMirrorQuirk *mirror; |
501 | |
502 | /* Only enable on newer devices where BAR2 is 64bit */ |
503 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_ATI, PCI_ANY_ID) || |
504 | !vdev->vga || nr != 2 || !vdev->bars[2].mem64) { |
505 | return; |
506 | } |
507 | |
508 | quirk = vfio_quirk_alloc(1); |
509 | mirror = quirk->data = g_malloc0(sizeof(*mirror)); |
510 | mirror->mem = quirk->mem; |
511 | mirror->vdev = vdev; |
512 | mirror->offset = 0x4000; |
513 | mirror->bar = nr; |
514 | |
515 | memory_region_init_io(mirror->mem, OBJECT(vdev), |
516 | &vfio_generic_mirror_quirk, mirror, |
517 | "vfio-ati-bar2-4000-quirk" , PCI_CONFIG_SPACE_SIZE); |
518 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
519 | mirror->offset, mirror->mem, 1); |
520 | |
521 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
522 | |
523 | trace_vfio_quirk_ati_bar2_probe(vdev->vbasedev.name); |
524 | } |
525 | |
526 | /* |
527 | * Older ATI/AMD cards like the X550 have a similar window to that above. |
528 | * I/O port BAR1 provides a window to a mirror of PCI config space located |
529 | * in BAR2 at offset 0xf00. We don't care to support such older cards, but |
530 | * note it for future reference. |
531 | */ |
532 | |
533 | /* |
534 | * Nvidia has several different methods to get to config space, the |
535 | * nouveu project has several of these documented here: |
536 | * https://github.com/pathscale/envytools/tree/master/hwdocs |
537 | * |
538 | * The first quirk is actually not documented in envytools and is found |
539 | * on 10de:01d1 (NVIDIA Corporation G72 [GeForce 7300 LE]). This is an |
540 | * NV46 chipset. The backdoor uses the legacy VGA I/O ports to access |
541 | * the mirror of PCI config space found at BAR0 offset 0x1800. The access |
542 | * sequence first writes 0x338 to I/O port 0x3d4. The target offset is |
543 | * then written to 0x3d0. Finally 0x538 is written for a read and 0x738 |
544 | * is written for a write to 0x3d4. The BAR0 offset is then accessible |
545 | * through 0x3d0. This quirk doesn't seem to be necessary on newer cards |
546 | * that use the I/O port BAR5 window but it doesn't hurt to leave it. |
547 | */ |
548 | typedef enum {NONE = 0, SELECT, WINDOW, READ, WRITE} VFIONvidia3d0State; |
549 | static const char *nv3d0_states[] = { "NONE" , "SELECT" , |
550 | "WINDOW" , "READ" , "WRITE" }; |
551 | |
552 | typedef struct VFIONvidia3d0Quirk { |
553 | VFIOPCIDevice *vdev; |
554 | VFIONvidia3d0State state; |
555 | uint32_t offset; |
556 | } VFIONvidia3d0Quirk; |
557 | |
558 | static uint64_t vfio_nvidia_3d4_quirk_read(void *opaque, |
559 | hwaddr addr, unsigned size) |
560 | { |
561 | VFIONvidia3d0Quirk *quirk = opaque; |
562 | VFIOPCIDevice *vdev = quirk->vdev; |
563 | |
564 | quirk->state = NONE; |
565 | |
566 | return vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
567 | addr + 0x14, size); |
568 | } |
569 | |
570 | static void vfio_nvidia_3d4_quirk_write(void *opaque, hwaddr addr, |
571 | uint64_t data, unsigned size) |
572 | { |
573 | VFIONvidia3d0Quirk *quirk = opaque; |
574 | VFIOPCIDevice *vdev = quirk->vdev; |
575 | VFIONvidia3d0State old_state = quirk->state; |
576 | |
577 | quirk->state = NONE; |
578 | |
579 | switch (data) { |
580 | case 0x338: |
581 | if (old_state == NONE) { |
582 | quirk->state = SELECT; |
583 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, |
584 | nv3d0_states[quirk->state]); |
585 | } |
586 | break; |
587 | case 0x538: |
588 | if (old_state == WINDOW) { |
589 | quirk->state = READ; |
590 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, |
591 | nv3d0_states[quirk->state]); |
592 | } |
593 | break; |
594 | case 0x738: |
595 | if (old_state == WINDOW) { |
596 | quirk->state = WRITE; |
597 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, |
598 | nv3d0_states[quirk->state]); |
599 | } |
600 | break; |
601 | } |
602 | |
603 | vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
604 | addr + 0x14, data, size); |
605 | } |
606 | |
607 | static const MemoryRegionOps vfio_nvidia_3d4_quirk = { |
608 | .read = vfio_nvidia_3d4_quirk_read, |
609 | .write = vfio_nvidia_3d4_quirk_write, |
610 | .endianness = DEVICE_LITTLE_ENDIAN, |
611 | }; |
612 | |
613 | static uint64_t vfio_nvidia_3d0_quirk_read(void *opaque, |
614 | hwaddr addr, unsigned size) |
615 | { |
616 | VFIONvidia3d0Quirk *quirk = opaque; |
617 | VFIOPCIDevice *vdev = quirk->vdev; |
618 | VFIONvidia3d0State old_state = quirk->state; |
619 | uint64_t data = vfio_vga_read(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
620 | addr + 0x10, size); |
621 | |
622 | quirk->state = NONE; |
623 | |
624 | if (old_state == READ && |
625 | (quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) { |
626 | uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1); |
627 | |
628 | data = vfio_pci_read_config(&vdev->pdev, offset, size); |
629 | trace_vfio_quirk_nvidia_3d0_read(vdev->vbasedev.name, |
630 | offset, size, data); |
631 | } |
632 | |
633 | return data; |
634 | } |
635 | |
636 | static void vfio_nvidia_3d0_quirk_write(void *opaque, hwaddr addr, |
637 | uint64_t data, unsigned size) |
638 | { |
639 | VFIONvidia3d0Quirk *quirk = opaque; |
640 | VFIOPCIDevice *vdev = quirk->vdev; |
641 | VFIONvidia3d0State old_state = quirk->state; |
642 | |
643 | quirk->state = NONE; |
644 | |
645 | if (old_state == SELECT) { |
646 | quirk->offset = (uint32_t)data; |
647 | quirk->state = WINDOW; |
648 | trace_vfio_quirk_nvidia_3d0_state(vdev->vbasedev.name, |
649 | nv3d0_states[quirk->state]); |
650 | } else if (old_state == WRITE) { |
651 | if ((quirk->offset & ~(PCI_CONFIG_SPACE_SIZE - 1)) == 0x1800) { |
652 | uint8_t offset = quirk->offset & (PCI_CONFIG_SPACE_SIZE - 1); |
653 | |
654 | vfio_pci_write_config(&vdev->pdev, offset, data, size); |
655 | trace_vfio_quirk_nvidia_3d0_write(vdev->vbasedev.name, |
656 | offset, data, size); |
657 | return; |
658 | } |
659 | } |
660 | |
661 | vfio_vga_write(&vdev->vga->region[QEMU_PCI_VGA_IO_HI], |
662 | addr + 0x10, data, size); |
663 | } |
664 | |
665 | static const MemoryRegionOps vfio_nvidia_3d0_quirk = { |
666 | .read = vfio_nvidia_3d0_quirk_read, |
667 | .write = vfio_nvidia_3d0_quirk_write, |
668 | .endianness = DEVICE_LITTLE_ENDIAN, |
669 | }; |
670 | |
671 | static void vfio_vga_probe_nvidia_3d0_quirk(VFIOPCIDevice *vdev) |
672 | { |
673 | VFIOQuirk *quirk; |
674 | VFIONvidia3d0Quirk *data; |
675 | |
676 | if (vdev->no_geforce_quirks || |
677 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || |
678 | !vdev->bars[1].region.size) { |
679 | return; |
680 | } |
681 | |
682 | quirk = vfio_quirk_alloc(2); |
683 | quirk->data = data = g_malloc0(sizeof(*data)); |
684 | data->vdev = vdev; |
685 | |
686 | memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_nvidia_3d4_quirk, |
687 | data, "vfio-nvidia-3d4-quirk" , 2); |
688 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
689 | 0x14 /* 0x3c0 + 0x14 */, &quirk->mem[0]); |
690 | |
691 | memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_nvidia_3d0_quirk, |
692 | data, "vfio-nvidia-3d0-quirk" , 2); |
693 | memory_region_add_subregion(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem, |
694 | 0x10 /* 0x3c0 + 0x10 */, &quirk->mem[1]); |
695 | |
696 | QLIST_INSERT_HEAD(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks, |
697 | quirk, next); |
698 | |
699 | trace_vfio_quirk_nvidia_3d0_probe(vdev->vbasedev.name); |
700 | } |
701 | |
702 | /* |
703 | * The second quirk is documented in envytools. The I/O port BAR5 is just |
704 | * a set of address/data ports to the MMIO BARs. The BAR we care about is |
705 | * again BAR0. This backdoor is apparently a bit newer than the one above |
706 | * so we need to not only trap 256 bytes @0x1800, but all of PCI config |
707 | * space, including extended space is available at the 4k @0x88000. |
708 | */ |
709 | typedef struct VFIONvidiaBAR5Quirk { |
710 | uint32_t master; |
711 | uint32_t enable; |
712 | MemoryRegion *addr_mem; |
713 | MemoryRegion *data_mem; |
714 | bool enabled; |
715 | VFIOConfigWindowQuirk window; /* last for match data */ |
716 | } VFIONvidiaBAR5Quirk; |
717 | |
718 | static void vfio_nvidia_bar5_enable(VFIONvidiaBAR5Quirk *bar5) |
719 | { |
720 | VFIOPCIDevice *vdev = bar5->window.vdev; |
721 | |
722 | if (((bar5->master & bar5->enable) & 0x1) == bar5->enabled) { |
723 | return; |
724 | } |
725 | |
726 | bar5->enabled = !bar5->enabled; |
727 | trace_vfio_quirk_nvidia_bar5_state(vdev->vbasedev.name, |
728 | bar5->enabled ? "Enable" : "Disable" ); |
729 | memory_region_set_enabled(bar5->addr_mem, bar5->enabled); |
730 | memory_region_set_enabled(bar5->data_mem, bar5->enabled); |
731 | } |
732 | |
733 | static uint64_t vfio_nvidia_bar5_quirk_master_read(void *opaque, |
734 | hwaddr addr, unsigned size) |
735 | { |
736 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
737 | VFIOPCIDevice *vdev = bar5->window.vdev; |
738 | |
739 | return vfio_region_read(&vdev->bars[5].region, addr, size); |
740 | } |
741 | |
742 | static void vfio_nvidia_bar5_quirk_master_write(void *opaque, hwaddr addr, |
743 | uint64_t data, unsigned size) |
744 | { |
745 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
746 | VFIOPCIDevice *vdev = bar5->window.vdev; |
747 | |
748 | vfio_region_write(&vdev->bars[5].region, addr, data, size); |
749 | |
750 | bar5->master = data; |
751 | vfio_nvidia_bar5_enable(bar5); |
752 | } |
753 | |
754 | static const MemoryRegionOps vfio_nvidia_bar5_quirk_master = { |
755 | .read = vfio_nvidia_bar5_quirk_master_read, |
756 | .write = vfio_nvidia_bar5_quirk_master_write, |
757 | .endianness = DEVICE_LITTLE_ENDIAN, |
758 | }; |
759 | |
760 | static uint64_t vfio_nvidia_bar5_quirk_enable_read(void *opaque, |
761 | hwaddr addr, unsigned size) |
762 | { |
763 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
764 | VFIOPCIDevice *vdev = bar5->window.vdev; |
765 | |
766 | return vfio_region_read(&vdev->bars[5].region, addr + 4, size); |
767 | } |
768 | |
769 | static void vfio_nvidia_bar5_quirk_enable_write(void *opaque, hwaddr addr, |
770 | uint64_t data, unsigned size) |
771 | { |
772 | VFIONvidiaBAR5Quirk *bar5 = opaque; |
773 | VFIOPCIDevice *vdev = bar5->window.vdev; |
774 | |
775 | vfio_region_write(&vdev->bars[5].region, addr + 4, data, size); |
776 | |
777 | bar5->enable = data; |
778 | vfio_nvidia_bar5_enable(bar5); |
779 | } |
780 | |
781 | static const MemoryRegionOps vfio_nvidia_bar5_quirk_enable = { |
782 | .read = vfio_nvidia_bar5_quirk_enable_read, |
783 | .write = vfio_nvidia_bar5_quirk_enable_write, |
784 | .endianness = DEVICE_LITTLE_ENDIAN, |
785 | }; |
786 | |
787 | static void vfio_probe_nvidia_bar5_quirk(VFIOPCIDevice *vdev, int nr) |
788 | { |
789 | VFIOQuirk *quirk; |
790 | VFIONvidiaBAR5Quirk *bar5; |
791 | VFIOConfigWindowQuirk *window; |
792 | |
793 | if (vdev->no_geforce_quirks || |
794 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || |
795 | !vdev->vga || nr != 5 || !vdev->bars[5].ioport) { |
796 | return; |
797 | } |
798 | |
799 | quirk = vfio_quirk_alloc(4); |
800 | bar5 = quirk->data = g_malloc0(sizeof(*bar5) + |
801 | (sizeof(VFIOConfigWindowMatch) * 2)); |
802 | window = &bar5->window; |
803 | |
804 | window->vdev = vdev; |
805 | window->address_offset = 0x8; |
806 | window->data_offset = 0xc; |
807 | window->nr_matches = 2; |
808 | window->matches[0].match = 0x1800; |
809 | window->matches[0].mask = PCI_CONFIG_SPACE_SIZE - 1; |
810 | window->matches[1].match = 0x88000; |
811 | window->matches[1].mask = vdev->config_size - 1; |
812 | window->bar = nr; |
813 | window->addr_mem = bar5->addr_mem = &quirk->mem[0]; |
814 | window->data_mem = bar5->data_mem = &quirk->mem[1]; |
815 | |
816 | memory_region_init_io(window->addr_mem, OBJECT(vdev), |
817 | &vfio_generic_window_address_quirk, window, |
818 | "vfio-nvidia-bar5-window-address-quirk" , 4); |
819 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
820 | window->address_offset, |
821 | window->addr_mem, 1); |
822 | memory_region_set_enabled(window->addr_mem, false); |
823 | |
824 | memory_region_init_io(window->data_mem, OBJECT(vdev), |
825 | &vfio_generic_window_data_quirk, window, |
826 | "vfio-nvidia-bar5-window-data-quirk" , 4); |
827 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
828 | window->data_offset, |
829 | window->data_mem, 1); |
830 | memory_region_set_enabled(window->data_mem, false); |
831 | |
832 | memory_region_init_io(&quirk->mem[2], OBJECT(vdev), |
833 | &vfio_nvidia_bar5_quirk_master, bar5, |
834 | "vfio-nvidia-bar5-master-quirk" , 4); |
835 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
836 | 0, &quirk->mem[2], 1); |
837 | |
838 | memory_region_init_io(&quirk->mem[3], OBJECT(vdev), |
839 | &vfio_nvidia_bar5_quirk_enable, bar5, |
840 | "vfio-nvidia-bar5-enable-quirk" , 4); |
841 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
842 | 4, &quirk->mem[3], 1); |
843 | |
844 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
845 | |
846 | trace_vfio_quirk_nvidia_bar5_probe(vdev->vbasedev.name); |
847 | } |
848 | |
849 | typedef struct LastDataSet { |
850 | VFIOQuirk *quirk; |
851 | hwaddr addr; |
852 | uint64_t data; |
853 | unsigned size; |
854 | int hits; |
855 | int added; |
856 | } LastDataSet; |
857 | |
858 | #define MAX_DYN_IOEVENTFD 10 |
859 | #define HITS_FOR_IOEVENTFD 10 |
860 | |
861 | /* |
862 | * Finally, BAR0 itself. We want to redirect any accesses to either |
863 | * 0x1800 or 0x88000 through the PCI config space access functions. |
864 | */ |
865 | static void vfio_nvidia_quirk_mirror_write(void *opaque, hwaddr addr, |
866 | uint64_t data, unsigned size) |
867 | { |
868 | VFIOConfigMirrorQuirk *mirror = opaque; |
869 | VFIOPCIDevice *vdev = mirror->vdev; |
870 | PCIDevice *pdev = &vdev->pdev; |
871 | LastDataSet *last = (LastDataSet *)&mirror->data; |
872 | |
873 | vfio_generic_quirk_mirror_write(opaque, addr, data, size); |
874 | |
875 | /* |
876 | * Nvidia seems to acknowledge MSI interrupts by writing 0xff to the |
877 | * MSI capability ID register. Both the ID and next register are |
878 | * read-only, so we allow writes covering either of those to real hw. |
879 | */ |
880 | if ((pdev->cap_present & QEMU_PCI_CAP_MSI) && |
881 | vfio_range_contained(addr, size, pdev->msi_cap, PCI_MSI_FLAGS)) { |
882 | vfio_region_write(&vdev->bars[mirror->bar].region, |
883 | addr + mirror->offset, data, size); |
884 | trace_vfio_quirk_nvidia_bar0_msi_ack(vdev->vbasedev.name); |
885 | } |
886 | |
887 | /* |
888 | * Automatically add an ioeventfd to handle any repeated write with the |
889 | * same data and size above the standard PCI config space header. This is |
890 | * primarily expected to accelerate the MSI-ACK behavior, such as noted |
891 | * above. Current hardware/drivers should trigger an ioeventfd at config |
892 | * offset 0x704 (region offset 0x88704), with data 0x0, size 4. |
893 | * |
894 | * The criteria of 10 successive hits is arbitrary but reliably adds the |
895 | * MSI-ACK region. Note that as some writes are bypassed via the ioeventfd, |
896 | * the remaining ones have a greater chance of being seen successively. |
897 | * To avoid the pathological case of burning up all of QEMU's open file |
898 | * handles, arbitrarily limit this algorithm from adding no more than 10 |
899 | * ioeventfds, print an error if we would have added an 11th, and then |
900 | * stop counting. |
901 | */ |
902 | if (!vdev->no_kvm_ioeventfd && |
903 | addr >= PCI_STD_HEADER_SIZEOF && last->added <= MAX_DYN_IOEVENTFD) { |
904 | if (addr != last->addr || data != last->data || size != last->size) { |
905 | last->addr = addr; |
906 | last->data = data; |
907 | last->size = size; |
908 | last->hits = 1; |
909 | } else if (++last->hits >= HITS_FOR_IOEVENTFD) { |
910 | if (last->added < MAX_DYN_IOEVENTFD) { |
911 | VFIOIOEventFD *ioeventfd; |
912 | ioeventfd = vfio_ioeventfd_init(vdev, mirror->mem, addr, size, |
913 | data, &vdev->bars[mirror->bar].region, |
914 | mirror->offset + addr, true); |
915 | if (ioeventfd) { |
916 | VFIOQuirk *quirk = last->quirk; |
917 | |
918 | QLIST_INSERT_HEAD(&quirk->ioeventfds, ioeventfd, next); |
919 | last->added++; |
920 | } |
921 | } else { |
922 | last->added++; |
923 | warn_report("NVIDIA ioeventfd queue full for %s, unable to " |
924 | "accelerate 0x%" HWADDR_PRIx", data 0x%" PRIx64", " |
925 | "size %u" , vdev->vbasedev.name, addr, data, size); |
926 | } |
927 | } |
928 | } |
929 | } |
930 | |
931 | static const MemoryRegionOps vfio_nvidia_mirror_quirk = { |
932 | .read = vfio_generic_quirk_mirror_read, |
933 | .write = vfio_nvidia_quirk_mirror_write, |
934 | .endianness = DEVICE_LITTLE_ENDIAN, |
935 | }; |
936 | |
937 | static void vfio_nvidia_bar0_quirk_reset(VFIOPCIDevice *vdev, VFIOQuirk *quirk) |
938 | { |
939 | VFIOConfigMirrorQuirk *mirror = quirk->data; |
940 | LastDataSet *last = (LastDataSet *)&mirror->data; |
941 | |
942 | last->addr = last->data = last->size = last->hits = last->added = 0; |
943 | |
944 | vfio_drop_dynamic_eventfds(vdev, quirk); |
945 | } |
946 | |
947 | static void vfio_probe_nvidia_bar0_quirk(VFIOPCIDevice *vdev, int nr) |
948 | { |
949 | VFIOQuirk *quirk; |
950 | VFIOConfigMirrorQuirk *mirror; |
951 | LastDataSet *last; |
952 | |
953 | if (vdev->no_geforce_quirks || |
954 | !vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID) || |
955 | !vfio_is_vga(vdev) || nr != 0) { |
956 | return; |
957 | } |
958 | |
959 | quirk = vfio_quirk_alloc(1); |
960 | quirk->reset = vfio_nvidia_bar0_quirk_reset; |
961 | mirror = quirk->data = g_malloc0(sizeof(*mirror) + sizeof(LastDataSet)); |
962 | mirror->mem = quirk->mem; |
963 | mirror->vdev = vdev; |
964 | mirror->offset = 0x88000; |
965 | mirror->bar = nr; |
966 | last = (LastDataSet *)&mirror->data; |
967 | last->quirk = quirk; |
968 | |
969 | memory_region_init_io(mirror->mem, OBJECT(vdev), |
970 | &vfio_nvidia_mirror_quirk, mirror, |
971 | "vfio-nvidia-bar0-88000-mirror-quirk" , |
972 | vdev->config_size); |
973 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
974 | mirror->offset, mirror->mem, 1); |
975 | |
976 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
977 | |
978 | /* The 0x1800 offset mirror only seems to get used by legacy VGA */ |
979 | if (vdev->vga) { |
980 | quirk = vfio_quirk_alloc(1); |
981 | quirk->reset = vfio_nvidia_bar0_quirk_reset; |
982 | mirror = quirk->data = g_malloc0(sizeof(*mirror) + sizeof(LastDataSet)); |
983 | mirror->mem = quirk->mem; |
984 | mirror->vdev = vdev; |
985 | mirror->offset = 0x1800; |
986 | mirror->bar = nr; |
987 | last = (LastDataSet *)&mirror->data; |
988 | last->quirk = quirk; |
989 | |
990 | memory_region_init_io(mirror->mem, OBJECT(vdev), |
991 | &vfio_nvidia_mirror_quirk, mirror, |
992 | "vfio-nvidia-bar0-1800-mirror-quirk" , |
993 | PCI_CONFIG_SPACE_SIZE); |
994 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
995 | mirror->offset, mirror->mem, 1); |
996 | |
997 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
998 | } |
999 | |
1000 | trace_vfio_quirk_nvidia_bar0_probe(vdev->vbasedev.name); |
1001 | } |
1002 | |
1003 | /* |
1004 | * TODO - Some Nvidia devices provide config access to their companion HDA |
1005 | * device and even to their parent bridge via these config space mirrors. |
1006 | * Add quirks for those regions. |
1007 | */ |
1008 | |
1009 | #define PCI_VENDOR_ID_REALTEK 0x10ec |
1010 | |
1011 | /* |
1012 | * RTL8168 devices have a backdoor that can access the MSI-X table. At BAR2 |
1013 | * offset 0x70 there is a dword data register, offset 0x74 is a dword address |
1014 | * register. According to the Linux r8169 driver, the MSI-X table is addressed |
1015 | * when the "type" portion of the address register is set to 0x1. This appears |
1016 | * to be bits 16:30. Bit 31 is both a write indicator and some sort of |
1017 | * "address latched" indicator. Bits 12:15 are a mask field, which we can |
1018 | * ignore because the MSI-X table should always be accessed as a dword (full |
1019 | * mask). Bits 0:11 is offset within the type. |
1020 | * |
1021 | * Example trace: |
1022 | * |
1023 | * Read from MSI-X table offset 0 |
1024 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x1f000, 4) // store read addr |
1025 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x8001f000 // latch |
1026 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x70, 4) = 0xfee00398 // read data |
1027 | * |
1028 | * Write 0xfee00000 to MSI-X table offset 0 |
1029 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x70, 0xfee00000, 4) // write data |
1030 | * vfio: vfio_bar_write(0000:05:00.0:BAR2+0x74, 0x8001f000, 4) // do write |
1031 | * vfio: vfio_bar_read(0000:05:00.0:BAR2+0x74, 4) = 0x1f000 // complete |
1032 | */ |
1033 | typedef struct VFIOrtl8168Quirk { |
1034 | VFIOPCIDevice *vdev; |
1035 | uint32_t addr; |
1036 | uint32_t data; |
1037 | bool enabled; |
1038 | } VFIOrtl8168Quirk; |
1039 | |
1040 | static uint64_t vfio_rtl8168_quirk_address_read(void *opaque, |
1041 | hwaddr addr, unsigned size) |
1042 | { |
1043 | VFIOrtl8168Quirk *rtl = opaque; |
1044 | VFIOPCIDevice *vdev = rtl->vdev; |
1045 | uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x74, size); |
1046 | |
1047 | if (rtl->enabled) { |
1048 | data = rtl->addr ^ 0x80000000U; /* latch/complete */ |
1049 | trace_vfio_quirk_rtl8168_fake_latch(vdev->vbasedev.name, data); |
1050 | } |
1051 | |
1052 | return data; |
1053 | } |
1054 | |
1055 | static void vfio_rtl8168_quirk_address_write(void *opaque, hwaddr addr, |
1056 | uint64_t data, unsigned size) |
1057 | { |
1058 | VFIOrtl8168Quirk *rtl = opaque; |
1059 | VFIOPCIDevice *vdev = rtl->vdev; |
1060 | |
1061 | rtl->enabled = false; |
1062 | |
1063 | if ((data & 0x7fff0000) == 0x10000) { /* MSI-X table */ |
1064 | rtl->enabled = true; |
1065 | rtl->addr = (uint32_t)data; |
1066 | |
1067 | if (data & 0x80000000U) { /* Do write */ |
1068 | if (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX) { |
1069 | hwaddr offset = data & 0xfff; |
1070 | uint64_t val = rtl->data; |
1071 | |
1072 | trace_vfio_quirk_rtl8168_msix_write(vdev->vbasedev.name, |
1073 | (uint16_t)offset, val); |
1074 | |
1075 | /* Write to the proper guest MSI-X table instead */ |
1076 | memory_region_dispatch_write(&vdev->pdev.msix_table_mmio, |
1077 | offset, val, |
1078 | size_memop(size) | MO_LE, |
1079 | MEMTXATTRS_UNSPECIFIED); |
1080 | } |
1081 | return; /* Do not write guest MSI-X data to hardware */ |
1082 | } |
1083 | } |
1084 | |
1085 | vfio_region_write(&vdev->bars[2].region, addr + 0x74, data, size); |
1086 | } |
1087 | |
1088 | static const MemoryRegionOps vfio_rtl_address_quirk = { |
1089 | .read = vfio_rtl8168_quirk_address_read, |
1090 | .write = vfio_rtl8168_quirk_address_write, |
1091 | .valid = { |
1092 | .min_access_size = 4, |
1093 | .max_access_size = 4, |
1094 | .unaligned = false, |
1095 | }, |
1096 | .endianness = DEVICE_LITTLE_ENDIAN, |
1097 | }; |
1098 | |
1099 | static uint64_t vfio_rtl8168_quirk_data_read(void *opaque, |
1100 | hwaddr addr, unsigned size) |
1101 | { |
1102 | VFIOrtl8168Quirk *rtl = opaque; |
1103 | VFIOPCIDevice *vdev = rtl->vdev; |
1104 | uint64_t data = vfio_region_read(&vdev->bars[2].region, addr + 0x70, size); |
1105 | |
1106 | if (rtl->enabled && (vdev->pdev.cap_present & QEMU_PCI_CAP_MSIX)) { |
1107 | hwaddr offset = rtl->addr & 0xfff; |
1108 | memory_region_dispatch_read(&vdev->pdev.msix_table_mmio, offset, |
1109 | &data, size_memop(size) | MO_LE, |
1110 | MEMTXATTRS_UNSPECIFIED); |
1111 | trace_vfio_quirk_rtl8168_msix_read(vdev->vbasedev.name, offset, data); |
1112 | } |
1113 | |
1114 | return data; |
1115 | } |
1116 | |
1117 | static void vfio_rtl8168_quirk_data_write(void *opaque, hwaddr addr, |
1118 | uint64_t data, unsigned size) |
1119 | { |
1120 | VFIOrtl8168Quirk *rtl = opaque; |
1121 | VFIOPCIDevice *vdev = rtl->vdev; |
1122 | |
1123 | rtl->data = (uint32_t)data; |
1124 | |
1125 | vfio_region_write(&vdev->bars[2].region, addr + 0x70, data, size); |
1126 | } |
1127 | |
1128 | static const MemoryRegionOps vfio_rtl_data_quirk = { |
1129 | .read = vfio_rtl8168_quirk_data_read, |
1130 | .write = vfio_rtl8168_quirk_data_write, |
1131 | .valid = { |
1132 | .min_access_size = 4, |
1133 | .max_access_size = 4, |
1134 | .unaligned = false, |
1135 | }, |
1136 | .endianness = DEVICE_LITTLE_ENDIAN, |
1137 | }; |
1138 | |
1139 | static void vfio_probe_rtl8168_bar2_quirk(VFIOPCIDevice *vdev, int nr) |
1140 | { |
1141 | VFIOQuirk *quirk; |
1142 | VFIOrtl8168Quirk *rtl; |
1143 | |
1144 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_REALTEK, 0x8168) || nr != 2) { |
1145 | return; |
1146 | } |
1147 | |
1148 | quirk = vfio_quirk_alloc(2); |
1149 | quirk->data = rtl = g_malloc0(sizeof(*rtl)); |
1150 | rtl->vdev = vdev; |
1151 | |
1152 | memory_region_init_io(&quirk->mem[0], OBJECT(vdev), |
1153 | &vfio_rtl_address_quirk, rtl, |
1154 | "vfio-rtl8168-window-address-quirk" , 4); |
1155 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
1156 | 0x74, &quirk->mem[0], 1); |
1157 | |
1158 | memory_region_init_io(&quirk->mem[1], OBJECT(vdev), |
1159 | &vfio_rtl_data_quirk, rtl, |
1160 | "vfio-rtl8168-window-data-quirk" , 4); |
1161 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
1162 | 0x70, &quirk->mem[1], 1); |
1163 | |
1164 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
1165 | |
1166 | trace_vfio_quirk_rtl8168_probe(vdev->vbasedev.name); |
1167 | } |
1168 | |
1169 | /* |
1170 | * Intel IGD support |
1171 | * |
1172 | * Obviously IGD is not a discrete device, this is evidenced not only by it |
1173 | * being integrated into the CPU, but by the various chipset and BIOS |
1174 | * dependencies that it brings along with it. Intel is trying to move away |
1175 | * from this and Broadwell and newer devices can run in what Intel calls |
1176 | * "Universal Pass-Through" mode, or UPT. Theoretically in UPT mode, nothing |
1177 | * more is required beyond assigning the IGD device to a VM. There are |
1178 | * however support limitations to this mode. It only supports IGD as a |
1179 | * secondary graphics device in the VM and it doesn't officially support any |
1180 | * physical outputs. |
1181 | * |
1182 | * The code here attempts to enable what we'll call legacy mode assignment, |
1183 | * IGD retains most of the capabilities we expect for it to have on bare |
1184 | * metal. To enable this mode, the IGD device must be assigned to the VM |
1185 | * at PCI address 00:02.0, it must have a ROM, it very likely needs VGA |
1186 | * support, we must have VM BIOS support for reserving and populating some |
1187 | * of the required tables, and we need to tweak the chipset with revisions |
1188 | * and IDs and an LPC/ISA bridge device. The intention is to make all of |
1189 | * this happen automatically by installing the device at the correct VM PCI |
1190 | * bus address. If any of the conditions are not met, we cross our fingers |
1191 | * and hope the user knows better. |
1192 | * |
1193 | * NB - It is possible to enable physical outputs in UPT mode by supplying |
1194 | * an OpRegion table. We don't do this by default because the guest driver |
1195 | * behaves differently if an OpRegion is provided and no monitor is attached |
1196 | * vs no OpRegion and a monitor being attached or not. Effectively, if a |
1197 | * headless setup is desired, the OpRegion gets in the way of that. |
1198 | */ |
1199 | |
1200 | /* |
1201 | * This presumes the device is already known to be an Intel VGA device, so we |
1202 | * take liberties in which device ID bits match which generation. This should |
1203 | * not be taken as an indication that all the devices are supported, or even |
1204 | * supportable, some of them don't even support VT-d. |
1205 | * See linux:include/drm/i915_pciids.h for IDs. |
1206 | */ |
1207 | static int igd_gen(VFIOPCIDevice *vdev) |
1208 | { |
1209 | if ((vdev->device_id & 0xfff) == 0xa84) { |
1210 | return 8; /* Broxton */ |
1211 | } |
1212 | |
1213 | switch (vdev->device_id & 0xff00) { |
1214 | /* Old, untested, unavailable, unknown */ |
1215 | case 0x0000: |
1216 | case 0x2500: |
1217 | case 0x2700: |
1218 | case 0x2900: |
1219 | case 0x2a00: |
1220 | case 0x2e00: |
1221 | case 0x3500: |
1222 | case 0xa000: |
1223 | return -1; |
1224 | /* SandyBridge, IvyBridge, ValleyView, Haswell */ |
1225 | case 0x0100: |
1226 | case 0x0400: |
1227 | case 0x0a00: |
1228 | case 0x0c00: |
1229 | case 0x0d00: |
1230 | case 0x0f00: |
1231 | return 6; |
1232 | /* BroadWell, CherryView, SkyLake, KabyLake */ |
1233 | case 0x1600: |
1234 | case 0x1900: |
1235 | case 0x2200: |
1236 | case 0x5900: |
1237 | return 8; |
1238 | } |
1239 | |
1240 | return 8; /* Assume newer is compatible */ |
1241 | } |
1242 | |
1243 | typedef struct VFIOIGDQuirk { |
1244 | struct VFIOPCIDevice *vdev; |
1245 | uint32_t index; |
1246 | uint32_t bdsm; |
1247 | } VFIOIGDQuirk; |
1248 | |
1249 | #define IGD_GMCH 0x50 /* Graphics Control Register */ |
1250 | #define IGD_BDSM 0x5c /* Base Data of Stolen Memory */ |
1251 | #define IGD_ASLS 0xfc /* ASL Storage Register */ |
1252 | |
1253 | /* |
1254 | * The OpRegion includes the Video BIOS Table, which seems important for |
1255 | * telling the driver what sort of outputs it has. Without this, the device |
1256 | * may work in the guest, but we may not get output. This also requires BIOS |
1257 | * support to reserve and populate a section of guest memory sufficient for |
1258 | * the table and to write the base address of that memory to the ASLS register |
1259 | * of the IGD device. |
1260 | */ |
1261 | int vfio_pci_igd_opregion_init(VFIOPCIDevice *vdev, |
1262 | struct vfio_region_info *info, Error **errp) |
1263 | { |
1264 | int ret; |
1265 | |
1266 | vdev->igd_opregion = g_malloc0(info->size); |
1267 | ret = pread(vdev->vbasedev.fd, vdev->igd_opregion, |
1268 | info->size, info->offset); |
1269 | if (ret != info->size) { |
1270 | error_setg(errp, "failed to read IGD OpRegion" ); |
1271 | g_free(vdev->igd_opregion); |
1272 | vdev->igd_opregion = NULL; |
1273 | return -EINVAL; |
1274 | } |
1275 | |
1276 | /* |
1277 | * Provide fw_cfg with a copy of the OpRegion which the VM firmware is to |
1278 | * allocate 32bit reserved memory for, copy these contents into, and write |
1279 | * the reserved memory base address to the device ASLS register at 0xFC. |
1280 | * Alignment of this reserved region seems flexible, but using a 4k page |
1281 | * alignment seems to work well. This interface assumes a single IGD |
1282 | * device, which may be at VM address 00:02.0 in legacy mode or another |
1283 | * address in UPT mode. |
1284 | * |
1285 | * NB, there may be future use cases discovered where the VM should have |
1286 | * direct interaction with the host OpRegion, in which case the write to |
1287 | * the ASLS register would trigger MemoryRegion setup to enable that. |
1288 | */ |
1289 | fw_cfg_add_file(fw_cfg_find(), "etc/igd-opregion" , |
1290 | vdev->igd_opregion, info->size); |
1291 | |
1292 | trace_vfio_pci_igd_opregion_enabled(vdev->vbasedev.name); |
1293 | |
1294 | pci_set_long(vdev->pdev.config + IGD_ASLS, 0); |
1295 | pci_set_long(vdev->pdev.wmask + IGD_ASLS, ~0); |
1296 | pci_set_long(vdev->emulated_config_bits + IGD_ASLS, ~0); |
1297 | |
1298 | return 0; |
1299 | } |
1300 | |
1301 | /* |
1302 | * The rather short list of registers that we copy from the host devices. |
1303 | * The LPC/ISA bridge values are definitely needed to support the vBIOS, the |
1304 | * host bridge values may or may not be needed depending on the guest OS. |
1305 | * Since we're only munging revision and subsystem values on the host bridge, |
1306 | * we don't require our own device. The LPC/ISA bridge needs to be our very |
1307 | * own though. |
1308 | */ |
1309 | typedef struct { |
1310 | uint8_t offset; |
1311 | uint8_t len; |
1312 | } IGDHostInfo; |
1313 | |
1314 | static const IGDHostInfo igd_host_bridge_infos[] = { |
1315 | {PCI_REVISION_ID, 2}, |
1316 | {PCI_SUBSYSTEM_VENDOR_ID, 2}, |
1317 | {PCI_SUBSYSTEM_ID, 2}, |
1318 | }; |
1319 | |
1320 | static const IGDHostInfo igd_lpc_bridge_infos[] = { |
1321 | {PCI_VENDOR_ID, 2}, |
1322 | {PCI_DEVICE_ID, 2}, |
1323 | {PCI_REVISION_ID, 2}, |
1324 | {PCI_SUBSYSTEM_VENDOR_ID, 2}, |
1325 | {PCI_SUBSYSTEM_ID, 2}, |
1326 | }; |
1327 | |
1328 | static int vfio_pci_igd_copy(VFIOPCIDevice *vdev, PCIDevice *pdev, |
1329 | struct vfio_region_info *info, |
1330 | const IGDHostInfo *list, int len) |
1331 | { |
1332 | int i, ret; |
1333 | |
1334 | for (i = 0; i < len; i++) { |
1335 | ret = pread(vdev->vbasedev.fd, pdev->config + list[i].offset, |
1336 | list[i].len, info->offset + list[i].offset); |
1337 | if (ret != list[i].len) { |
1338 | error_report("IGD copy failed: %m" ); |
1339 | return -errno; |
1340 | } |
1341 | } |
1342 | |
1343 | return 0; |
1344 | } |
1345 | |
1346 | /* |
1347 | * Stuff a few values into the host bridge. |
1348 | */ |
1349 | static int vfio_pci_igd_host_init(VFIOPCIDevice *vdev, |
1350 | struct vfio_region_info *info) |
1351 | { |
1352 | PCIBus *bus; |
1353 | PCIDevice *host_bridge; |
1354 | int ret; |
1355 | |
1356 | bus = pci_device_root_bus(&vdev->pdev); |
1357 | host_bridge = pci_find_device(bus, 0, PCI_DEVFN(0, 0)); |
1358 | |
1359 | if (!host_bridge) { |
1360 | error_report("Can't find host bridge" ); |
1361 | return -ENODEV; |
1362 | } |
1363 | |
1364 | ret = vfio_pci_igd_copy(vdev, host_bridge, info, igd_host_bridge_infos, |
1365 | ARRAY_SIZE(igd_host_bridge_infos)); |
1366 | if (!ret) { |
1367 | trace_vfio_pci_igd_host_bridge_enabled(vdev->vbasedev.name); |
1368 | } |
1369 | |
1370 | return ret; |
1371 | } |
1372 | |
1373 | /* |
1374 | * IGD LPC/ISA bridge support code. The vBIOS needs this, but we can't write |
1375 | * arbitrary values into just any bridge, so we must create our own. We try |
1376 | * to handle if the user has created it for us, which they might want to do |
1377 | * to enable multifunction so we don't occupy the whole PCI slot. |
1378 | */ |
1379 | static void vfio_pci_igd_lpc_bridge_realize(PCIDevice *pdev, Error **errp) |
1380 | { |
1381 | if (pdev->devfn != PCI_DEVFN(0x1f, 0)) { |
1382 | error_setg(errp, "VFIO dummy ISA/LPC bridge must have address 1f.0" ); |
1383 | } |
1384 | } |
1385 | |
1386 | static void vfio_pci_igd_lpc_bridge_class_init(ObjectClass *klass, void *data) |
1387 | { |
1388 | DeviceClass *dc = DEVICE_CLASS(klass); |
1389 | PCIDeviceClass *k = PCI_DEVICE_CLASS(klass); |
1390 | |
1391 | set_bit(DEVICE_CATEGORY_BRIDGE, dc->categories); |
1392 | dc->desc = "VFIO dummy ISA/LPC bridge for IGD assignment" ; |
1393 | dc->hotpluggable = false; |
1394 | k->realize = vfio_pci_igd_lpc_bridge_realize; |
1395 | k->class_id = PCI_CLASS_BRIDGE_ISA; |
1396 | } |
1397 | |
1398 | static TypeInfo vfio_pci_igd_lpc_bridge_info = { |
1399 | .name = "vfio-pci-igd-lpc-bridge" , |
1400 | .parent = TYPE_PCI_DEVICE, |
1401 | .class_init = vfio_pci_igd_lpc_bridge_class_init, |
1402 | .interfaces = (InterfaceInfo[]) { |
1403 | { INTERFACE_CONVENTIONAL_PCI_DEVICE }, |
1404 | { }, |
1405 | }, |
1406 | }; |
1407 | |
1408 | static void vfio_pci_igd_register_types(void) |
1409 | { |
1410 | type_register_static(&vfio_pci_igd_lpc_bridge_info); |
1411 | } |
1412 | |
1413 | type_init(vfio_pci_igd_register_types) |
1414 | |
1415 | static int vfio_pci_igd_lpc_init(VFIOPCIDevice *vdev, |
1416 | struct vfio_region_info *info) |
1417 | { |
1418 | PCIDevice *lpc_bridge; |
1419 | int ret; |
1420 | |
1421 | lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev), |
1422 | 0, PCI_DEVFN(0x1f, 0)); |
1423 | if (!lpc_bridge) { |
1424 | lpc_bridge = pci_create_simple(pci_device_root_bus(&vdev->pdev), |
1425 | PCI_DEVFN(0x1f, 0), "vfio-pci-igd-lpc-bridge" ); |
1426 | } |
1427 | |
1428 | ret = vfio_pci_igd_copy(vdev, lpc_bridge, info, igd_lpc_bridge_infos, |
1429 | ARRAY_SIZE(igd_lpc_bridge_infos)); |
1430 | if (!ret) { |
1431 | trace_vfio_pci_igd_lpc_bridge_enabled(vdev->vbasedev.name); |
1432 | } |
1433 | |
1434 | return ret; |
1435 | } |
1436 | |
1437 | /* |
1438 | * IGD Gen8 and newer support up to 8MB for the GTT and use a 64bit PTE |
1439 | * entry, older IGDs use 2MB and 32bit. Each PTE maps a 4k page. Therefore |
1440 | * we either have 2M/4k * 4 = 2k or 8M/4k * 8 = 16k as the maximum iobar index |
1441 | * for programming the GTT. |
1442 | * |
1443 | * See linux:include/drm/i915_drm.h for shift and mask values. |
1444 | */ |
1445 | static int vfio_igd_gtt_max(VFIOPCIDevice *vdev) |
1446 | { |
1447 | uint32_t gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch)); |
1448 | int ggms, gen = igd_gen(vdev); |
1449 | |
1450 | gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, sizeof(gmch)); |
1451 | ggms = (gmch >> (gen < 8 ? 8 : 6)) & 0x3; |
1452 | if (gen > 6) { |
1453 | ggms = 1 << ggms; |
1454 | } |
1455 | |
1456 | ggms *= MiB; |
1457 | |
1458 | return (ggms / (4 * KiB)) * (gen < 8 ? 4 : 8); |
1459 | } |
1460 | |
1461 | /* |
1462 | * The IGD ROM will make use of stolen memory (GGMS) for support of VESA modes. |
1463 | * Somehow the host stolen memory range is used for this, but how the ROM gets |
1464 | * it is a mystery, perhaps it's hardcoded into the ROM. Thankfully though, it |
1465 | * reprograms the GTT through the IOBAR where we can trap it and transpose the |
1466 | * programming to the VM allocated buffer. That buffer gets reserved by the VM |
1467 | * firmware via the fw_cfg entry added below. Here we're just monitoring the |
1468 | * IOBAR address and data registers to detect a write sequence targeting the |
1469 | * GTTADR. This code is developed by observed behavior and doesn't have a |
1470 | * direct spec reference, unfortunately. |
1471 | */ |
1472 | static uint64_t vfio_igd_quirk_data_read(void *opaque, |
1473 | hwaddr addr, unsigned size) |
1474 | { |
1475 | VFIOIGDQuirk *igd = opaque; |
1476 | VFIOPCIDevice *vdev = igd->vdev; |
1477 | |
1478 | igd->index = ~0; |
1479 | |
1480 | return vfio_region_read(&vdev->bars[4].region, addr + 4, size); |
1481 | } |
1482 | |
1483 | static void vfio_igd_quirk_data_write(void *opaque, hwaddr addr, |
1484 | uint64_t data, unsigned size) |
1485 | { |
1486 | VFIOIGDQuirk *igd = opaque; |
1487 | VFIOPCIDevice *vdev = igd->vdev; |
1488 | uint64_t val = data; |
1489 | int gen = igd_gen(vdev); |
1490 | |
1491 | /* |
1492 | * Programming the GGMS starts at index 0x1 and uses every 4th index (ie. |
1493 | * 0x1, 0x5, 0x9, 0xd,...). For pre-Gen8 each 4-byte write is a whole PTE |
1494 | * entry, with 0th bit enable set. For Gen8 and up, PTEs are 64bit, so |
1495 | * entries 0x5 & 0xd are the high dword, in our case zero. Each PTE points |
1496 | * to a 4k page, which we translate to a page from the VM allocated region, |
1497 | * pointed to by the BDSM register. If this is not set, we fail. |
1498 | * |
1499 | * We trap writes to the full configured GTT size, but we typically only |
1500 | * see the vBIOS writing up to (nearly) the 1MB barrier. In fact it often |
1501 | * seems to miss the last entry for an even 1MB GTT. Doing a gratuitous |
1502 | * write of that last entry does work, but is hopefully unnecessary since |
1503 | * we clear the previous GTT on initialization. |
1504 | */ |
1505 | if ((igd->index % 4 == 1) && igd->index < vfio_igd_gtt_max(vdev)) { |
1506 | if (gen < 8 || (igd->index % 8 == 1)) { |
1507 | uint32_t base; |
1508 | |
1509 | base = pci_get_long(vdev->pdev.config + IGD_BDSM); |
1510 | if (!base) { |
1511 | hw_error("vfio-igd: Guest attempted to program IGD GTT before " |
1512 | "BIOS reserved stolen memory. Unsupported BIOS?" ); |
1513 | } |
1514 | |
1515 | val = data - igd->bdsm + base; |
1516 | } else { |
1517 | val = 0; /* upper 32bits of pte, we only enable below 4G PTEs */ |
1518 | } |
1519 | |
1520 | trace_vfio_pci_igd_bar4_write(vdev->vbasedev.name, |
1521 | igd->index, data, val); |
1522 | } |
1523 | |
1524 | vfio_region_write(&vdev->bars[4].region, addr + 4, val, size); |
1525 | |
1526 | igd->index = ~0; |
1527 | } |
1528 | |
1529 | static const MemoryRegionOps vfio_igd_data_quirk = { |
1530 | .read = vfio_igd_quirk_data_read, |
1531 | .write = vfio_igd_quirk_data_write, |
1532 | .endianness = DEVICE_LITTLE_ENDIAN, |
1533 | }; |
1534 | |
1535 | static uint64_t vfio_igd_quirk_index_read(void *opaque, |
1536 | hwaddr addr, unsigned size) |
1537 | { |
1538 | VFIOIGDQuirk *igd = opaque; |
1539 | VFIOPCIDevice *vdev = igd->vdev; |
1540 | |
1541 | igd->index = ~0; |
1542 | |
1543 | return vfio_region_read(&vdev->bars[4].region, addr, size); |
1544 | } |
1545 | |
1546 | static void vfio_igd_quirk_index_write(void *opaque, hwaddr addr, |
1547 | uint64_t data, unsigned size) |
1548 | { |
1549 | VFIOIGDQuirk *igd = opaque; |
1550 | VFIOPCIDevice *vdev = igd->vdev; |
1551 | |
1552 | igd->index = data; |
1553 | |
1554 | vfio_region_write(&vdev->bars[4].region, addr, data, size); |
1555 | } |
1556 | |
1557 | static const MemoryRegionOps vfio_igd_index_quirk = { |
1558 | .read = vfio_igd_quirk_index_read, |
1559 | .write = vfio_igd_quirk_index_write, |
1560 | .endianness = DEVICE_LITTLE_ENDIAN, |
1561 | }; |
1562 | |
1563 | static void vfio_probe_igd_bar4_quirk(VFIOPCIDevice *vdev, int nr) |
1564 | { |
1565 | struct vfio_region_info *rom = NULL, *opregion = NULL, |
1566 | *host = NULL, *lpc = NULL; |
1567 | VFIOQuirk *quirk; |
1568 | VFIOIGDQuirk *igd; |
1569 | PCIDevice *lpc_bridge; |
1570 | int i, ret, ggms_mb, gms_mb = 0, gen; |
1571 | uint64_t *bdsm_size; |
1572 | uint32_t gmch; |
1573 | uint16_t cmd_orig, cmd; |
1574 | Error *err = NULL; |
1575 | |
1576 | /* |
1577 | * This must be an Intel VGA device at address 00:02.0 for us to even |
1578 | * consider enabling legacy mode. The vBIOS has dependencies on the |
1579 | * PCI bus address. |
1580 | */ |
1581 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_INTEL, PCI_ANY_ID) || |
1582 | !vfio_is_vga(vdev) || nr != 4 || |
1583 | &vdev->pdev != pci_find_device(pci_device_root_bus(&vdev->pdev), |
1584 | 0, PCI_DEVFN(0x2, 0))) { |
1585 | return; |
1586 | } |
1587 | |
1588 | /* |
1589 | * We need to create an LPC/ISA bridge at PCI bus address 00:1f.0 that we |
1590 | * can stuff host values into, so if there's already one there and it's not |
1591 | * one we can hack on, legacy mode is no-go. Sorry Q35. |
1592 | */ |
1593 | lpc_bridge = pci_find_device(pci_device_root_bus(&vdev->pdev), |
1594 | 0, PCI_DEVFN(0x1f, 0)); |
1595 | if (lpc_bridge && !object_dynamic_cast(OBJECT(lpc_bridge), |
1596 | "vfio-pci-igd-lpc-bridge" )) { |
1597 | error_report("IGD device %s cannot support legacy mode due to existing " |
1598 | "devices at address 1f.0" , vdev->vbasedev.name); |
1599 | return; |
1600 | } |
1601 | |
1602 | /* |
1603 | * IGD is not a standard, they like to change their specs often. We |
1604 | * only attempt to support back to SandBridge and we hope that newer |
1605 | * devices maintain compatibility with generation 8. |
1606 | */ |
1607 | gen = igd_gen(vdev); |
1608 | if (gen != 6 && gen != 8) { |
1609 | error_report("IGD device %s is unsupported in legacy mode, " |
1610 | "try SandyBridge or newer" , vdev->vbasedev.name); |
1611 | return; |
1612 | } |
1613 | |
1614 | /* |
1615 | * Most of what we're doing here is to enable the ROM to run, so if |
1616 | * there's no ROM, there's no point in setting up this quirk. |
1617 | * NB. We only seem to get BIOS ROMs, so a UEFI VM would need CSM support. |
1618 | */ |
1619 | ret = vfio_get_region_info(&vdev->vbasedev, |
1620 | VFIO_PCI_ROM_REGION_INDEX, &rom); |
1621 | if ((ret || !rom->size) && !vdev->pdev.romfile) { |
1622 | error_report("IGD device %s has no ROM, legacy mode disabled" , |
1623 | vdev->vbasedev.name); |
1624 | goto out; |
1625 | } |
1626 | |
1627 | /* |
1628 | * Ignore the hotplug corner case, mark the ROM failed, we can't |
1629 | * create the devices we need for legacy mode in the hotplug scenario. |
1630 | */ |
1631 | if (vdev->pdev.qdev.hotplugged) { |
1632 | error_report("IGD device %s hotplugged, ROM disabled, " |
1633 | "legacy mode disabled" , vdev->vbasedev.name); |
1634 | vdev->rom_read_failed = true; |
1635 | goto out; |
1636 | } |
1637 | |
1638 | /* |
1639 | * Check whether we have all the vfio device specific regions to |
1640 | * support legacy mode (added in Linux v4.6). If not, bail. |
1641 | */ |
1642 | ret = vfio_get_dev_region_info(&vdev->vbasedev, |
1643 | VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, |
1644 | VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion); |
1645 | if (ret) { |
1646 | error_report("IGD device %s does not support OpRegion access," |
1647 | "legacy mode disabled" , vdev->vbasedev.name); |
1648 | goto out; |
1649 | } |
1650 | |
1651 | ret = vfio_get_dev_region_info(&vdev->vbasedev, |
1652 | VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, |
1653 | VFIO_REGION_SUBTYPE_INTEL_IGD_HOST_CFG, &host); |
1654 | if (ret) { |
1655 | error_report("IGD device %s does not support host bridge access," |
1656 | "legacy mode disabled" , vdev->vbasedev.name); |
1657 | goto out; |
1658 | } |
1659 | |
1660 | ret = vfio_get_dev_region_info(&vdev->vbasedev, |
1661 | VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL, |
1662 | VFIO_REGION_SUBTYPE_INTEL_IGD_LPC_CFG, &lpc); |
1663 | if (ret) { |
1664 | error_report("IGD device %s does not support LPC bridge access," |
1665 | "legacy mode disabled" , vdev->vbasedev.name); |
1666 | goto out; |
1667 | } |
1668 | |
1669 | gmch = vfio_pci_read_config(&vdev->pdev, IGD_GMCH, 4); |
1670 | |
1671 | /* |
1672 | * If IGD VGA Disable is clear (expected) and VGA is not already enabled, |
1673 | * try to enable it. Probably shouldn't be using legacy mode without VGA, |
1674 | * but also no point in us enabling VGA if disabled in hardware. |
1675 | */ |
1676 | if (!(gmch & 0x2) && !vdev->vga && vfio_populate_vga(vdev, &err)) { |
1677 | error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); |
1678 | error_report("IGD device %s failed to enable VGA access, " |
1679 | "legacy mode disabled" , vdev->vbasedev.name); |
1680 | goto out; |
1681 | } |
1682 | |
1683 | /* Create our LPC/ISA bridge */ |
1684 | ret = vfio_pci_igd_lpc_init(vdev, lpc); |
1685 | if (ret) { |
1686 | error_report("IGD device %s failed to create LPC bridge, " |
1687 | "legacy mode disabled" , vdev->vbasedev.name); |
1688 | goto out; |
1689 | } |
1690 | |
1691 | /* Stuff some host values into the VM PCI host bridge */ |
1692 | ret = vfio_pci_igd_host_init(vdev, host); |
1693 | if (ret) { |
1694 | error_report("IGD device %s failed to modify host bridge, " |
1695 | "legacy mode disabled" , vdev->vbasedev.name); |
1696 | goto out; |
1697 | } |
1698 | |
1699 | /* Setup OpRegion access */ |
1700 | ret = vfio_pci_igd_opregion_init(vdev, opregion, &err); |
1701 | if (ret) { |
1702 | error_append_hint(&err, "IGD legacy mode disabled\n" ); |
1703 | error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name); |
1704 | goto out; |
1705 | } |
1706 | |
1707 | /* Setup our quirk to munge GTT addresses to the VM allocated buffer */ |
1708 | quirk = vfio_quirk_alloc(2); |
1709 | igd = quirk->data = g_malloc0(sizeof(*igd)); |
1710 | igd->vdev = vdev; |
1711 | igd->index = ~0; |
1712 | igd->bdsm = vfio_pci_read_config(&vdev->pdev, IGD_BDSM, 4); |
1713 | igd->bdsm &= ~((1 * MiB) - 1); /* 1MB aligned */ |
1714 | |
1715 | memory_region_init_io(&quirk->mem[0], OBJECT(vdev), &vfio_igd_index_quirk, |
1716 | igd, "vfio-igd-index-quirk" , 4); |
1717 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
1718 | 0, &quirk->mem[0], 1); |
1719 | |
1720 | memory_region_init_io(&quirk->mem[1], OBJECT(vdev), &vfio_igd_data_quirk, |
1721 | igd, "vfio-igd-data-quirk" , 4); |
1722 | memory_region_add_subregion_overlap(vdev->bars[nr].region.mem, |
1723 | 4, &quirk->mem[1], 1); |
1724 | |
1725 | QLIST_INSERT_HEAD(&vdev->bars[nr].quirks, quirk, next); |
1726 | |
1727 | /* Determine the size of stolen memory needed for GTT */ |
1728 | ggms_mb = (gmch >> (gen < 8 ? 8 : 6)) & 0x3; |
1729 | if (gen > 6) { |
1730 | ggms_mb = 1 << ggms_mb; |
1731 | } |
1732 | |
1733 | /* |
1734 | * Assume we have no GMS memory, but allow it to be overrided by device |
1735 | * option (experimental). The spec doesn't actually allow zero GMS when |
1736 | * when IVD (IGD VGA Disable) is clear, but the claim is that it's unused, |
1737 | * so let's not waste VM memory for it. |
1738 | */ |
1739 | gmch &= ~((gen < 8 ? 0x1f : 0xff) << (gen < 8 ? 3 : 8)); |
1740 | |
1741 | if (vdev->igd_gms) { |
1742 | if (vdev->igd_gms <= 0x10) { |
1743 | gms_mb = vdev->igd_gms * 32; |
1744 | gmch |= vdev->igd_gms << (gen < 8 ? 3 : 8); |
1745 | } else { |
1746 | error_report("Unsupported IGD GMS value 0x%x" , vdev->igd_gms); |
1747 | vdev->igd_gms = 0; |
1748 | } |
1749 | } |
1750 | |
1751 | /* |
1752 | * Request reserved memory for stolen memory via fw_cfg. VM firmware |
1753 | * must allocate a 1MB aligned reserved memory region below 4GB with |
1754 | * the requested size (in bytes) for use by the Intel PCI class VGA |
1755 | * device at VM address 00:02.0. The base address of this reserved |
1756 | * memory region must be written to the device BDSM regsiter at PCI |
1757 | * config offset 0x5C. |
1758 | */ |
1759 | bdsm_size = g_malloc(sizeof(*bdsm_size)); |
1760 | *bdsm_size = cpu_to_le64((ggms_mb + gms_mb) * MiB); |
1761 | fw_cfg_add_file(fw_cfg_find(), "etc/igd-bdsm-size" , |
1762 | bdsm_size, sizeof(*bdsm_size)); |
1763 | |
1764 | /* GMCH is read-only, emulated */ |
1765 | pci_set_long(vdev->pdev.config + IGD_GMCH, gmch); |
1766 | pci_set_long(vdev->pdev.wmask + IGD_GMCH, 0); |
1767 | pci_set_long(vdev->emulated_config_bits + IGD_GMCH, ~0); |
1768 | |
1769 | /* BDSM is read-write, emulated. The BIOS needs to be able to write it */ |
1770 | pci_set_long(vdev->pdev.config + IGD_BDSM, 0); |
1771 | pci_set_long(vdev->pdev.wmask + IGD_BDSM, ~0); |
1772 | pci_set_long(vdev->emulated_config_bits + IGD_BDSM, ~0); |
1773 | |
1774 | /* |
1775 | * This IOBAR gives us access to GTTADR, which allows us to write to |
1776 | * the GTT itself. So let's go ahead and write zero to all the GTT |
1777 | * entries to avoid spurious DMA faults. Be sure I/O access is enabled |
1778 | * before talking to the device. |
1779 | */ |
1780 | if (pread(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig), |
1781 | vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) { |
1782 | error_report("IGD device %s - failed to read PCI command register" , |
1783 | vdev->vbasedev.name); |
1784 | } |
1785 | |
1786 | cmd = cmd_orig | PCI_COMMAND_IO; |
1787 | |
1788 | if (pwrite(vdev->vbasedev.fd, &cmd, sizeof(cmd), |
1789 | vdev->config_offset + PCI_COMMAND) != sizeof(cmd)) { |
1790 | error_report("IGD device %s - failed to write PCI command register" , |
1791 | vdev->vbasedev.name); |
1792 | } |
1793 | |
1794 | for (i = 1; i < vfio_igd_gtt_max(vdev); i += 4) { |
1795 | vfio_region_write(&vdev->bars[4].region, 0, i, 4); |
1796 | vfio_region_write(&vdev->bars[4].region, 4, 0, 4); |
1797 | } |
1798 | |
1799 | if (pwrite(vdev->vbasedev.fd, &cmd_orig, sizeof(cmd_orig), |
1800 | vdev->config_offset + PCI_COMMAND) != sizeof(cmd_orig)) { |
1801 | error_report("IGD device %s - failed to restore PCI command register" , |
1802 | vdev->vbasedev.name); |
1803 | } |
1804 | |
1805 | trace_vfio_pci_igd_bdsm_enabled(vdev->vbasedev.name, ggms_mb + gms_mb); |
1806 | |
1807 | out: |
1808 | g_free(rom); |
1809 | g_free(opregion); |
1810 | g_free(host); |
1811 | g_free(lpc); |
1812 | } |
1813 | |
1814 | /* |
1815 | * Common quirk probe entry points. |
1816 | */ |
1817 | void vfio_vga_quirk_setup(VFIOPCIDevice *vdev) |
1818 | { |
1819 | vfio_vga_probe_ati_3c3_quirk(vdev); |
1820 | vfio_vga_probe_nvidia_3d0_quirk(vdev); |
1821 | } |
1822 | |
1823 | void vfio_vga_quirk_exit(VFIOPCIDevice *vdev) |
1824 | { |
1825 | VFIOQuirk *quirk; |
1826 | int i, j; |
1827 | |
1828 | for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) { |
1829 | QLIST_FOREACH(quirk, &vdev->vga->region[i].quirks, next) { |
1830 | for (j = 0; j < quirk->nr_mem; j++) { |
1831 | memory_region_del_subregion(&vdev->vga->region[i].mem, |
1832 | &quirk->mem[j]); |
1833 | } |
1834 | } |
1835 | } |
1836 | } |
1837 | |
1838 | void vfio_vga_quirk_finalize(VFIOPCIDevice *vdev) |
1839 | { |
1840 | int i, j; |
1841 | |
1842 | for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) { |
1843 | while (!QLIST_EMPTY(&vdev->vga->region[i].quirks)) { |
1844 | VFIOQuirk *quirk = QLIST_FIRST(&vdev->vga->region[i].quirks); |
1845 | QLIST_REMOVE(quirk, next); |
1846 | for (j = 0; j < quirk->nr_mem; j++) { |
1847 | object_unparent(OBJECT(&quirk->mem[j])); |
1848 | } |
1849 | g_free(quirk->mem); |
1850 | g_free(quirk->data); |
1851 | g_free(quirk); |
1852 | } |
1853 | } |
1854 | } |
1855 | |
1856 | void vfio_bar_quirk_setup(VFIOPCIDevice *vdev, int nr) |
1857 | { |
1858 | vfio_probe_ati_bar4_quirk(vdev, nr); |
1859 | vfio_probe_ati_bar2_quirk(vdev, nr); |
1860 | vfio_probe_nvidia_bar5_quirk(vdev, nr); |
1861 | vfio_probe_nvidia_bar0_quirk(vdev, nr); |
1862 | vfio_probe_rtl8168_bar2_quirk(vdev, nr); |
1863 | vfio_probe_igd_bar4_quirk(vdev, nr); |
1864 | } |
1865 | |
1866 | void vfio_bar_quirk_exit(VFIOPCIDevice *vdev, int nr) |
1867 | { |
1868 | VFIOBAR *bar = &vdev->bars[nr]; |
1869 | VFIOQuirk *quirk; |
1870 | int i; |
1871 | |
1872 | QLIST_FOREACH(quirk, &bar->quirks, next) { |
1873 | while (!QLIST_EMPTY(&quirk->ioeventfds)) { |
1874 | vfio_ioeventfd_exit(vdev, QLIST_FIRST(&quirk->ioeventfds)); |
1875 | } |
1876 | |
1877 | for (i = 0; i < quirk->nr_mem; i++) { |
1878 | memory_region_del_subregion(bar->region.mem, &quirk->mem[i]); |
1879 | } |
1880 | } |
1881 | } |
1882 | |
1883 | void vfio_bar_quirk_finalize(VFIOPCIDevice *vdev, int nr) |
1884 | { |
1885 | VFIOBAR *bar = &vdev->bars[nr]; |
1886 | int i; |
1887 | |
1888 | while (!QLIST_EMPTY(&bar->quirks)) { |
1889 | VFIOQuirk *quirk = QLIST_FIRST(&bar->quirks); |
1890 | QLIST_REMOVE(quirk, next); |
1891 | for (i = 0; i < quirk->nr_mem; i++) { |
1892 | object_unparent(OBJECT(&quirk->mem[i])); |
1893 | } |
1894 | g_free(quirk->mem); |
1895 | g_free(quirk->data); |
1896 | g_free(quirk); |
1897 | } |
1898 | } |
1899 | |
1900 | /* |
1901 | * Reset quirks |
1902 | */ |
1903 | void vfio_quirk_reset(VFIOPCIDevice *vdev) |
1904 | { |
1905 | int i; |
1906 | |
1907 | for (i = 0; i < PCI_ROM_SLOT; i++) { |
1908 | VFIOQuirk *quirk; |
1909 | VFIOBAR *bar = &vdev->bars[i]; |
1910 | |
1911 | QLIST_FOREACH(quirk, &bar->quirks, next) { |
1912 | if (quirk->reset) { |
1913 | quirk->reset(vdev, quirk); |
1914 | } |
1915 | } |
1916 | } |
1917 | } |
1918 | |
1919 | /* |
1920 | * AMD Radeon PCI config reset, based on Linux: |
1921 | * drivers/gpu/drm/radeon/ci_smc.c:ci_is_smc_running() |
1922 | * drivers/gpu/drm/radeon/radeon_device.c:radeon_pci_config_reset |
1923 | * drivers/gpu/drm/radeon/ci_smc.c:ci_reset_smc() |
1924 | * drivers/gpu/drm/radeon/ci_smc.c:ci_stop_smc_clock() |
1925 | * IDs: include/drm/drm_pciids.h |
1926 | * Registers: http://cgit.freedesktop.org/~agd5f/linux/commit/?id=4e2aa447f6f0 |
1927 | * |
1928 | * Bonaire and Hawaii GPUs do not respond to a bus reset. This is a bug in the |
1929 | * hardware that should be fixed on future ASICs. The symptom of this is that |
1930 | * once the accerlated driver loads, Windows guests will bsod on subsequent |
1931 | * attmpts to load the driver, such as after VM reset or shutdown/restart. To |
1932 | * work around this, we do an AMD specific PCI config reset, followed by an SMC |
1933 | * reset. The PCI config reset only works if SMC firmware is running, so we |
1934 | * have a dependency on the state of the device as to whether this reset will |
1935 | * be effective. There are still cases where we won't be able to kick the |
1936 | * device into working, but this greatly improves the usability overall. The |
1937 | * config reset magic is relatively common on AMD GPUs, but the setup and SMC |
1938 | * poking is largely ASIC specific. |
1939 | */ |
1940 | static bool vfio_radeon_smc_is_running(VFIOPCIDevice *vdev) |
1941 | { |
1942 | uint32_t clk, pc_c; |
1943 | |
1944 | /* |
1945 | * Registers 200h and 204h are index and data registers for accessing |
1946 | * indirect configuration registers within the device. |
1947 | */ |
1948 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4); |
1949 | clk = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
1950 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000370, 4); |
1951 | pc_c = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
1952 | |
1953 | return (!(clk & 1) && (0x20100 <= pc_c)); |
1954 | } |
1955 | |
1956 | /* |
1957 | * The scope of a config reset is controlled by a mode bit in the misc register |
1958 | * and a fuse, exposed as a bit in another register. The fuse is the default |
1959 | * (0 = GFX, 1 = whole GPU), the misc bit is a toggle, with the forumula |
1960 | * scope = !(misc ^ fuse), where the resulting scope is defined the same as |
1961 | * the fuse. A truth table therefore tells us that if misc == fuse, we need |
1962 | * to flip the value of the bit in the misc register. |
1963 | */ |
1964 | static void vfio_radeon_set_gfx_only_reset(VFIOPCIDevice *vdev) |
1965 | { |
1966 | uint32_t misc, fuse; |
1967 | bool a, b; |
1968 | |
1969 | vfio_region_write(&vdev->bars[5].region, 0x200, 0xc00c0000, 4); |
1970 | fuse = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
1971 | b = fuse & 64; |
1972 | |
1973 | vfio_region_write(&vdev->bars[5].region, 0x200, 0xc0000010, 4); |
1974 | misc = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
1975 | a = misc & 2; |
1976 | |
1977 | if (a == b) { |
1978 | vfio_region_write(&vdev->bars[5].region, 0x204, misc ^ 2, 4); |
1979 | vfio_region_read(&vdev->bars[5].region, 0x204, 4); /* flush */ |
1980 | } |
1981 | } |
1982 | |
1983 | static int vfio_radeon_reset(VFIOPCIDevice *vdev) |
1984 | { |
1985 | PCIDevice *pdev = &vdev->pdev; |
1986 | int i, ret = 0; |
1987 | uint32_t data; |
1988 | |
1989 | /* Defer to a kernel implemented reset */ |
1990 | if (vdev->vbasedev.reset_works) { |
1991 | trace_vfio_quirk_ati_bonaire_reset_skipped(vdev->vbasedev.name); |
1992 | return -ENODEV; |
1993 | } |
1994 | |
1995 | /* Enable only memory BAR access */ |
1996 | vfio_pci_write_config(pdev, PCI_COMMAND, PCI_COMMAND_MEMORY, 2); |
1997 | |
1998 | /* Reset only works if SMC firmware is loaded and running */ |
1999 | if (!vfio_radeon_smc_is_running(vdev)) { |
2000 | ret = -EINVAL; |
2001 | trace_vfio_quirk_ati_bonaire_reset_no_smc(vdev->vbasedev.name); |
2002 | goto out; |
2003 | } |
2004 | |
2005 | /* Make sure only the GFX function is reset */ |
2006 | vfio_radeon_set_gfx_only_reset(vdev); |
2007 | |
2008 | /* AMD PCI config reset */ |
2009 | vfio_pci_write_config(pdev, 0x7c, 0x39d5e86b, 4); |
2010 | usleep(100); |
2011 | |
2012 | /* Read back the memory size to make sure we're out of reset */ |
2013 | for (i = 0; i < 100000; i++) { |
2014 | if (vfio_region_read(&vdev->bars[5].region, 0x5428, 4) != 0xffffffff) { |
2015 | goto reset_smc; |
2016 | } |
2017 | usleep(1); |
2018 | } |
2019 | |
2020 | trace_vfio_quirk_ati_bonaire_reset_timeout(vdev->vbasedev.name); |
2021 | |
2022 | reset_smc: |
2023 | /* Reset SMC */ |
2024 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000000, 4); |
2025 | data = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
2026 | data |= 1; |
2027 | vfio_region_write(&vdev->bars[5].region, 0x204, data, 4); |
2028 | |
2029 | /* Disable SMC clock */ |
2030 | vfio_region_write(&vdev->bars[5].region, 0x200, 0x80000004, 4); |
2031 | data = vfio_region_read(&vdev->bars[5].region, 0x204, 4); |
2032 | data |= 1; |
2033 | vfio_region_write(&vdev->bars[5].region, 0x204, data, 4); |
2034 | |
2035 | trace_vfio_quirk_ati_bonaire_reset_done(vdev->vbasedev.name); |
2036 | |
2037 | out: |
2038 | /* Restore PCI command register */ |
2039 | vfio_pci_write_config(pdev, PCI_COMMAND, 0, 2); |
2040 | |
2041 | return ret; |
2042 | } |
2043 | |
2044 | void vfio_setup_resetfn_quirk(VFIOPCIDevice *vdev) |
2045 | { |
2046 | switch (vdev->vendor_id) { |
2047 | case 0x1002: |
2048 | switch (vdev->device_id) { |
2049 | /* Bonaire */ |
2050 | case 0x6649: /* Bonaire [FirePro W5100] */ |
2051 | case 0x6650: |
2052 | case 0x6651: |
2053 | case 0x6658: /* Bonaire XTX [Radeon R7 260X] */ |
2054 | case 0x665c: /* Bonaire XT [Radeon HD 7790/8770 / R9 260 OEM] */ |
2055 | case 0x665d: /* Bonaire [Radeon R7 200 Series] */ |
2056 | /* Hawaii */ |
2057 | case 0x67A0: /* Hawaii XT GL [FirePro W9100] */ |
2058 | case 0x67A1: /* Hawaii PRO GL [FirePro W8100] */ |
2059 | case 0x67A2: |
2060 | case 0x67A8: |
2061 | case 0x67A9: |
2062 | case 0x67AA: |
2063 | case 0x67B0: /* Hawaii XT [Radeon R9 290X] */ |
2064 | case 0x67B1: /* Hawaii PRO [Radeon R9 290] */ |
2065 | case 0x67B8: |
2066 | case 0x67B9: |
2067 | case 0x67BA: |
2068 | case 0x67BE: |
2069 | vdev->resetfn = vfio_radeon_reset; |
2070 | trace_vfio_quirk_ati_bonaire_reset(vdev->vbasedev.name); |
2071 | break; |
2072 | } |
2073 | break; |
2074 | } |
2075 | } |
2076 | |
2077 | /* |
2078 | * The NVIDIA GPUDirect P2P Vendor capability allows the user to specify |
2079 | * devices as a member of a clique. Devices within the same clique ID |
2080 | * are capable of direct P2P. It's the user's responsibility that this |
2081 | * is correct. The spec says that this may reside at any unused config |
2082 | * offset, but reserves and recommends hypervisors place this at C8h. |
2083 | * The spec also states that the hypervisor should place this capability |
2084 | * at the end of the capability list, thus next is defined as 0h. |
2085 | * |
2086 | * +----------------+----------------+----------------+----------------+ |
2087 | * | sig 7:0 ('P') | vndr len (8h) | next (0h) | cap id (9h) | |
2088 | * +----------------+----------------+----------------+----------------+ |
2089 | * | rsvd 15:7(0h),id 6:3,ver 2:0(0h)| sig 23:8 ('P2') | |
2090 | * +---------------------------------+---------------------------------+ |
2091 | * |
2092 | * https://lists.gnu.org/archive/html/qemu-devel/2017-08/pdfUda5iEpgOS.pdf |
2093 | */ |
2094 | static void get_nv_gpudirect_clique_id(Object *obj, Visitor *v, |
2095 | const char *name, void *opaque, |
2096 | Error **errp) |
2097 | { |
2098 | DeviceState *dev = DEVICE(obj); |
2099 | Property *prop = opaque; |
2100 | uint8_t *ptr = qdev_get_prop_ptr(dev, prop); |
2101 | |
2102 | visit_type_uint8(v, name, ptr, errp); |
2103 | } |
2104 | |
2105 | static void set_nv_gpudirect_clique_id(Object *obj, Visitor *v, |
2106 | const char *name, void *opaque, |
2107 | Error **errp) |
2108 | { |
2109 | DeviceState *dev = DEVICE(obj); |
2110 | Property *prop = opaque; |
2111 | uint8_t value, *ptr = qdev_get_prop_ptr(dev, prop); |
2112 | Error *local_err = NULL; |
2113 | |
2114 | if (dev->realized) { |
2115 | qdev_prop_set_after_realize(dev, name, errp); |
2116 | return; |
2117 | } |
2118 | |
2119 | visit_type_uint8(v, name, &value, &local_err); |
2120 | if (local_err) { |
2121 | error_propagate(errp, local_err); |
2122 | return; |
2123 | } |
2124 | |
2125 | if (value & ~0xF) { |
2126 | error_setg(errp, "Property %s: valid range 0-15" , name); |
2127 | return; |
2128 | } |
2129 | |
2130 | *ptr = value; |
2131 | } |
2132 | |
2133 | const PropertyInfo qdev_prop_nv_gpudirect_clique = { |
2134 | .name = "uint4" , |
2135 | .description = "NVIDIA GPUDirect Clique ID (0 - 15)" , |
2136 | .get = get_nv_gpudirect_clique_id, |
2137 | .set = set_nv_gpudirect_clique_id, |
2138 | }; |
2139 | |
2140 | static int vfio_add_nv_gpudirect_cap(VFIOPCIDevice *vdev, Error **errp) |
2141 | { |
2142 | PCIDevice *pdev = &vdev->pdev; |
2143 | int ret, pos = 0xC8; |
2144 | |
2145 | if (vdev->nv_gpudirect_clique == 0xFF) { |
2146 | return 0; |
2147 | } |
2148 | |
2149 | if (!vfio_pci_is(vdev, PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID)) { |
2150 | error_setg(errp, "NVIDIA GPUDirect Clique ID: invalid device vendor" ); |
2151 | return -EINVAL; |
2152 | } |
2153 | |
2154 | if (pci_get_byte(pdev->config + PCI_CLASS_DEVICE + 1) != |
2155 | PCI_BASE_CLASS_DISPLAY) { |
2156 | error_setg(errp, "NVIDIA GPUDirect Clique ID: unsupported PCI class" ); |
2157 | return -EINVAL; |
2158 | } |
2159 | |
2160 | ret = pci_add_capability(pdev, PCI_CAP_ID_VNDR, pos, 8, errp); |
2161 | if (ret < 0) { |
2162 | error_prepend(errp, "Failed to add NVIDIA GPUDirect cap: " ); |
2163 | return ret; |
2164 | } |
2165 | |
2166 | memset(vdev->emulated_config_bits + pos, 0xFF, 8); |
2167 | pos += PCI_CAP_FLAGS; |
2168 | pci_set_byte(pdev->config + pos++, 8); |
2169 | pci_set_byte(pdev->config + pos++, 'P'); |
2170 | pci_set_byte(pdev->config + pos++, '2'); |
2171 | pci_set_byte(pdev->config + pos++, 'P'); |
2172 | pci_set_byte(pdev->config + pos++, vdev->nv_gpudirect_clique << 3); |
2173 | pci_set_byte(pdev->config + pos, 0); |
2174 | |
2175 | return 0; |
2176 | } |
2177 | |
2178 | int vfio_add_virt_caps(VFIOPCIDevice *vdev, Error **errp) |
2179 | { |
2180 | int ret; |
2181 | |
2182 | ret = vfio_add_nv_gpudirect_cap(vdev, errp); |
2183 | if (ret) { |
2184 | return ret; |
2185 | } |
2186 | |
2187 | return 0; |
2188 | } |
2189 | |
2190 | static void vfio_pci_nvlink2_get_tgt(Object *obj, Visitor *v, |
2191 | const char *name, |
2192 | void *opaque, Error **errp) |
2193 | { |
2194 | uint64_t tgt = (uintptr_t) opaque; |
2195 | visit_type_uint64(v, name, &tgt, errp); |
2196 | } |
2197 | |
2198 | static void vfio_pci_nvlink2_get_link_speed(Object *obj, Visitor *v, |
2199 | const char *name, |
2200 | void *opaque, Error **errp) |
2201 | { |
2202 | uint32_t link_speed = (uint32_t)(uintptr_t) opaque; |
2203 | visit_type_uint32(v, name, &link_speed, errp); |
2204 | } |
2205 | |
2206 | int vfio_pci_nvidia_v100_ram_init(VFIOPCIDevice *vdev, Error **errp) |
2207 | { |
2208 | int ret; |
2209 | void *p; |
2210 | struct vfio_region_info *nv2reg = NULL; |
2211 | struct vfio_info_cap_header *hdr; |
2212 | struct vfio_region_info_cap_nvlink2_ssatgt *cap; |
2213 | VFIOQuirk *quirk; |
2214 | |
2215 | ret = vfio_get_dev_region_info(&vdev->vbasedev, |
2216 | VFIO_REGION_TYPE_PCI_VENDOR_TYPE | |
2217 | PCI_VENDOR_ID_NVIDIA, |
2218 | VFIO_REGION_SUBTYPE_NVIDIA_NVLINK2_RAM, |
2219 | &nv2reg); |
2220 | if (ret) { |
2221 | return ret; |
2222 | } |
2223 | |
2224 | hdr = vfio_get_region_info_cap(nv2reg, VFIO_REGION_INFO_CAP_NVLINK2_SSATGT); |
2225 | if (!hdr) { |
2226 | ret = -ENODEV; |
2227 | goto free_exit; |
2228 | } |
2229 | cap = (void *) hdr; |
2230 | |
2231 | p = mmap(NULL, nv2reg->size, PROT_READ | PROT_WRITE | PROT_EXEC, |
2232 | MAP_SHARED, vdev->vbasedev.fd, nv2reg->offset); |
2233 | if (p == MAP_FAILED) { |
2234 | ret = -errno; |
2235 | goto free_exit; |
2236 | } |
2237 | |
2238 | quirk = vfio_quirk_alloc(1); |
2239 | memory_region_init_ram_ptr(&quirk->mem[0], OBJECT(vdev), "nvlink2-mr" , |
2240 | nv2reg->size, p); |
2241 | QLIST_INSERT_HEAD(&vdev->bars[0].quirks, quirk, next); |
2242 | |
2243 | object_property_add(OBJECT(vdev), "nvlink2-tgt" , "uint64" , |
2244 | vfio_pci_nvlink2_get_tgt, NULL, NULL, |
2245 | (void *) (uintptr_t) cap->tgt, NULL); |
2246 | trace_vfio_pci_nvidia_gpu_setup_quirk(vdev->vbasedev.name, cap->tgt, |
2247 | nv2reg->size); |
2248 | free_exit: |
2249 | g_free(nv2reg); |
2250 | |
2251 | return ret; |
2252 | } |
2253 | |
2254 | int vfio_pci_nvlink2_init(VFIOPCIDevice *vdev, Error **errp) |
2255 | { |
2256 | int ret; |
2257 | void *p; |
2258 | struct vfio_region_info *atsdreg = NULL; |
2259 | struct vfio_info_cap_header *hdr; |
2260 | struct vfio_region_info_cap_nvlink2_ssatgt *captgt; |
2261 | struct vfio_region_info_cap_nvlink2_lnkspd *capspeed; |
2262 | VFIOQuirk *quirk; |
2263 | |
2264 | ret = vfio_get_dev_region_info(&vdev->vbasedev, |
2265 | VFIO_REGION_TYPE_PCI_VENDOR_TYPE | |
2266 | PCI_VENDOR_ID_IBM, |
2267 | VFIO_REGION_SUBTYPE_IBM_NVLINK2_ATSD, |
2268 | &atsdreg); |
2269 | if (ret) { |
2270 | return ret; |
2271 | } |
2272 | |
2273 | hdr = vfio_get_region_info_cap(atsdreg, |
2274 | VFIO_REGION_INFO_CAP_NVLINK2_SSATGT); |
2275 | if (!hdr) { |
2276 | ret = -ENODEV; |
2277 | goto free_exit; |
2278 | } |
2279 | captgt = (void *) hdr; |
2280 | |
2281 | hdr = vfio_get_region_info_cap(atsdreg, |
2282 | VFIO_REGION_INFO_CAP_NVLINK2_LNKSPD); |
2283 | if (!hdr) { |
2284 | ret = -ENODEV; |
2285 | goto free_exit; |
2286 | } |
2287 | capspeed = (void *) hdr; |
2288 | |
2289 | /* Some NVLink bridges may not have assigned ATSD */ |
2290 | if (atsdreg->size) { |
2291 | p = mmap(NULL, atsdreg->size, PROT_READ | PROT_WRITE | PROT_EXEC, |
2292 | MAP_SHARED, vdev->vbasedev.fd, atsdreg->offset); |
2293 | if (p == MAP_FAILED) { |
2294 | ret = -errno; |
2295 | goto free_exit; |
2296 | } |
2297 | |
2298 | quirk = vfio_quirk_alloc(1); |
2299 | memory_region_init_ram_device_ptr(&quirk->mem[0], OBJECT(vdev), |
2300 | "nvlink2-atsd-mr" , atsdreg->size, p); |
2301 | QLIST_INSERT_HEAD(&vdev->bars[0].quirks, quirk, next); |
2302 | } |
2303 | |
2304 | object_property_add(OBJECT(vdev), "nvlink2-tgt" , "uint64" , |
2305 | vfio_pci_nvlink2_get_tgt, NULL, NULL, |
2306 | (void *) (uintptr_t) captgt->tgt, NULL); |
2307 | trace_vfio_pci_nvlink2_setup_quirk_ssatgt(vdev->vbasedev.name, captgt->tgt, |
2308 | atsdreg->size); |
2309 | |
2310 | object_property_add(OBJECT(vdev), "nvlink2-link-speed" , "uint32" , |
2311 | vfio_pci_nvlink2_get_link_speed, NULL, NULL, |
2312 | (void *) (uintptr_t) capspeed->link_speed, NULL); |
2313 | trace_vfio_pci_nvlink2_setup_quirk_lnkspd(vdev->vbasedev.name, |
2314 | capspeed->link_speed); |
2315 | free_exit: |
2316 | g_free(atsdreg); |
2317 | |
2318 | return ret; |
2319 | } |
2320 | |