1/*
2 * vfio based device assignment support
3 *
4 * Copyright Red Hat, Inc. 2012
5 *
6 * Authors:
7 * Alex Williamson <alex.williamson@redhat.com>
8 *
9 * This work is licensed under the terms of the GNU GPL, version 2. See
10 * the COPYING file in the top-level directory.
11 *
12 * Based on qemu-kvm device-assignment:
13 * Adapted for KVM by Qumranet.
14 * Copyright (c) 2007, Neocleus, Alex Novik (alex@neocleus.com)
15 * Copyright (c) 2007, Neocleus, Guy Zana (guy@neocleus.com)
16 * Copyright (C) 2008, Qumranet, Amit Shah (amit.shah@qumranet.com)
17 * Copyright (C) 2008, Red Hat, Amit Shah (amit.shah@redhat.com)
18 * Copyright (C) 2008, IBM, Muli Ben-Yehuda (muli@il.ibm.com)
19 */
20
21#include "qemu/osdep.h"
22#include <linux/vfio.h>
23#include <sys/ioctl.h>
24
25#include "hw/hw.h"
26#include "hw/pci/msi.h"
27#include "hw/pci/msix.h"
28#include "hw/pci/pci_bridge.h"
29#include "hw/qdev-properties.h"
30#include "migration/vmstate.h"
31#include "qemu/error-report.h"
32#include "qemu/main-loop.h"
33#include "qemu/module.h"
34#include "qemu/option.h"
35#include "qemu/range.h"
36#include "qemu/units.h"
37#include "sysemu/kvm.h"
38#include "sysemu/runstate.h"
39#include "sysemu/sysemu.h"
40#include "pci.h"
41#include "trace.h"
42#include "qapi/error.h"
43
44#define TYPE_VFIO_PCI "vfio-pci"
45#define PCI_VFIO(obj) OBJECT_CHECK(VFIOPCIDevice, obj, TYPE_VFIO_PCI)
46
47#define TYPE_VIFO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
48
49static void vfio_disable_interrupts(VFIOPCIDevice *vdev);
50static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled);
51
52/*
53 * Disabling BAR mmaping can be slow, but toggling it around INTx can
54 * also be a huge overhead. We try to get the best of both worlds by
55 * waiting until an interrupt to disable mmaps (subsequent transitions
56 * to the same state are effectively no overhead). If the interrupt has
57 * been serviced and the time gap is long enough, we re-enable mmaps for
58 * performance. This works well for things like graphics cards, which
59 * may not use their interrupt at all and are penalized to an unusable
60 * level by read/write BAR traps. Other devices, like NICs, have more
61 * regular interrupts and see much better latency by staying in non-mmap
62 * mode. We therefore set the default mmap_timeout such that a ping
63 * is just enough to keep the mmap disabled. Users can experiment with
64 * other options with the x-intx-mmap-timeout-ms parameter (a value of
65 * zero disables the timer).
66 */
67static void vfio_intx_mmap_enable(void *opaque)
68{
69 VFIOPCIDevice *vdev = opaque;
70
71 if (vdev->intx.pending) {
72 timer_mod(vdev->intx.mmap_timer,
73 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
74 return;
75 }
76
77 vfio_mmap_set_enabled(vdev, true);
78}
79
80static void vfio_intx_interrupt(void *opaque)
81{
82 VFIOPCIDevice *vdev = opaque;
83
84 if (!event_notifier_test_and_clear(&vdev->intx.interrupt)) {
85 return;
86 }
87
88 trace_vfio_intx_interrupt(vdev->vbasedev.name, 'A' + vdev->intx.pin);
89
90 vdev->intx.pending = true;
91 pci_irq_assert(&vdev->pdev);
92 vfio_mmap_set_enabled(vdev, false);
93 if (vdev->intx.mmap_timeout) {
94 timer_mod(vdev->intx.mmap_timer,
95 qemu_clock_get_ms(QEMU_CLOCK_VIRTUAL) + vdev->intx.mmap_timeout);
96 }
97}
98
99static void vfio_intx_eoi(VFIODevice *vbasedev)
100{
101 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
102
103 if (!vdev->intx.pending) {
104 return;
105 }
106
107 trace_vfio_intx_eoi(vbasedev->name);
108
109 vdev->intx.pending = false;
110 pci_irq_deassert(&vdev->pdev);
111 vfio_unmask_single_irqindex(vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
112}
113
114static void vfio_intx_enable_kvm(VFIOPCIDevice *vdev, Error **errp)
115{
116#ifdef CONFIG_KVM
117 struct kvm_irqfd irqfd = {
118 .fd = event_notifier_get_fd(&vdev->intx.interrupt),
119 .gsi = vdev->intx.route.irq,
120 .flags = KVM_IRQFD_FLAG_RESAMPLE,
121 };
122 Error *err = NULL;
123
124 if (vdev->no_kvm_intx || !kvm_irqfds_enabled() ||
125 vdev->intx.route.mode != PCI_INTX_ENABLED ||
126 !kvm_resamplefds_enabled()) {
127 return;
128 }
129
130 /* Get to a known interrupt state */
131 qemu_set_fd_handler(irqfd.fd, NULL, NULL, vdev);
132 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
133 vdev->intx.pending = false;
134 pci_irq_deassert(&vdev->pdev);
135
136 /* Get an eventfd for resample/unmask */
137 if (event_notifier_init(&vdev->intx.unmask, 0)) {
138 error_setg(errp, "event_notifier_init failed eoi");
139 goto fail;
140 }
141
142 /* KVM triggers it, VFIO listens for it */
143 irqfd.resamplefd = event_notifier_get_fd(&vdev->intx.unmask);
144
145 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
146 error_setg_errno(errp, errno, "failed to setup resample irqfd");
147 goto fail_irqfd;
148 }
149
150 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
151 VFIO_IRQ_SET_ACTION_UNMASK,
152 irqfd.resamplefd, &err)) {
153 error_propagate(errp, err);
154 goto fail_vfio;
155 }
156
157 /* Let'em rip */
158 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
159
160 vdev->intx.kvm_accel = true;
161
162 trace_vfio_intx_enable_kvm(vdev->vbasedev.name);
163
164 return;
165
166fail_vfio:
167 irqfd.flags = KVM_IRQFD_FLAG_DEASSIGN;
168 kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd);
169fail_irqfd:
170 event_notifier_cleanup(&vdev->intx.unmask);
171fail:
172 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
173 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
174#endif
175}
176
177static void vfio_intx_disable_kvm(VFIOPCIDevice *vdev)
178{
179#ifdef CONFIG_KVM
180 struct kvm_irqfd irqfd = {
181 .fd = event_notifier_get_fd(&vdev->intx.interrupt),
182 .gsi = vdev->intx.route.irq,
183 .flags = KVM_IRQFD_FLAG_DEASSIGN,
184 };
185
186 if (!vdev->intx.kvm_accel) {
187 return;
188 }
189
190 /*
191 * Get to a known state, hardware masked, QEMU ready to accept new
192 * interrupts, QEMU IRQ de-asserted.
193 */
194 vfio_mask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
195 vdev->intx.pending = false;
196 pci_irq_deassert(&vdev->pdev);
197
198 /* Tell KVM to stop listening for an INTx irqfd */
199 if (kvm_vm_ioctl(kvm_state, KVM_IRQFD, &irqfd)) {
200 error_report("vfio: Error: Failed to disable INTx irqfd: %m");
201 }
202
203 /* We only need to close the eventfd for VFIO to cleanup the kernel side */
204 event_notifier_cleanup(&vdev->intx.unmask);
205
206 /* QEMU starts listening for interrupt events. */
207 qemu_set_fd_handler(irqfd.fd, vfio_intx_interrupt, NULL, vdev);
208
209 vdev->intx.kvm_accel = false;
210
211 /* If we've missed an event, let it re-fire through QEMU */
212 vfio_unmask_single_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
213
214 trace_vfio_intx_disable_kvm(vdev->vbasedev.name);
215#endif
216}
217
218static void vfio_intx_update(PCIDevice *pdev)
219{
220 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
221 PCIINTxRoute route;
222 Error *err = NULL;
223
224 if (vdev->interrupt != VFIO_INT_INTx) {
225 return;
226 }
227
228 route = pci_device_route_intx_to_irq(&vdev->pdev, vdev->intx.pin);
229
230 if (!pci_intx_route_changed(&vdev->intx.route, &route)) {
231 return; /* Nothing changed */
232 }
233
234 trace_vfio_intx_update(vdev->vbasedev.name,
235 vdev->intx.route.irq, route.irq);
236
237 vfio_intx_disable_kvm(vdev);
238
239 vdev->intx.route = route;
240
241 if (route.mode != PCI_INTX_ENABLED) {
242 return;
243 }
244
245 vfio_intx_enable_kvm(vdev, &err);
246 if (err) {
247 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
248 }
249
250 /* Re-enable the interrupt in cased we missed an EOI */
251 vfio_intx_eoi(&vdev->vbasedev);
252}
253
254static int vfio_intx_enable(VFIOPCIDevice *vdev, Error **errp)
255{
256 uint8_t pin = vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1);
257 Error *err = NULL;
258 int32_t fd;
259 int ret;
260
261
262 if (!pin) {
263 return 0;
264 }
265
266 vfio_disable_interrupts(vdev);
267
268 vdev->intx.pin = pin - 1; /* Pin A (1) -> irq[0] */
269 pci_config_set_interrupt_pin(vdev->pdev.config, pin);
270
271#ifdef CONFIG_KVM
272 /*
273 * Only conditional to avoid generating error messages on platforms
274 * where we won't actually use the result anyway.
275 */
276 if (kvm_irqfds_enabled() && kvm_resamplefds_enabled()) {
277 vdev->intx.route = pci_device_route_intx_to_irq(&vdev->pdev,
278 vdev->intx.pin);
279 }
280#endif
281
282 ret = event_notifier_init(&vdev->intx.interrupt, 0);
283 if (ret) {
284 error_setg_errno(errp, -ret, "event_notifier_init failed");
285 return ret;
286 }
287 fd = event_notifier_get_fd(&vdev->intx.interrupt);
288 qemu_set_fd_handler(fd, vfio_intx_interrupt, NULL, vdev);
289
290 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX, 0,
291 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
292 error_propagate(errp, err);
293 qemu_set_fd_handler(fd, NULL, NULL, vdev);
294 event_notifier_cleanup(&vdev->intx.interrupt);
295 return -errno;
296 }
297
298 vfio_intx_enable_kvm(vdev, &err);
299 if (err) {
300 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
301 }
302
303 vdev->interrupt = VFIO_INT_INTx;
304
305 trace_vfio_intx_enable(vdev->vbasedev.name);
306 return 0;
307}
308
309static void vfio_intx_disable(VFIOPCIDevice *vdev)
310{
311 int fd;
312
313 timer_del(vdev->intx.mmap_timer);
314 vfio_intx_disable_kvm(vdev);
315 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_INTX_IRQ_INDEX);
316 vdev->intx.pending = false;
317 pci_irq_deassert(&vdev->pdev);
318 vfio_mmap_set_enabled(vdev, true);
319
320 fd = event_notifier_get_fd(&vdev->intx.interrupt);
321 qemu_set_fd_handler(fd, NULL, NULL, vdev);
322 event_notifier_cleanup(&vdev->intx.interrupt);
323
324 vdev->interrupt = VFIO_INT_NONE;
325
326 trace_vfio_intx_disable(vdev->vbasedev.name);
327}
328
329/*
330 * MSI/X
331 */
332static void vfio_msi_interrupt(void *opaque)
333{
334 VFIOMSIVector *vector = opaque;
335 VFIOPCIDevice *vdev = vector->vdev;
336 MSIMessage (*get_msg)(PCIDevice *dev, unsigned vector);
337 void (*notify)(PCIDevice *dev, unsigned vector);
338 MSIMessage msg;
339 int nr = vector - vdev->msi_vectors;
340
341 if (!event_notifier_test_and_clear(&vector->interrupt)) {
342 return;
343 }
344
345 if (vdev->interrupt == VFIO_INT_MSIX) {
346 get_msg = msix_get_message;
347 notify = msix_notify;
348
349 /* A masked vector firing needs to use the PBA, enable it */
350 if (msix_is_masked(&vdev->pdev, nr)) {
351 set_bit(nr, vdev->msix->pending);
352 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, true);
353 trace_vfio_msix_pba_enable(vdev->vbasedev.name);
354 }
355 } else if (vdev->interrupt == VFIO_INT_MSI) {
356 get_msg = msi_get_message;
357 notify = msi_notify;
358 } else {
359 abort();
360 }
361
362 msg = get_msg(&vdev->pdev, nr);
363 trace_vfio_msi_interrupt(vdev->vbasedev.name, nr, msg.address, msg.data);
364 notify(&vdev->pdev, nr);
365}
366
367static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool msix)
368{
369 struct vfio_irq_set *irq_set;
370 int ret = 0, i, argsz;
371 int32_t *fds;
372
373 argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
374
375 irq_set = g_malloc0(argsz);
376 irq_set->argsz = argsz;
377 irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
378 irq_set->index = msix ? VFIO_PCI_MSIX_IRQ_INDEX : VFIO_PCI_MSI_IRQ_INDEX;
379 irq_set->start = 0;
380 irq_set->count = vdev->nr_vectors;
381 fds = (int32_t *)&irq_set->data;
382
383 for (i = 0; i < vdev->nr_vectors; i++) {
384 int fd = -1;
385
386 /*
387 * MSI vs MSI-X - The guest has direct access to MSI mask and pending
388 * bits, therefore we always use the KVM signaling path when setup.
389 * MSI-X mask and pending bits are emulated, so we want to use the
390 * KVM signaling path only when configured and unmasked.
391 */
392 if (vdev->msi_vectors[i].use) {
393 if (vdev->msi_vectors[i].virq < 0 ||
394 (msix && msix_is_masked(&vdev->pdev, i))) {
395 fd = event_notifier_get_fd(&vdev->msi_vectors[i].interrupt);
396 } else {
397 fd = event_notifier_get_fd(&vdev->msi_vectors[i].kvm_interrupt);
398 }
399 }
400
401 fds[i] = fd;
402 }
403
404 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_SET_IRQS, irq_set);
405
406 g_free(irq_set);
407
408 return ret;
409}
410
411static void vfio_add_kvm_msi_virq(VFIOPCIDevice *vdev, VFIOMSIVector *vector,
412 int vector_n, bool msix)
413{
414 int virq;
415
416 if ((msix && vdev->no_kvm_msix) || (!msix && vdev->no_kvm_msi)) {
417 return;
418 }
419
420 if (event_notifier_init(&vector->kvm_interrupt, 0)) {
421 return;
422 }
423
424 virq = kvm_irqchip_add_msi_route(kvm_state, vector_n, &vdev->pdev);
425 if (virq < 0) {
426 event_notifier_cleanup(&vector->kvm_interrupt);
427 return;
428 }
429
430 if (kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
431 NULL, virq) < 0) {
432 kvm_irqchip_release_virq(kvm_state, virq);
433 event_notifier_cleanup(&vector->kvm_interrupt);
434 return;
435 }
436
437 vector->virq = virq;
438}
439
440static void vfio_remove_kvm_msi_virq(VFIOMSIVector *vector)
441{
442 kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, &vector->kvm_interrupt,
443 vector->virq);
444 kvm_irqchip_release_virq(kvm_state, vector->virq);
445 vector->virq = -1;
446 event_notifier_cleanup(&vector->kvm_interrupt);
447}
448
449static void vfio_update_kvm_msi_virq(VFIOMSIVector *vector, MSIMessage msg,
450 PCIDevice *pdev)
451{
452 kvm_irqchip_update_msi_route(kvm_state, vector->virq, msg, pdev);
453 kvm_irqchip_commit_routes(kvm_state);
454}
455
456static int vfio_msix_vector_do_use(PCIDevice *pdev, unsigned int nr,
457 MSIMessage *msg, IOHandler *handler)
458{
459 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
460 VFIOMSIVector *vector;
461 int ret;
462
463 trace_vfio_msix_vector_do_use(vdev->vbasedev.name, nr);
464
465 vector = &vdev->msi_vectors[nr];
466
467 if (!vector->use) {
468 vector->vdev = vdev;
469 vector->virq = -1;
470 if (event_notifier_init(&vector->interrupt, 0)) {
471 error_report("vfio: Error: event_notifier_init failed");
472 }
473 vector->use = true;
474 msix_vector_use(pdev, nr);
475 }
476
477 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
478 handler, NULL, vector);
479
480 /*
481 * Attempt to enable route through KVM irqchip,
482 * default to userspace handling if unavailable.
483 */
484 if (vector->virq >= 0) {
485 if (!msg) {
486 vfio_remove_kvm_msi_virq(vector);
487 } else {
488 vfio_update_kvm_msi_virq(vector, *msg, pdev);
489 }
490 } else {
491 if (msg) {
492 vfio_add_kvm_msi_virq(vdev, vector, nr, true);
493 }
494 }
495
496 /*
497 * We don't want to have the host allocate all possible MSI vectors
498 * for a device if they're not in use, so we shutdown and incrementally
499 * increase them as needed.
500 */
501 if (vdev->nr_vectors < nr + 1) {
502 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
503 vdev->nr_vectors = nr + 1;
504 ret = vfio_enable_vectors(vdev, true);
505 if (ret) {
506 error_report("vfio: failed to enable vectors, %d", ret);
507 }
508 } else {
509 Error *err = NULL;
510 int32_t fd;
511
512 if (vector->virq >= 0) {
513 fd = event_notifier_get_fd(&vector->kvm_interrupt);
514 } else {
515 fd = event_notifier_get_fd(&vector->interrupt);
516 }
517
518 if (vfio_set_irq_signaling(&vdev->vbasedev,
519 VFIO_PCI_MSIX_IRQ_INDEX, nr,
520 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
521 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
522 }
523 }
524
525 /* Disable PBA emulation when nothing more is pending. */
526 clear_bit(nr, vdev->msix->pending);
527 if (find_first_bit(vdev->msix->pending,
528 vdev->nr_vectors) == vdev->nr_vectors) {
529 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
530 trace_vfio_msix_pba_disable(vdev->vbasedev.name);
531 }
532
533 return 0;
534}
535
536static int vfio_msix_vector_use(PCIDevice *pdev,
537 unsigned int nr, MSIMessage msg)
538{
539 return vfio_msix_vector_do_use(pdev, nr, &msg, vfio_msi_interrupt);
540}
541
542static void vfio_msix_vector_release(PCIDevice *pdev, unsigned int nr)
543{
544 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
545 VFIOMSIVector *vector = &vdev->msi_vectors[nr];
546
547 trace_vfio_msix_vector_release(vdev->vbasedev.name, nr);
548
549 /*
550 * There are still old guests that mask and unmask vectors on every
551 * interrupt. If we're using QEMU bypass with a KVM irqfd, leave all of
552 * the KVM setup in place, simply switch VFIO to use the non-bypass
553 * eventfd. We'll then fire the interrupt through QEMU and the MSI-X
554 * core will mask the interrupt and set pending bits, allowing it to
555 * be re-asserted on unmask. Nothing to do if already using QEMU mode.
556 */
557 if (vector->virq >= 0) {
558 int32_t fd = event_notifier_get_fd(&vector->interrupt);
559 Error *err = NULL;
560
561 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX, nr,
562 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
563 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
564 }
565 }
566}
567
568static void vfio_msix_enable(VFIOPCIDevice *vdev)
569{
570 vfio_disable_interrupts(vdev);
571
572 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->msix->entries);
573
574 vdev->interrupt = VFIO_INT_MSIX;
575
576 /*
577 * Some communication channels between VF & PF or PF & fw rely on the
578 * physical state of the device and expect that enabling MSI-X from the
579 * guest enables the same on the host. When our guest is Linux, the
580 * guest driver call to pci_enable_msix() sets the enabling bit in the
581 * MSI-X capability, but leaves the vector table masked. We therefore
582 * can't rely on a vector_use callback (from request_irq() in the guest)
583 * to switch the physical device into MSI-X mode because that may come a
584 * long time after pci_enable_msix(). This code enables vector 0 with
585 * triggering to userspace, then immediately release the vector, leaving
586 * the physical device with no vectors enabled, but MSI-X enabled, just
587 * like the guest view.
588 */
589 vfio_msix_vector_do_use(&vdev->pdev, 0, NULL, NULL);
590 vfio_msix_vector_release(&vdev->pdev, 0);
591
592 if (msix_set_vector_notifiers(&vdev->pdev, vfio_msix_vector_use,
593 vfio_msix_vector_release, NULL)) {
594 error_report("vfio: msix_set_vector_notifiers failed");
595 }
596
597 trace_vfio_msix_enable(vdev->vbasedev.name);
598}
599
600static void vfio_msi_enable(VFIOPCIDevice *vdev)
601{
602 int ret, i;
603
604 vfio_disable_interrupts(vdev);
605
606 vdev->nr_vectors = msi_nr_vectors_allocated(&vdev->pdev);
607retry:
608 vdev->msi_vectors = g_new0(VFIOMSIVector, vdev->nr_vectors);
609
610 for (i = 0; i < vdev->nr_vectors; i++) {
611 VFIOMSIVector *vector = &vdev->msi_vectors[i];
612
613 vector->vdev = vdev;
614 vector->virq = -1;
615 vector->use = true;
616
617 if (event_notifier_init(&vector->interrupt, 0)) {
618 error_report("vfio: Error: event_notifier_init failed");
619 }
620
621 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
622 vfio_msi_interrupt, NULL, vector);
623
624 /*
625 * Attempt to enable route through KVM irqchip,
626 * default to userspace handling if unavailable.
627 */
628 vfio_add_kvm_msi_virq(vdev, vector, i, false);
629 }
630
631 /* Set interrupt type prior to possible interrupts */
632 vdev->interrupt = VFIO_INT_MSI;
633
634 ret = vfio_enable_vectors(vdev, false);
635 if (ret) {
636 if (ret < 0) {
637 error_report("vfio: Error: Failed to setup MSI fds: %m");
638 } else if (ret != vdev->nr_vectors) {
639 error_report("vfio: Error: Failed to enable %d "
640 "MSI vectors, retry with %d", vdev->nr_vectors, ret);
641 }
642
643 for (i = 0; i < vdev->nr_vectors; i++) {
644 VFIOMSIVector *vector = &vdev->msi_vectors[i];
645 if (vector->virq >= 0) {
646 vfio_remove_kvm_msi_virq(vector);
647 }
648 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
649 NULL, NULL, NULL);
650 event_notifier_cleanup(&vector->interrupt);
651 }
652
653 g_free(vdev->msi_vectors);
654
655 if (ret > 0 && ret != vdev->nr_vectors) {
656 vdev->nr_vectors = ret;
657 goto retry;
658 }
659 vdev->nr_vectors = 0;
660
661 /*
662 * Failing to setup MSI doesn't really fall within any specification.
663 * Let's try leaving interrupts disabled and hope the guest figures
664 * out to fall back to INTx for this device.
665 */
666 error_report("vfio: Error: Failed to enable MSI");
667 vdev->interrupt = VFIO_INT_NONE;
668
669 return;
670 }
671
672 trace_vfio_msi_enable(vdev->vbasedev.name, vdev->nr_vectors);
673}
674
675static void vfio_msi_disable_common(VFIOPCIDevice *vdev)
676{
677 Error *err = NULL;
678 int i;
679
680 for (i = 0; i < vdev->nr_vectors; i++) {
681 VFIOMSIVector *vector = &vdev->msi_vectors[i];
682 if (vdev->msi_vectors[i].use) {
683 if (vector->virq >= 0) {
684 vfio_remove_kvm_msi_virq(vector);
685 }
686 qemu_set_fd_handler(event_notifier_get_fd(&vector->interrupt),
687 NULL, NULL, NULL);
688 event_notifier_cleanup(&vector->interrupt);
689 }
690 }
691
692 g_free(vdev->msi_vectors);
693 vdev->msi_vectors = NULL;
694 vdev->nr_vectors = 0;
695 vdev->interrupt = VFIO_INT_NONE;
696
697 vfio_intx_enable(vdev, &err);
698 if (err) {
699 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
700 }
701}
702
703static void vfio_msix_disable(VFIOPCIDevice *vdev)
704{
705 int i;
706
707 msix_unset_vector_notifiers(&vdev->pdev);
708
709 /*
710 * MSI-X will only release vectors if MSI-X is still enabled on the
711 * device, check through the rest and release it ourselves if necessary.
712 */
713 for (i = 0; i < vdev->nr_vectors; i++) {
714 if (vdev->msi_vectors[i].use) {
715 vfio_msix_vector_release(&vdev->pdev, i);
716 msix_vector_unuse(&vdev->pdev, i);
717 }
718 }
719
720 if (vdev->nr_vectors) {
721 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
722 }
723
724 vfio_msi_disable_common(vdev);
725
726 memset(vdev->msix->pending, 0,
727 BITS_TO_LONGS(vdev->msix->entries) * sizeof(unsigned long));
728
729 trace_vfio_msix_disable(vdev->vbasedev.name);
730}
731
732static void vfio_msi_disable(VFIOPCIDevice *vdev)
733{
734 vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSI_IRQ_INDEX);
735 vfio_msi_disable_common(vdev);
736
737 trace_vfio_msi_disable(vdev->vbasedev.name);
738}
739
740static void vfio_update_msi(VFIOPCIDevice *vdev)
741{
742 int i;
743
744 for (i = 0; i < vdev->nr_vectors; i++) {
745 VFIOMSIVector *vector = &vdev->msi_vectors[i];
746 MSIMessage msg;
747
748 if (!vector->use || vector->virq < 0) {
749 continue;
750 }
751
752 msg = msi_get_message(&vdev->pdev, i);
753 vfio_update_kvm_msi_virq(vector, msg, &vdev->pdev);
754 }
755}
756
757static void vfio_pci_load_rom(VFIOPCIDevice *vdev)
758{
759 struct vfio_region_info *reg_info;
760 uint64_t size;
761 off_t off = 0;
762 ssize_t bytes;
763
764 if (vfio_get_region_info(&vdev->vbasedev,
765 VFIO_PCI_ROM_REGION_INDEX, &reg_info)) {
766 error_report("vfio: Error getting ROM info: %m");
767 return;
768 }
769
770 trace_vfio_pci_load_rom(vdev->vbasedev.name, (unsigned long)reg_info->size,
771 (unsigned long)reg_info->offset,
772 (unsigned long)reg_info->flags);
773
774 vdev->rom_size = size = reg_info->size;
775 vdev->rom_offset = reg_info->offset;
776
777 g_free(reg_info);
778
779 if (!vdev->rom_size) {
780 vdev->rom_read_failed = true;
781 error_report("vfio-pci: Cannot read device rom at "
782 "%s", vdev->vbasedev.name);
783 error_printf("Device option ROM contents are probably invalid "
784 "(check dmesg).\nSkip option ROM probe with rombar=0, "
785 "or load from file with romfile=\n");
786 return;
787 }
788
789 vdev->rom = g_malloc(size);
790 memset(vdev->rom, 0xff, size);
791
792 while (size) {
793 bytes = pread(vdev->vbasedev.fd, vdev->rom + off,
794 size, vdev->rom_offset + off);
795 if (bytes == 0) {
796 break;
797 } else if (bytes > 0) {
798 off += bytes;
799 size -= bytes;
800 } else {
801 if (errno == EINTR || errno == EAGAIN) {
802 continue;
803 }
804 error_report("vfio: Error reading device ROM: %m");
805 break;
806 }
807 }
808
809 /*
810 * Test the ROM signature against our device, if the vendor is correct
811 * but the device ID doesn't match, store the correct device ID and
812 * recompute the checksum. Intel IGD devices need this and are known
813 * to have bogus checksums so we can't simply adjust the checksum.
814 */
815 if (pci_get_word(vdev->rom) == 0xaa55 &&
816 pci_get_word(vdev->rom + 0x18) + 8 < vdev->rom_size &&
817 !memcmp(vdev->rom + pci_get_word(vdev->rom + 0x18), "PCIR", 4)) {
818 uint16_t vid, did;
819
820 vid = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 4);
821 did = pci_get_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6);
822
823 if (vid == vdev->vendor_id && did != vdev->device_id) {
824 int i;
825 uint8_t csum, *data = vdev->rom;
826
827 pci_set_word(vdev->rom + pci_get_word(vdev->rom + 0x18) + 6,
828 vdev->device_id);
829 data[6] = 0;
830
831 for (csum = 0, i = 0; i < vdev->rom_size; i++) {
832 csum += data[i];
833 }
834
835 data[6] = -csum;
836 }
837 }
838}
839
840static uint64_t vfio_rom_read(void *opaque, hwaddr addr, unsigned size)
841{
842 VFIOPCIDevice *vdev = opaque;
843 union {
844 uint8_t byte;
845 uint16_t word;
846 uint32_t dword;
847 uint64_t qword;
848 } val;
849 uint64_t data = 0;
850
851 /* Load the ROM lazily when the guest tries to read it */
852 if (unlikely(!vdev->rom && !vdev->rom_read_failed)) {
853 vfio_pci_load_rom(vdev);
854 }
855
856 memcpy(&val, vdev->rom + addr,
857 (addr < vdev->rom_size) ? MIN(size, vdev->rom_size - addr) : 0);
858
859 switch (size) {
860 case 1:
861 data = val.byte;
862 break;
863 case 2:
864 data = le16_to_cpu(val.word);
865 break;
866 case 4:
867 data = le32_to_cpu(val.dword);
868 break;
869 default:
870 hw_error("vfio: unsupported read size, %d bytes\n", size);
871 break;
872 }
873
874 trace_vfio_rom_read(vdev->vbasedev.name, addr, size, data);
875
876 return data;
877}
878
879static void vfio_rom_write(void *opaque, hwaddr addr,
880 uint64_t data, unsigned size)
881{
882}
883
884static const MemoryRegionOps vfio_rom_ops = {
885 .read = vfio_rom_read,
886 .write = vfio_rom_write,
887 .endianness = DEVICE_LITTLE_ENDIAN,
888};
889
890static void vfio_pci_size_rom(VFIOPCIDevice *vdev)
891{
892 uint32_t orig, size = cpu_to_le32((uint32_t)PCI_ROM_ADDRESS_MASK);
893 off_t offset = vdev->config_offset + PCI_ROM_ADDRESS;
894 DeviceState *dev = DEVICE(vdev);
895 char *name;
896 int fd = vdev->vbasedev.fd;
897
898 if (vdev->pdev.romfile || !vdev->pdev.rom_bar) {
899 /* Since pci handles romfile, just print a message and return */
900 if (vfio_blacklist_opt_rom(vdev) && vdev->pdev.romfile) {
901 warn_report("Device at %s is known to cause system instability"
902 " issues during option rom execution",
903 vdev->vbasedev.name);
904 error_printf("Proceeding anyway since user specified romfile\n");
905 }
906 return;
907 }
908
909 /*
910 * Use the same size ROM BAR as the physical device. The contents
911 * will get filled in later when the guest tries to read it.
912 */
913 if (pread(fd, &orig, 4, offset) != 4 ||
914 pwrite(fd, &size, 4, offset) != 4 ||
915 pread(fd, &size, 4, offset) != 4 ||
916 pwrite(fd, &orig, 4, offset) != 4) {
917 error_report("%s(%s) failed: %m", __func__, vdev->vbasedev.name);
918 return;
919 }
920
921 size = ~(le32_to_cpu(size) & PCI_ROM_ADDRESS_MASK) + 1;
922
923 if (!size) {
924 return;
925 }
926
927 if (vfio_blacklist_opt_rom(vdev)) {
928 if (dev->opts && qemu_opt_get(dev->opts, "rombar")) {
929 warn_report("Device at %s is known to cause system instability"
930 " issues during option rom execution",
931 vdev->vbasedev.name);
932 error_printf("Proceeding anyway since user specified"
933 " non zero value for rombar\n");
934 } else {
935 warn_report("Rom loading for device at %s has been disabled"
936 " due to system instability issues",
937 vdev->vbasedev.name);
938 error_printf("Specify rombar=1 or romfile to force\n");
939 return;
940 }
941 }
942
943 trace_vfio_pci_size_rom(vdev->vbasedev.name, size);
944
945 name = g_strdup_printf("vfio[%s].rom", vdev->vbasedev.name);
946
947 memory_region_init_io(&vdev->pdev.rom, OBJECT(vdev),
948 &vfio_rom_ops, vdev, name, size);
949 g_free(name);
950
951 pci_register_bar(&vdev->pdev, PCI_ROM_SLOT,
952 PCI_BASE_ADDRESS_SPACE_MEMORY, &vdev->pdev.rom);
953
954 vdev->rom_read_failed = false;
955}
956
957void vfio_vga_write(void *opaque, hwaddr addr,
958 uint64_t data, unsigned size)
959{
960 VFIOVGARegion *region = opaque;
961 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
962 union {
963 uint8_t byte;
964 uint16_t word;
965 uint32_t dword;
966 uint64_t qword;
967 } buf;
968 off_t offset = vga->fd_offset + region->offset + addr;
969
970 switch (size) {
971 case 1:
972 buf.byte = data;
973 break;
974 case 2:
975 buf.word = cpu_to_le16(data);
976 break;
977 case 4:
978 buf.dword = cpu_to_le32(data);
979 break;
980 default:
981 hw_error("vfio: unsupported write size, %d bytes", size);
982 break;
983 }
984
985 if (pwrite(vga->fd, &buf, size, offset) != size) {
986 error_report("%s(,0x%"HWADDR_PRIx", 0x%"PRIx64", %d) failed: %m",
987 __func__, region->offset + addr, data, size);
988 }
989
990 trace_vfio_vga_write(region->offset + addr, data, size);
991}
992
993uint64_t vfio_vga_read(void *opaque, hwaddr addr, unsigned size)
994{
995 VFIOVGARegion *region = opaque;
996 VFIOVGA *vga = container_of(region, VFIOVGA, region[region->nr]);
997 union {
998 uint8_t byte;
999 uint16_t word;
1000 uint32_t dword;
1001 uint64_t qword;
1002 } buf;
1003 uint64_t data = 0;
1004 off_t offset = vga->fd_offset + region->offset + addr;
1005
1006 if (pread(vga->fd, &buf, size, offset) != size) {
1007 error_report("%s(,0x%"HWADDR_PRIx", %d) failed: %m",
1008 __func__, region->offset + addr, size);
1009 return (uint64_t)-1;
1010 }
1011
1012 switch (size) {
1013 case 1:
1014 data = buf.byte;
1015 break;
1016 case 2:
1017 data = le16_to_cpu(buf.word);
1018 break;
1019 case 4:
1020 data = le32_to_cpu(buf.dword);
1021 break;
1022 default:
1023 hw_error("vfio: unsupported read size, %d bytes", size);
1024 break;
1025 }
1026
1027 trace_vfio_vga_read(region->offset + addr, size, data);
1028
1029 return data;
1030}
1031
1032static const MemoryRegionOps vfio_vga_ops = {
1033 .read = vfio_vga_read,
1034 .write = vfio_vga_write,
1035 .endianness = DEVICE_LITTLE_ENDIAN,
1036};
1037
1038/*
1039 * Expand memory region of sub-page(size < PAGE_SIZE) MMIO BAR to page
1040 * size if the BAR is in an exclusive page in host so that we could map
1041 * this BAR to guest. But this sub-page BAR may not occupy an exclusive
1042 * page in guest. So we should set the priority of the expanded memory
1043 * region to zero in case of overlap with BARs which share the same page
1044 * with the sub-page BAR in guest. Besides, we should also recover the
1045 * size of this sub-page BAR when its base address is changed in guest
1046 * and not page aligned any more.
1047 */
1048static void vfio_sub_page_bar_update_mapping(PCIDevice *pdev, int bar)
1049{
1050 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
1051 VFIORegion *region = &vdev->bars[bar].region;
1052 MemoryRegion *mmap_mr, *region_mr, *base_mr;
1053 PCIIORegion *r;
1054 pcibus_t bar_addr;
1055 uint64_t size = region->size;
1056
1057 /* Make sure that the whole region is allowed to be mmapped */
1058 if (region->nr_mmaps != 1 || !region->mmaps[0].mmap ||
1059 region->mmaps[0].size != region->size) {
1060 return;
1061 }
1062
1063 r = &pdev->io_regions[bar];
1064 bar_addr = r->addr;
1065 base_mr = vdev->bars[bar].mr;
1066 region_mr = region->mem;
1067 mmap_mr = &region->mmaps[0].mem;
1068
1069 /* If BAR is mapped and page aligned, update to fill PAGE_SIZE */
1070 if (bar_addr != PCI_BAR_UNMAPPED &&
1071 !(bar_addr & ~qemu_real_host_page_mask)) {
1072 size = qemu_real_host_page_size;
1073 }
1074
1075 memory_region_transaction_begin();
1076
1077 if (vdev->bars[bar].size < size) {
1078 memory_region_set_size(base_mr, size);
1079 }
1080 memory_region_set_size(region_mr, size);
1081 memory_region_set_size(mmap_mr, size);
1082 if (size != vdev->bars[bar].size && memory_region_is_mapped(base_mr)) {
1083 memory_region_del_subregion(r->address_space, base_mr);
1084 memory_region_add_subregion_overlap(r->address_space,
1085 bar_addr, base_mr, 0);
1086 }
1087
1088 memory_region_transaction_commit();
1089}
1090
1091/*
1092 * PCI config space
1093 */
1094uint32_t vfio_pci_read_config(PCIDevice *pdev, uint32_t addr, int len)
1095{
1096 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
1097 uint32_t emu_bits = 0, emu_val = 0, phys_val = 0, val;
1098
1099 memcpy(&emu_bits, vdev->emulated_config_bits + addr, len);
1100 emu_bits = le32_to_cpu(emu_bits);
1101
1102 if (emu_bits) {
1103 emu_val = pci_default_read_config(pdev, addr, len);
1104 }
1105
1106 if (~emu_bits & (0xffffffffU >> (32 - len * 8))) {
1107 ssize_t ret;
1108
1109 ret = pread(vdev->vbasedev.fd, &phys_val, len,
1110 vdev->config_offset + addr);
1111 if (ret != len) {
1112 error_report("%s(%s, 0x%x, 0x%x) failed: %m",
1113 __func__, vdev->vbasedev.name, addr, len);
1114 return -errno;
1115 }
1116 phys_val = le32_to_cpu(phys_val);
1117 }
1118
1119 val = (emu_val & emu_bits) | (phys_val & ~emu_bits);
1120
1121 trace_vfio_pci_read_config(vdev->vbasedev.name, addr, len, val);
1122
1123 return val;
1124}
1125
1126void vfio_pci_write_config(PCIDevice *pdev,
1127 uint32_t addr, uint32_t val, int len)
1128{
1129 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
1130 uint32_t val_le = cpu_to_le32(val);
1131
1132 trace_vfio_pci_write_config(vdev->vbasedev.name, addr, val, len);
1133
1134 /* Write everything to VFIO, let it filter out what we can't write */
1135 if (pwrite(vdev->vbasedev.fd, &val_le, len, vdev->config_offset + addr)
1136 != len) {
1137 error_report("%s(%s, 0x%x, 0x%x, 0x%x) failed: %m",
1138 __func__, vdev->vbasedev.name, addr, val, len);
1139 }
1140
1141 /* MSI/MSI-X Enabling/Disabling */
1142 if (pdev->cap_present & QEMU_PCI_CAP_MSI &&
1143 ranges_overlap(addr, len, pdev->msi_cap, vdev->msi_cap_size)) {
1144 int is_enabled, was_enabled = msi_enabled(pdev);
1145
1146 pci_default_write_config(pdev, addr, val, len);
1147
1148 is_enabled = msi_enabled(pdev);
1149
1150 if (!was_enabled) {
1151 if (is_enabled) {
1152 vfio_msi_enable(vdev);
1153 }
1154 } else {
1155 if (!is_enabled) {
1156 vfio_msi_disable(vdev);
1157 } else {
1158 vfio_update_msi(vdev);
1159 }
1160 }
1161 } else if (pdev->cap_present & QEMU_PCI_CAP_MSIX &&
1162 ranges_overlap(addr, len, pdev->msix_cap, MSIX_CAP_LENGTH)) {
1163 int is_enabled, was_enabled = msix_enabled(pdev);
1164
1165 pci_default_write_config(pdev, addr, val, len);
1166
1167 is_enabled = msix_enabled(pdev);
1168
1169 if (!was_enabled && is_enabled) {
1170 vfio_msix_enable(vdev);
1171 } else if (was_enabled && !is_enabled) {
1172 vfio_msix_disable(vdev);
1173 }
1174 } else if (ranges_overlap(addr, len, PCI_BASE_ADDRESS_0, 24) ||
1175 range_covers_byte(addr, len, PCI_COMMAND)) {
1176 pcibus_t old_addr[PCI_NUM_REGIONS - 1];
1177 int bar;
1178
1179 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1180 old_addr[bar] = pdev->io_regions[bar].addr;
1181 }
1182
1183 pci_default_write_config(pdev, addr, val, len);
1184
1185 for (bar = 0; bar < PCI_ROM_SLOT; bar++) {
1186 if (old_addr[bar] != pdev->io_regions[bar].addr &&
1187 vdev->bars[bar].region.size > 0 &&
1188 vdev->bars[bar].region.size < qemu_real_host_page_size) {
1189 vfio_sub_page_bar_update_mapping(pdev, bar);
1190 }
1191 }
1192 } else {
1193 /* Write everything to QEMU to keep emulated bits correct */
1194 pci_default_write_config(pdev, addr, val, len);
1195 }
1196}
1197
1198/*
1199 * Interrupt setup
1200 */
1201static void vfio_disable_interrupts(VFIOPCIDevice *vdev)
1202{
1203 /*
1204 * More complicated than it looks. Disabling MSI/X transitions the
1205 * device to INTx mode (if supported). Therefore we need to first
1206 * disable MSI/X and then cleanup by disabling INTx.
1207 */
1208 if (vdev->interrupt == VFIO_INT_MSIX) {
1209 vfio_msix_disable(vdev);
1210 } else if (vdev->interrupt == VFIO_INT_MSI) {
1211 vfio_msi_disable(vdev);
1212 }
1213
1214 if (vdev->interrupt == VFIO_INT_INTx) {
1215 vfio_intx_disable(vdev);
1216 }
1217}
1218
1219static int vfio_msi_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1220{
1221 uint16_t ctrl;
1222 bool msi_64bit, msi_maskbit;
1223 int ret, entries;
1224 Error *err = NULL;
1225
1226 if (pread(vdev->vbasedev.fd, &ctrl, sizeof(ctrl),
1227 vdev->config_offset + pos + PCI_CAP_FLAGS) != sizeof(ctrl)) {
1228 error_setg_errno(errp, errno, "failed reading MSI PCI_CAP_FLAGS");
1229 return -errno;
1230 }
1231 ctrl = le16_to_cpu(ctrl);
1232
1233 msi_64bit = !!(ctrl & PCI_MSI_FLAGS_64BIT);
1234 msi_maskbit = !!(ctrl & PCI_MSI_FLAGS_MASKBIT);
1235 entries = 1 << ((ctrl & PCI_MSI_FLAGS_QMASK) >> 1);
1236
1237 trace_vfio_msi_setup(vdev->vbasedev.name, pos);
1238
1239 ret = msi_init(&vdev->pdev, pos, entries, msi_64bit, msi_maskbit, &err);
1240 if (ret < 0) {
1241 if (ret == -ENOTSUP) {
1242 return 0;
1243 }
1244 error_propagate_prepend(errp, err, "msi_init failed: ");
1245 return ret;
1246 }
1247 vdev->msi_cap_size = 0xa + (msi_maskbit ? 0xa : 0) + (msi_64bit ? 0x4 : 0);
1248
1249 return 0;
1250}
1251
1252static void vfio_pci_fixup_msix_region(VFIOPCIDevice *vdev)
1253{
1254 off_t start, end;
1255 VFIORegion *region = &vdev->bars[vdev->msix->table_bar].region;
1256
1257 /*
1258 * If the host driver allows mapping of a MSIX data, we are going to
1259 * do map the entire BAR and emulate MSIX table on top of that.
1260 */
1261 if (vfio_has_region_cap(&vdev->vbasedev, region->nr,
1262 VFIO_REGION_INFO_CAP_MSIX_MAPPABLE)) {
1263 return;
1264 }
1265
1266 /*
1267 * We expect to find a single mmap covering the whole BAR, anything else
1268 * means it's either unsupported or already setup.
1269 */
1270 if (region->nr_mmaps != 1 || region->mmaps[0].offset ||
1271 region->size != region->mmaps[0].size) {
1272 return;
1273 }
1274
1275 /* MSI-X table start and end aligned to host page size */
1276 start = vdev->msix->table_offset & qemu_real_host_page_mask;
1277 end = REAL_HOST_PAGE_ALIGN((uint64_t)vdev->msix->table_offset +
1278 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE));
1279
1280 /*
1281 * Does the MSI-X table cover the beginning of the BAR? The whole BAR?
1282 * NB - Host page size is necessarily a power of two and so is the PCI
1283 * BAR (not counting EA yet), therefore if we have host page aligned
1284 * @start and @end, then any remainder of the BAR before or after those
1285 * must be at least host page sized and therefore mmap'able.
1286 */
1287 if (!start) {
1288 if (end >= region->size) {
1289 region->nr_mmaps = 0;
1290 g_free(region->mmaps);
1291 region->mmaps = NULL;
1292 trace_vfio_msix_fixup(vdev->vbasedev.name,
1293 vdev->msix->table_bar, 0, 0);
1294 } else {
1295 region->mmaps[0].offset = end;
1296 region->mmaps[0].size = region->size - end;
1297 trace_vfio_msix_fixup(vdev->vbasedev.name,
1298 vdev->msix->table_bar, region->mmaps[0].offset,
1299 region->mmaps[0].offset + region->mmaps[0].size);
1300 }
1301
1302 /* Maybe it's aligned at the end of the BAR */
1303 } else if (end >= region->size) {
1304 region->mmaps[0].size = start;
1305 trace_vfio_msix_fixup(vdev->vbasedev.name,
1306 vdev->msix->table_bar, region->mmaps[0].offset,
1307 region->mmaps[0].offset + region->mmaps[0].size);
1308
1309 /* Otherwise it must split the BAR */
1310 } else {
1311 region->nr_mmaps = 2;
1312 region->mmaps = g_renew(VFIOMmap, region->mmaps, 2);
1313
1314 memcpy(&region->mmaps[1], &region->mmaps[0], sizeof(VFIOMmap));
1315
1316 region->mmaps[0].size = start;
1317 trace_vfio_msix_fixup(vdev->vbasedev.name,
1318 vdev->msix->table_bar, region->mmaps[0].offset,
1319 region->mmaps[0].offset + region->mmaps[0].size);
1320
1321 region->mmaps[1].offset = end;
1322 region->mmaps[1].size = region->size - end;
1323 trace_vfio_msix_fixup(vdev->vbasedev.name,
1324 vdev->msix->table_bar, region->mmaps[1].offset,
1325 region->mmaps[1].offset + region->mmaps[1].size);
1326 }
1327}
1328
1329static void vfio_pci_relocate_msix(VFIOPCIDevice *vdev, Error **errp)
1330{
1331 int target_bar = -1;
1332 size_t msix_sz;
1333
1334 if (!vdev->msix || vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1335 return;
1336 }
1337
1338 /* The actual minimum size of MSI-X structures */
1339 msix_sz = (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE) +
1340 (QEMU_ALIGN_UP(vdev->msix->entries, 64) / 8);
1341 /* Round up to host pages, we don't want to share a page */
1342 msix_sz = REAL_HOST_PAGE_ALIGN(msix_sz);
1343 /* PCI BARs must be a power of 2 */
1344 msix_sz = pow2ceil(msix_sz);
1345
1346 if (vdev->msix_relo == OFF_AUTOPCIBAR_AUTO) {
1347 /*
1348 * TODO: Lookup table for known devices.
1349 *
1350 * Logically we might use an algorithm here to select the BAR adding
1351 * the least additional MMIO space, but we cannot programatically
1352 * predict the driver dependency on BAR ordering or sizing, therefore
1353 * 'auto' becomes a lookup for combinations reported to work.
1354 */
1355 if (target_bar < 0) {
1356 error_setg(errp, "No automatic MSI-X relocation available for "
1357 "device %04x:%04x", vdev->vendor_id, vdev->device_id);
1358 return;
1359 }
1360 } else {
1361 target_bar = (int)(vdev->msix_relo - OFF_AUTOPCIBAR_BAR0);
1362 }
1363
1364 /* I/O port BARs cannot host MSI-X structures */
1365 if (vdev->bars[target_bar].ioport) {
1366 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1367 "I/O port BAR", target_bar);
1368 return;
1369 }
1370
1371 /* Cannot use a BAR in the "shadow" of a 64-bit BAR */
1372 if (!vdev->bars[target_bar].size &&
1373 target_bar > 0 && vdev->bars[target_bar - 1].mem64) {
1374 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1375 "consumed by 64-bit BAR %d", target_bar, target_bar - 1);
1376 return;
1377 }
1378
1379 /* 2GB max size for 32-bit BARs, cannot double if already > 1G */
1380 if (vdev->bars[target_bar].size > 1 * GiB &&
1381 !vdev->bars[target_bar].mem64) {
1382 error_setg(errp, "Invalid MSI-X relocation BAR %d, "
1383 "no space to extend 32-bit BAR", target_bar);
1384 return;
1385 }
1386
1387 /*
1388 * If adding a new BAR, test if we can make it 64bit. We make it
1389 * prefetchable since QEMU MSI-X emulation has no read side effects
1390 * and doing so makes mapping more flexible.
1391 */
1392 if (!vdev->bars[target_bar].size) {
1393 if (target_bar < (PCI_ROM_SLOT - 1) &&
1394 !vdev->bars[target_bar + 1].size) {
1395 vdev->bars[target_bar].mem64 = true;
1396 vdev->bars[target_bar].type = PCI_BASE_ADDRESS_MEM_TYPE_64;
1397 }
1398 vdev->bars[target_bar].type |= PCI_BASE_ADDRESS_MEM_PREFETCH;
1399 vdev->bars[target_bar].size = msix_sz;
1400 vdev->msix->table_offset = 0;
1401 } else {
1402 vdev->bars[target_bar].size = MAX(vdev->bars[target_bar].size * 2,
1403 msix_sz * 2);
1404 /*
1405 * Due to above size calc, MSI-X always starts halfway into the BAR,
1406 * which will always be a separate host page.
1407 */
1408 vdev->msix->table_offset = vdev->bars[target_bar].size / 2;
1409 }
1410
1411 vdev->msix->table_bar = target_bar;
1412 vdev->msix->pba_bar = target_bar;
1413 /* Requires 8-byte alignment, but PCI_MSIX_ENTRY_SIZE guarantees that */
1414 vdev->msix->pba_offset = vdev->msix->table_offset +
1415 (vdev->msix->entries * PCI_MSIX_ENTRY_SIZE);
1416
1417 trace_vfio_msix_relo(vdev->vbasedev.name,
1418 vdev->msix->table_bar, vdev->msix->table_offset);
1419}
1420
1421/*
1422 * We don't have any control over how pci_add_capability() inserts
1423 * capabilities into the chain. In order to setup MSI-X we need a
1424 * MemoryRegion for the BAR. In order to setup the BAR and not
1425 * attempt to mmap the MSI-X table area, which VFIO won't allow, we
1426 * need to first look for where the MSI-X table lives. So we
1427 * unfortunately split MSI-X setup across two functions.
1428 */
1429static void vfio_msix_early_setup(VFIOPCIDevice *vdev, Error **errp)
1430{
1431 uint8_t pos;
1432 uint16_t ctrl;
1433 uint32_t table, pba;
1434 int fd = vdev->vbasedev.fd;
1435 VFIOMSIXInfo *msix;
1436
1437 pos = pci_find_capability(&vdev->pdev, PCI_CAP_ID_MSIX);
1438 if (!pos) {
1439 return;
1440 }
1441
1442 if (pread(fd, &ctrl, sizeof(ctrl),
1443 vdev->config_offset + pos + PCI_MSIX_FLAGS) != sizeof(ctrl)) {
1444 error_setg_errno(errp, errno, "failed to read PCI MSIX FLAGS");
1445 return;
1446 }
1447
1448 if (pread(fd, &table, sizeof(table),
1449 vdev->config_offset + pos + PCI_MSIX_TABLE) != sizeof(table)) {
1450 error_setg_errno(errp, errno, "failed to read PCI MSIX TABLE");
1451 return;
1452 }
1453
1454 if (pread(fd, &pba, sizeof(pba),
1455 vdev->config_offset + pos + PCI_MSIX_PBA) != sizeof(pba)) {
1456 error_setg_errno(errp, errno, "failed to read PCI MSIX PBA");
1457 return;
1458 }
1459
1460 ctrl = le16_to_cpu(ctrl);
1461 table = le32_to_cpu(table);
1462 pba = le32_to_cpu(pba);
1463
1464 msix = g_malloc0(sizeof(*msix));
1465 msix->table_bar = table & PCI_MSIX_FLAGS_BIRMASK;
1466 msix->table_offset = table & ~PCI_MSIX_FLAGS_BIRMASK;
1467 msix->pba_bar = pba & PCI_MSIX_FLAGS_BIRMASK;
1468 msix->pba_offset = pba & ~PCI_MSIX_FLAGS_BIRMASK;
1469 msix->entries = (ctrl & PCI_MSIX_FLAGS_QSIZE) + 1;
1470
1471 /*
1472 * Test the size of the pba_offset variable and catch if it extends outside
1473 * of the specified BAR. If it is the case, we need to apply a hardware
1474 * specific quirk if the device is known or we have a broken configuration.
1475 */
1476 if (msix->pba_offset >= vdev->bars[msix->pba_bar].region.size) {
1477 /*
1478 * Chelsio T5 Virtual Function devices are encoded as 0x58xx for T5
1479 * adapters. The T5 hardware returns an incorrect value of 0x8000 for
1480 * the VF PBA offset while the BAR itself is only 8k. The correct value
1481 * is 0x1000, so we hard code that here.
1482 */
1483 if (vdev->vendor_id == PCI_VENDOR_ID_CHELSIO &&
1484 (vdev->device_id & 0xff00) == 0x5800) {
1485 msix->pba_offset = 0x1000;
1486 } else if (vdev->msix_relo == OFF_AUTOPCIBAR_OFF) {
1487 error_setg(errp, "hardware reports invalid configuration, "
1488 "MSIX PBA outside of specified BAR");
1489 g_free(msix);
1490 return;
1491 }
1492 }
1493
1494 trace_vfio_msix_early_setup(vdev->vbasedev.name, pos, msix->table_bar,
1495 msix->table_offset, msix->entries);
1496 vdev->msix = msix;
1497
1498 vfio_pci_fixup_msix_region(vdev);
1499
1500 vfio_pci_relocate_msix(vdev, errp);
1501}
1502
1503static int vfio_msix_setup(VFIOPCIDevice *vdev, int pos, Error **errp)
1504{
1505 int ret;
1506 Error *err = NULL;
1507
1508 vdev->msix->pending = g_malloc0(BITS_TO_LONGS(vdev->msix->entries) *
1509 sizeof(unsigned long));
1510 ret = msix_init(&vdev->pdev, vdev->msix->entries,
1511 vdev->bars[vdev->msix->table_bar].mr,
1512 vdev->msix->table_bar, vdev->msix->table_offset,
1513 vdev->bars[vdev->msix->pba_bar].mr,
1514 vdev->msix->pba_bar, vdev->msix->pba_offset, pos,
1515 &err);
1516 if (ret < 0) {
1517 if (ret == -ENOTSUP) {
1518 warn_report_err(err);
1519 return 0;
1520 }
1521
1522 error_propagate(errp, err);
1523 return ret;
1524 }
1525
1526 /*
1527 * The PCI spec suggests that devices provide additional alignment for
1528 * MSI-X structures and avoid overlapping non-MSI-X related registers.
1529 * For an assigned device, this hopefully means that emulation of MSI-X
1530 * structures does not affect the performance of the device. If devices
1531 * fail to provide that alignment, a significant performance penalty may
1532 * result, for instance Mellanox MT27500 VFs:
1533 * http://www.spinics.net/lists/kvm/msg125881.html
1534 *
1535 * The PBA is simply not that important for such a serious regression and
1536 * most drivers do not appear to look at it. The solution for this is to
1537 * disable the PBA MemoryRegion unless it's being used. We disable it
1538 * here and only enable it if a masked vector fires through QEMU. As the
1539 * vector-use notifier is called, which occurs on unmask, we test whether
1540 * PBA emulation is needed and again disable if not.
1541 */
1542 memory_region_set_enabled(&vdev->pdev.msix_pba_mmio, false);
1543
1544 /*
1545 * The emulated machine may provide a paravirt interface for MSIX setup
1546 * so it is not strictly necessary to emulate MSIX here. This becomes
1547 * helpful when frequently accessed MMIO registers are located in
1548 * subpages adjacent to the MSIX table but the MSIX data containing page
1549 * cannot be mapped because of a host page size bigger than the MSIX table
1550 * alignment.
1551 */
1552 if (object_property_get_bool(OBJECT(qdev_get_machine()),
1553 "vfio-no-msix-emulation", NULL)) {
1554 memory_region_set_enabled(&vdev->pdev.msix_table_mmio, false);
1555 }
1556
1557 return 0;
1558}
1559
1560static void vfio_teardown_msi(VFIOPCIDevice *vdev)
1561{
1562 msi_uninit(&vdev->pdev);
1563
1564 if (vdev->msix) {
1565 msix_uninit(&vdev->pdev,
1566 vdev->bars[vdev->msix->table_bar].mr,
1567 vdev->bars[vdev->msix->pba_bar].mr);
1568 g_free(vdev->msix->pending);
1569 }
1570}
1571
1572/*
1573 * Resource setup
1574 */
1575static void vfio_mmap_set_enabled(VFIOPCIDevice *vdev, bool enabled)
1576{
1577 int i;
1578
1579 for (i = 0; i < PCI_ROM_SLOT; i++) {
1580 vfio_region_mmaps_set_enabled(&vdev->bars[i].region, enabled);
1581 }
1582}
1583
1584static void vfio_bar_prepare(VFIOPCIDevice *vdev, int nr)
1585{
1586 VFIOBAR *bar = &vdev->bars[nr];
1587
1588 uint32_t pci_bar;
1589 int ret;
1590
1591 /* Skip both unimplemented BARs and the upper half of 64bit BARS. */
1592 if (!bar->region.size) {
1593 return;
1594 }
1595
1596 /* Determine what type of BAR this is for registration */
1597 ret = pread(vdev->vbasedev.fd, &pci_bar, sizeof(pci_bar),
1598 vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr));
1599 if (ret != sizeof(pci_bar)) {
1600 error_report("vfio: Failed to read BAR %d (%m)", nr);
1601 return;
1602 }
1603
1604 pci_bar = le32_to_cpu(pci_bar);
1605 bar->ioport = (pci_bar & PCI_BASE_ADDRESS_SPACE_IO);
1606 bar->mem64 = bar->ioport ? 0 : (pci_bar & PCI_BASE_ADDRESS_MEM_TYPE_64);
1607 bar->type = pci_bar & (bar->ioport ? ~PCI_BASE_ADDRESS_IO_MASK :
1608 ~PCI_BASE_ADDRESS_MEM_MASK);
1609 bar->size = bar->region.size;
1610}
1611
1612static void vfio_bars_prepare(VFIOPCIDevice *vdev)
1613{
1614 int i;
1615
1616 for (i = 0; i < PCI_ROM_SLOT; i++) {
1617 vfio_bar_prepare(vdev, i);
1618 }
1619}
1620
1621static void vfio_bar_register(VFIOPCIDevice *vdev, int nr)
1622{
1623 VFIOBAR *bar = &vdev->bars[nr];
1624 char *name;
1625
1626 if (!bar->size) {
1627 return;
1628 }
1629
1630 bar->mr = g_new0(MemoryRegion, 1);
1631 name = g_strdup_printf("%s base BAR %d", vdev->vbasedev.name, nr);
1632 memory_region_init_io(bar->mr, OBJECT(vdev), NULL, NULL, name, bar->size);
1633 g_free(name);
1634
1635 if (bar->region.size) {
1636 memory_region_add_subregion(bar->mr, 0, bar->region.mem);
1637
1638 if (vfio_region_mmap(&bar->region)) {
1639 error_report("Failed to mmap %s BAR %d. Performance may be slow",
1640 vdev->vbasedev.name, nr);
1641 }
1642 }
1643
1644 pci_register_bar(&vdev->pdev, nr, bar->type, bar->mr);
1645}
1646
1647static void vfio_bars_register(VFIOPCIDevice *vdev)
1648{
1649 int i;
1650
1651 for (i = 0; i < PCI_ROM_SLOT; i++) {
1652 vfio_bar_register(vdev, i);
1653 }
1654}
1655
1656static void vfio_bars_exit(VFIOPCIDevice *vdev)
1657{
1658 int i;
1659
1660 for (i = 0; i < PCI_ROM_SLOT; i++) {
1661 VFIOBAR *bar = &vdev->bars[i];
1662
1663 vfio_bar_quirk_exit(vdev, i);
1664 vfio_region_exit(&bar->region);
1665 if (bar->region.size) {
1666 memory_region_del_subregion(bar->mr, bar->region.mem);
1667 }
1668 }
1669
1670 if (vdev->vga) {
1671 pci_unregister_vga(&vdev->pdev);
1672 vfio_vga_quirk_exit(vdev);
1673 }
1674}
1675
1676static void vfio_bars_finalize(VFIOPCIDevice *vdev)
1677{
1678 int i;
1679
1680 for (i = 0; i < PCI_ROM_SLOT; i++) {
1681 VFIOBAR *bar = &vdev->bars[i];
1682
1683 vfio_bar_quirk_finalize(vdev, i);
1684 vfio_region_finalize(&bar->region);
1685 if (bar->size) {
1686 object_unparent(OBJECT(bar->mr));
1687 g_free(bar->mr);
1688 }
1689 }
1690
1691 if (vdev->vga) {
1692 vfio_vga_quirk_finalize(vdev);
1693 for (i = 0; i < ARRAY_SIZE(vdev->vga->region); i++) {
1694 object_unparent(OBJECT(&vdev->vga->region[i].mem));
1695 }
1696 g_free(vdev->vga);
1697 }
1698}
1699
1700/*
1701 * General setup
1702 */
1703static uint8_t vfio_std_cap_max_size(PCIDevice *pdev, uint8_t pos)
1704{
1705 uint8_t tmp;
1706 uint16_t next = PCI_CONFIG_SPACE_SIZE;
1707
1708 for (tmp = pdev->config[PCI_CAPABILITY_LIST]; tmp;
1709 tmp = pdev->config[tmp + PCI_CAP_LIST_NEXT]) {
1710 if (tmp > pos && tmp < next) {
1711 next = tmp;
1712 }
1713 }
1714
1715 return next - pos;
1716}
1717
1718
1719static uint16_t vfio_ext_cap_max_size(const uint8_t *config, uint16_t pos)
1720{
1721 uint16_t tmp, next = PCIE_CONFIG_SPACE_SIZE;
1722
1723 for (tmp = PCI_CONFIG_SPACE_SIZE; tmp;
1724 tmp = PCI_EXT_CAP_NEXT(pci_get_long(config + tmp))) {
1725 if (tmp > pos && tmp < next) {
1726 next = tmp;
1727 }
1728 }
1729
1730 return next - pos;
1731}
1732
1733static void vfio_set_word_bits(uint8_t *buf, uint16_t val, uint16_t mask)
1734{
1735 pci_set_word(buf, (pci_get_word(buf) & ~mask) | val);
1736}
1737
1738static void vfio_add_emulated_word(VFIOPCIDevice *vdev, int pos,
1739 uint16_t val, uint16_t mask)
1740{
1741 vfio_set_word_bits(vdev->pdev.config + pos, val, mask);
1742 vfio_set_word_bits(vdev->pdev.wmask + pos, ~mask, mask);
1743 vfio_set_word_bits(vdev->emulated_config_bits + pos, mask, mask);
1744}
1745
1746static void vfio_set_long_bits(uint8_t *buf, uint32_t val, uint32_t mask)
1747{
1748 pci_set_long(buf, (pci_get_long(buf) & ~mask) | val);
1749}
1750
1751static void vfio_add_emulated_long(VFIOPCIDevice *vdev, int pos,
1752 uint32_t val, uint32_t mask)
1753{
1754 vfio_set_long_bits(vdev->pdev.config + pos, val, mask);
1755 vfio_set_long_bits(vdev->pdev.wmask + pos, ~mask, mask);
1756 vfio_set_long_bits(vdev->emulated_config_bits + pos, mask, mask);
1757}
1758
1759static int vfio_setup_pcie_cap(VFIOPCIDevice *vdev, int pos, uint8_t size,
1760 Error **errp)
1761{
1762 uint16_t flags;
1763 uint8_t type;
1764
1765 flags = pci_get_word(vdev->pdev.config + pos + PCI_CAP_FLAGS);
1766 type = (flags & PCI_EXP_FLAGS_TYPE) >> 4;
1767
1768 if (type != PCI_EXP_TYPE_ENDPOINT &&
1769 type != PCI_EXP_TYPE_LEG_END &&
1770 type != PCI_EXP_TYPE_RC_END) {
1771
1772 error_setg(errp, "assignment of PCIe type 0x%x "
1773 "devices is not currently supported", type);
1774 return -EINVAL;
1775 }
1776
1777 if (!pci_bus_is_express(pci_get_bus(&vdev->pdev))) {
1778 PCIBus *bus = pci_get_bus(&vdev->pdev);
1779 PCIDevice *bridge;
1780
1781 /*
1782 * Traditionally PCI device assignment exposes the PCIe capability
1783 * as-is on non-express buses. The reason being that some drivers
1784 * simply assume that it's there, for example tg3. However when
1785 * we're running on a native PCIe machine type, like Q35, we need
1786 * to hide the PCIe capability. The reason for this is twofold;
1787 * first Windows guests get a Code 10 error when the PCIe capability
1788 * is exposed in this configuration. Therefore express devices won't
1789 * work at all unless they're attached to express buses in the VM.
1790 * Second, a native PCIe machine introduces the possibility of fine
1791 * granularity IOMMUs supporting both translation and isolation.
1792 * Guest code to discover the IOMMU visibility of a device, such as
1793 * IOMMU grouping code on Linux, is very aware of device types and
1794 * valid transitions between bus types. An express device on a non-
1795 * express bus is not a valid combination on bare metal systems.
1796 *
1797 * Drivers that require a PCIe capability to make the device
1798 * functional are simply going to need to have their devices placed
1799 * on a PCIe bus in the VM.
1800 */
1801 while (!pci_bus_is_root(bus)) {
1802 bridge = pci_bridge_get_device(bus);
1803 bus = pci_get_bus(bridge);
1804 }
1805
1806 if (pci_bus_is_express(bus)) {
1807 return 0;
1808 }
1809
1810 } else if (pci_bus_is_root(pci_get_bus(&vdev->pdev))) {
1811 /*
1812 * On a Root Complex bus Endpoints become Root Complex Integrated
1813 * Endpoints, which changes the type and clears the LNK & LNK2 fields.
1814 */
1815 if (type == PCI_EXP_TYPE_ENDPOINT) {
1816 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1817 PCI_EXP_TYPE_RC_END << 4,
1818 PCI_EXP_FLAGS_TYPE);
1819
1820 /* Link Capabilities, Status, and Control goes away */
1821 if (size > PCI_EXP_LNKCTL) {
1822 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP, 0, ~0);
1823 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1824 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA, 0, ~0);
1825
1826#ifndef PCI_EXP_LNKCAP2
1827#define PCI_EXP_LNKCAP2 44
1828#endif
1829#ifndef PCI_EXP_LNKSTA2
1830#define PCI_EXP_LNKSTA2 50
1831#endif
1832 /* Link 2 Capabilities, Status, and Control goes away */
1833 if (size > PCI_EXP_LNKCAP2) {
1834 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP2, 0, ~0);
1835 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL2, 0, ~0);
1836 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKSTA2, 0, ~0);
1837 }
1838 }
1839
1840 } else if (type == PCI_EXP_TYPE_LEG_END) {
1841 /*
1842 * Legacy endpoints don't belong on the root complex. Windows
1843 * seems to be happier with devices if we skip the capability.
1844 */
1845 return 0;
1846 }
1847
1848 } else {
1849 /*
1850 * Convert Root Complex Integrated Endpoints to regular endpoints.
1851 * These devices don't support LNK/LNK2 capabilities, so make them up.
1852 */
1853 if (type == PCI_EXP_TYPE_RC_END) {
1854 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1855 PCI_EXP_TYPE_ENDPOINT << 4,
1856 PCI_EXP_FLAGS_TYPE);
1857 vfio_add_emulated_long(vdev, pos + PCI_EXP_LNKCAP,
1858 QEMU_PCI_EXP_LNKCAP_MLW(QEMU_PCI_EXP_LNK_X1) |
1859 QEMU_PCI_EXP_LNKCAP_MLS(QEMU_PCI_EXP_LNK_2_5GT), ~0);
1860 vfio_add_emulated_word(vdev, pos + PCI_EXP_LNKCTL, 0, ~0);
1861 }
1862 }
1863
1864 /*
1865 * Intel 82599 SR-IOV VFs report an invalid PCIe capability version 0
1866 * (Niantic errate #35) causing Windows to error with a Code 10 for the
1867 * device on Q35. Fixup any such devices to report version 1. If we
1868 * were to remove the capability entirely the guest would lose extended
1869 * config space.
1870 */
1871 if ((flags & PCI_EXP_FLAGS_VERS) == 0) {
1872 vfio_add_emulated_word(vdev, pos + PCI_CAP_FLAGS,
1873 1, PCI_EXP_FLAGS_VERS);
1874 }
1875
1876 pos = pci_add_capability(&vdev->pdev, PCI_CAP_ID_EXP, pos, size,
1877 errp);
1878 if (pos < 0) {
1879 return pos;
1880 }
1881
1882 vdev->pdev.exp.exp_cap = pos;
1883
1884 return pos;
1885}
1886
1887static void vfio_check_pcie_flr(VFIOPCIDevice *vdev, uint8_t pos)
1888{
1889 uint32_t cap = pci_get_long(vdev->pdev.config + pos + PCI_EXP_DEVCAP);
1890
1891 if (cap & PCI_EXP_DEVCAP_FLR) {
1892 trace_vfio_check_pcie_flr(vdev->vbasedev.name);
1893 vdev->has_flr = true;
1894 }
1895}
1896
1897static void vfio_check_pm_reset(VFIOPCIDevice *vdev, uint8_t pos)
1898{
1899 uint16_t csr = pci_get_word(vdev->pdev.config + pos + PCI_PM_CTRL);
1900
1901 if (!(csr & PCI_PM_CTRL_NO_SOFT_RESET)) {
1902 trace_vfio_check_pm_reset(vdev->vbasedev.name);
1903 vdev->has_pm_reset = true;
1904 }
1905}
1906
1907static void vfio_check_af_flr(VFIOPCIDevice *vdev, uint8_t pos)
1908{
1909 uint8_t cap = pci_get_byte(vdev->pdev.config + pos + PCI_AF_CAP);
1910
1911 if ((cap & PCI_AF_CAP_TP) && (cap & PCI_AF_CAP_FLR)) {
1912 trace_vfio_check_af_flr(vdev->vbasedev.name);
1913 vdev->has_flr = true;
1914 }
1915}
1916
1917static int vfio_add_std_cap(VFIOPCIDevice *vdev, uint8_t pos, Error **errp)
1918{
1919 PCIDevice *pdev = &vdev->pdev;
1920 uint8_t cap_id, next, size;
1921 int ret;
1922
1923 cap_id = pdev->config[pos];
1924 next = pdev->config[pos + PCI_CAP_LIST_NEXT];
1925
1926 /*
1927 * If it becomes important to configure capabilities to their actual
1928 * size, use this as the default when it's something we don't recognize.
1929 * Since QEMU doesn't actually handle many of the config accesses,
1930 * exact size doesn't seem worthwhile.
1931 */
1932 size = vfio_std_cap_max_size(pdev, pos);
1933
1934 /*
1935 * pci_add_capability always inserts the new capability at the head
1936 * of the chain. Therefore to end up with a chain that matches the
1937 * physical device, we insert from the end by making this recursive.
1938 * This is also why we pre-calculate size above as cached config space
1939 * will be changed as we unwind the stack.
1940 */
1941 if (next) {
1942 ret = vfio_add_std_cap(vdev, next, errp);
1943 if (ret) {
1944 return ret;
1945 }
1946 } else {
1947 /* Begin the rebuild, use QEMU emulated list bits */
1948 pdev->config[PCI_CAPABILITY_LIST] = 0;
1949 vdev->emulated_config_bits[PCI_CAPABILITY_LIST] = 0xff;
1950 vdev->emulated_config_bits[PCI_STATUS] |= PCI_STATUS_CAP_LIST;
1951
1952 ret = vfio_add_virt_caps(vdev, errp);
1953 if (ret) {
1954 return ret;
1955 }
1956 }
1957
1958 /* Scale down size, esp in case virt caps were added above */
1959 size = MIN(size, vfio_std_cap_max_size(pdev, pos));
1960
1961 /* Use emulated next pointer to allow dropping caps */
1962 pci_set_byte(vdev->emulated_config_bits + pos + PCI_CAP_LIST_NEXT, 0xff);
1963
1964 switch (cap_id) {
1965 case PCI_CAP_ID_MSI:
1966 ret = vfio_msi_setup(vdev, pos, errp);
1967 break;
1968 case PCI_CAP_ID_EXP:
1969 vfio_check_pcie_flr(vdev, pos);
1970 ret = vfio_setup_pcie_cap(vdev, pos, size, errp);
1971 break;
1972 case PCI_CAP_ID_MSIX:
1973 ret = vfio_msix_setup(vdev, pos, errp);
1974 break;
1975 case PCI_CAP_ID_PM:
1976 vfio_check_pm_reset(vdev, pos);
1977 vdev->pm_cap = pos;
1978 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1979 break;
1980 case PCI_CAP_ID_AF:
1981 vfio_check_af_flr(vdev, pos);
1982 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1983 break;
1984 default:
1985 ret = pci_add_capability(pdev, cap_id, pos, size, errp);
1986 break;
1987 }
1988
1989 if (ret < 0) {
1990 error_prepend(errp,
1991 "failed to add PCI capability 0x%x[0x%x]@0x%x: ",
1992 cap_id, size, pos);
1993 return ret;
1994 }
1995
1996 return 0;
1997}
1998
1999static void vfio_add_ext_cap(VFIOPCIDevice *vdev)
2000{
2001 PCIDevice *pdev = &vdev->pdev;
2002 uint32_t header;
2003 uint16_t cap_id, next, size;
2004 uint8_t cap_ver;
2005 uint8_t *config;
2006
2007 /* Only add extended caps if we have them and the guest can see them */
2008 if (!pci_is_express(pdev) || !pci_bus_is_express(pci_get_bus(pdev)) ||
2009 !pci_get_long(pdev->config + PCI_CONFIG_SPACE_SIZE)) {
2010 return;
2011 }
2012
2013 /*
2014 * pcie_add_capability always inserts the new capability at the tail
2015 * of the chain. Therefore to end up with a chain that matches the
2016 * physical device, we cache the config space to avoid overwriting
2017 * the original config space when we parse the extended capabilities.
2018 */
2019 config = g_memdup(pdev->config, vdev->config_size);
2020
2021 /*
2022 * Extended capabilities are chained with each pointing to the next, so we
2023 * can drop anything other than the head of the chain simply by modifying
2024 * the previous next pointer. Seed the head of the chain here such that
2025 * we can simply skip any capabilities we want to drop below, regardless
2026 * of their position in the chain. If this stub capability still exists
2027 * after we add the capabilities we want to expose, update the capability
2028 * ID to zero. Note that we cannot seed with the capability header being
2029 * zero as this conflicts with definition of an absent capability chain
2030 * and prevents capabilities beyond the head of the list from being added.
2031 * By replacing the dummy capability ID with zero after walking the device
2032 * chain, we also transparently mark extended capabilities as absent if
2033 * no capabilities were added. Note that the PCIe spec defines an absence
2034 * of extended capabilities to be determined by a value of zero for the
2035 * capability ID, version, AND next pointer. A non-zero next pointer
2036 * should be sufficient to indicate additional capabilities are present,
2037 * which will occur if we call pcie_add_capability() below. The entire
2038 * first dword is emulated to support this.
2039 *
2040 * NB. The kernel side does similar masking, so be prepared that our
2041 * view of the device may also contain a capability ID zero in the head
2042 * of the chain. Skip it for the same reason that we cannot seed the
2043 * chain with a zero capability.
2044 */
2045 pci_set_long(pdev->config + PCI_CONFIG_SPACE_SIZE,
2046 PCI_EXT_CAP(0xFFFF, 0, 0));
2047 pci_set_long(pdev->wmask + PCI_CONFIG_SPACE_SIZE, 0);
2048 pci_set_long(vdev->emulated_config_bits + PCI_CONFIG_SPACE_SIZE, ~0);
2049
2050 for (next = PCI_CONFIG_SPACE_SIZE; next;
2051 next = PCI_EXT_CAP_NEXT(pci_get_long(config + next))) {
2052 header = pci_get_long(config + next);
2053 cap_id = PCI_EXT_CAP_ID(header);
2054 cap_ver = PCI_EXT_CAP_VER(header);
2055
2056 /*
2057 * If it becomes important to configure extended capabilities to their
2058 * actual size, use this as the default when it's something we don't
2059 * recognize. Since QEMU doesn't actually handle many of the config
2060 * accesses, exact size doesn't seem worthwhile.
2061 */
2062 size = vfio_ext_cap_max_size(config, next);
2063
2064 /* Use emulated next pointer to allow dropping extended caps */
2065 pci_long_test_and_set_mask(vdev->emulated_config_bits + next,
2066 PCI_EXT_CAP_NEXT_MASK);
2067
2068 switch (cap_id) {
2069 case 0: /* kernel masked capability */
2070 case PCI_EXT_CAP_ID_SRIOV: /* Read-only VF BARs confuse OVMF */
2071 case PCI_EXT_CAP_ID_ARI: /* XXX Needs next function virtualization */
2072 case PCI_EXT_CAP_ID_REBAR: /* Can't expose read-only */
2073 trace_vfio_add_ext_cap_dropped(vdev->vbasedev.name, cap_id, next);
2074 break;
2075 default:
2076 pcie_add_capability(pdev, cap_id, cap_ver, next, size);
2077 }
2078
2079 }
2080
2081 /* Cleanup chain head ID if necessary */
2082 if (pci_get_word(pdev->config + PCI_CONFIG_SPACE_SIZE) == 0xFFFF) {
2083 pci_set_word(pdev->config + PCI_CONFIG_SPACE_SIZE, 0);
2084 }
2085
2086 g_free(config);
2087 return;
2088}
2089
2090static int vfio_add_capabilities(VFIOPCIDevice *vdev, Error **errp)
2091{
2092 PCIDevice *pdev = &vdev->pdev;
2093 int ret;
2094
2095 if (!(pdev->config[PCI_STATUS] & PCI_STATUS_CAP_LIST) ||
2096 !pdev->config[PCI_CAPABILITY_LIST]) {
2097 return 0; /* Nothing to add */
2098 }
2099
2100 ret = vfio_add_std_cap(vdev, pdev->config[PCI_CAPABILITY_LIST], errp);
2101 if (ret) {
2102 return ret;
2103 }
2104
2105 vfio_add_ext_cap(vdev);
2106 return 0;
2107}
2108
2109static void vfio_pci_pre_reset(VFIOPCIDevice *vdev)
2110{
2111 PCIDevice *pdev = &vdev->pdev;
2112 uint16_t cmd;
2113
2114 vfio_disable_interrupts(vdev);
2115
2116 /* Make sure the device is in D0 */
2117 if (vdev->pm_cap) {
2118 uint16_t pmcsr;
2119 uint8_t state;
2120
2121 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2122 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2123 if (state) {
2124 pmcsr &= ~PCI_PM_CTRL_STATE_MASK;
2125 vfio_pci_write_config(pdev, vdev->pm_cap + PCI_PM_CTRL, pmcsr, 2);
2126 /* vfio handles the necessary delay here */
2127 pmcsr = vfio_pci_read_config(pdev, vdev->pm_cap + PCI_PM_CTRL, 2);
2128 state = pmcsr & PCI_PM_CTRL_STATE_MASK;
2129 if (state) {
2130 error_report("vfio: Unable to power on device, stuck in D%d",
2131 state);
2132 }
2133 }
2134 }
2135
2136 /*
2137 * Stop any ongoing DMA by disconecting I/O, MMIO, and bus master.
2138 * Also put INTx Disable in known state.
2139 */
2140 cmd = vfio_pci_read_config(pdev, PCI_COMMAND, 2);
2141 cmd &= ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER |
2142 PCI_COMMAND_INTX_DISABLE);
2143 vfio_pci_write_config(pdev, PCI_COMMAND, cmd, 2);
2144}
2145
2146static void vfio_pci_post_reset(VFIOPCIDevice *vdev)
2147{
2148 Error *err = NULL;
2149 int nr;
2150
2151 vfio_intx_enable(vdev, &err);
2152 if (err) {
2153 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2154 }
2155
2156 for (nr = 0; nr < PCI_NUM_REGIONS - 1; ++nr) {
2157 off_t addr = vdev->config_offset + PCI_BASE_ADDRESS_0 + (4 * nr);
2158 uint32_t val = 0;
2159 uint32_t len = sizeof(val);
2160
2161 if (pwrite(vdev->vbasedev.fd, &val, len, addr) != len) {
2162 error_report("%s(%s) reset bar %d failed: %m", __func__,
2163 vdev->vbasedev.name, nr);
2164 }
2165 }
2166
2167 vfio_quirk_reset(vdev);
2168}
2169
2170static bool vfio_pci_host_match(PCIHostDeviceAddress *addr, const char *name)
2171{
2172 char tmp[13];
2173
2174 sprintf(tmp, "%04x:%02x:%02x.%1x", addr->domain,
2175 addr->bus, addr->slot, addr->function);
2176
2177 return (strcmp(tmp, name) == 0);
2178}
2179
2180static int vfio_pci_hot_reset(VFIOPCIDevice *vdev, bool single)
2181{
2182 VFIOGroup *group;
2183 struct vfio_pci_hot_reset_info *info;
2184 struct vfio_pci_dependent_device *devices;
2185 struct vfio_pci_hot_reset *reset;
2186 int32_t *fds;
2187 int ret, i, count;
2188 bool multi = false;
2189
2190 trace_vfio_pci_hot_reset(vdev->vbasedev.name, single ? "one" : "multi");
2191
2192 if (!single) {
2193 vfio_pci_pre_reset(vdev);
2194 }
2195 vdev->vbasedev.needs_reset = false;
2196
2197 info = g_malloc0(sizeof(*info));
2198 info->argsz = sizeof(*info);
2199
2200 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2201 if (ret && errno != ENOSPC) {
2202 ret = -errno;
2203 if (!vdev->has_pm_reset) {
2204 error_report("vfio: Cannot reset device %s, "
2205 "no available reset mechanism.", vdev->vbasedev.name);
2206 }
2207 goto out_single;
2208 }
2209
2210 count = info->count;
2211 info = g_realloc(info, sizeof(*info) + (count * sizeof(*devices)));
2212 info->argsz = sizeof(*info) + (count * sizeof(*devices));
2213 devices = &info->devices[0];
2214
2215 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_PCI_HOT_RESET_INFO, info);
2216 if (ret) {
2217 ret = -errno;
2218 error_report("vfio: hot reset info failed: %m");
2219 goto out_single;
2220 }
2221
2222 trace_vfio_pci_hot_reset_has_dep_devices(vdev->vbasedev.name);
2223
2224 /* Verify that we have all the groups required */
2225 for (i = 0; i < info->count; i++) {
2226 PCIHostDeviceAddress host;
2227 VFIOPCIDevice *tmp;
2228 VFIODevice *vbasedev_iter;
2229
2230 host.domain = devices[i].segment;
2231 host.bus = devices[i].bus;
2232 host.slot = PCI_SLOT(devices[i].devfn);
2233 host.function = PCI_FUNC(devices[i].devfn);
2234
2235 trace_vfio_pci_hot_reset_dep_devices(host.domain,
2236 host.bus, host.slot, host.function, devices[i].group_id);
2237
2238 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2239 continue;
2240 }
2241
2242 QLIST_FOREACH(group, &vfio_group_list, next) {
2243 if (group->groupid == devices[i].group_id) {
2244 break;
2245 }
2246 }
2247
2248 if (!group) {
2249 if (!vdev->has_pm_reset) {
2250 error_report("vfio: Cannot reset device %s, "
2251 "depends on group %d which is not owned.",
2252 vdev->vbasedev.name, devices[i].group_id);
2253 }
2254 ret = -EPERM;
2255 goto out;
2256 }
2257
2258 /* Prep dependent devices for reset and clear our marker. */
2259 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2260 if (!vbasedev_iter->dev->realized ||
2261 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2262 continue;
2263 }
2264 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2265 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2266 if (single) {
2267 ret = -EINVAL;
2268 goto out_single;
2269 }
2270 vfio_pci_pre_reset(tmp);
2271 tmp->vbasedev.needs_reset = false;
2272 multi = true;
2273 break;
2274 }
2275 }
2276 }
2277
2278 if (!single && !multi) {
2279 ret = -EINVAL;
2280 goto out_single;
2281 }
2282
2283 /* Determine how many group fds need to be passed */
2284 count = 0;
2285 QLIST_FOREACH(group, &vfio_group_list, next) {
2286 for (i = 0; i < info->count; i++) {
2287 if (group->groupid == devices[i].group_id) {
2288 count++;
2289 break;
2290 }
2291 }
2292 }
2293
2294 reset = g_malloc0(sizeof(*reset) + (count * sizeof(*fds)));
2295 reset->argsz = sizeof(*reset) + (count * sizeof(*fds));
2296 fds = &reset->group_fds[0];
2297
2298 /* Fill in group fds */
2299 QLIST_FOREACH(group, &vfio_group_list, next) {
2300 for (i = 0; i < info->count; i++) {
2301 if (group->groupid == devices[i].group_id) {
2302 fds[reset->count++] = group->fd;
2303 break;
2304 }
2305 }
2306 }
2307
2308 /* Bus reset! */
2309 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_PCI_HOT_RESET, reset);
2310 g_free(reset);
2311
2312 trace_vfio_pci_hot_reset_result(vdev->vbasedev.name,
2313 ret ? "%m" : "Success");
2314
2315out:
2316 /* Re-enable INTx on affected devices */
2317 for (i = 0; i < info->count; i++) {
2318 PCIHostDeviceAddress host;
2319 VFIOPCIDevice *tmp;
2320 VFIODevice *vbasedev_iter;
2321
2322 host.domain = devices[i].segment;
2323 host.bus = devices[i].bus;
2324 host.slot = PCI_SLOT(devices[i].devfn);
2325 host.function = PCI_FUNC(devices[i].devfn);
2326
2327 if (vfio_pci_host_match(&host, vdev->vbasedev.name)) {
2328 continue;
2329 }
2330
2331 QLIST_FOREACH(group, &vfio_group_list, next) {
2332 if (group->groupid == devices[i].group_id) {
2333 break;
2334 }
2335 }
2336
2337 if (!group) {
2338 break;
2339 }
2340
2341 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2342 if (!vbasedev_iter->dev->realized ||
2343 vbasedev_iter->type != VFIO_DEVICE_TYPE_PCI) {
2344 continue;
2345 }
2346 tmp = container_of(vbasedev_iter, VFIOPCIDevice, vbasedev);
2347 if (vfio_pci_host_match(&host, tmp->vbasedev.name)) {
2348 vfio_pci_post_reset(tmp);
2349 break;
2350 }
2351 }
2352 }
2353out_single:
2354 if (!single) {
2355 vfio_pci_post_reset(vdev);
2356 }
2357 g_free(info);
2358
2359 return ret;
2360}
2361
2362/*
2363 * We want to differentiate hot reset of mulitple in-use devices vs hot reset
2364 * of a single in-use device. VFIO_DEVICE_RESET will already handle the case
2365 * of doing hot resets when there is only a single device per bus. The in-use
2366 * here refers to how many VFIODevices are affected. A hot reset that affects
2367 * multiple devices, but only a single in-use device, means that we can call
2368 * it from our bus ->reset() callback since the extent is effectively a single
2369 * device. This allows us to make use of it in the hotplug path. When there
2370 * are multiple in-use devices, we can only trigger the hot reset during a
2371 * system reset and thus from our reset handler. We separate _one vs _multi
2372 * here so that we don't overlap and do a double reset on the system reset
2373 * path where both our reset handler and ->reset() callback are used. Calling
2374 * _one() will only do a hot reset for the one in-use devices case, calling
2375 * _multi() will do nothing if a _one() would have been sufficient.
2376 */
2377static int vfio_pci_hot_reset_one(VFIOPCIDevice *vdev)
2378{
2379 return vfio_pci_hot_reset(vdev, true);
2380}
2381
2382static int vfio_pci_hot_reset_multi(VFIODevice *vbasedev)
2383{
2384 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2385 return vfio_pci_hot_reset(vdev, false);
2386}
2387
2388static void vfio_pci_compute_needs_reset(VFIODevice *vbasedev)
2389{
2390 VFIOPCIDevice *vdev = container_of(vbasedev, VFIOPCIDevice, vbasedev);
2391 if (!vbasedev->reset_works || (!vdev->has_flr && vdev->has_pm_reset)) {
2392 vbasedev->needs_reset = true;
2393 }
2394}
2395
2396static VFIODeviceOps vfio_pci_ops = {
2397 .vfio_compute_needs_reset = vfio_pci_compute_needs_reset,
2398 .vfio_hot_reset_multi = vfio_pci_hot_reset_multi,
2399 .vfio_eoi = vfio_intx_eoi,
2400};
2401
2402int vfio_populate_vga(VFIOPCIDevice *vdev, Error **errp)
2403{
2404 VFIODevice *vbasedev = &vdev->vbasedev;
2405 struct vfio_region_info *reg_info;
2406 int ret;
2407
2408 ret = vfio_get_region_info(vbasedev, VFIO_PCI_VGA_REGION_INDEX, &reg_info);
2409 if (ret) {
2410 error_setg_errno(errp, -ret,
2411 "failed getting region info for VGA region index %d",
2412 VFIO_PCI_VGA_REGION_INDEX);
2413 return ret;
2414 }
2415
2416 if (!(reg_info->flags & VFIO_REGION_INFO_FLAG_READ) ||
2417 !(reg_info->flags & VFIO_REGION_INFO_FLAG_WRITE) ||
2418 reg_info->size < 0xbffff + 1) {
2419 error_setg(errp, "unexpected VGA info, flags 0x%lx, size 0x%lx",
2420 (unsigned long)reg_info->flags,
2421 (unsigned long)reg_info->size);
2422 g_free(reg_info);
2423 return -EINVAL;
2424 }
2425
2426 vdev->vga = g_new0(VFIOVGA, 1);
2427
2428 vdev->vga->fd_offset = reg_info->offset;
2429 vdev->vga->fd = vdev->vbasedev.fd;
2430
2431 g_free(reg_info);
2432
2433 vdev->vga->region[QEMU_PCI_VGA_MEM].offset = QEMU_PCI_VGA_MEM_BASE;
2434 vdev->vga->region[QEMU_PCI_VGA_MEM].nr = QEMU_PCI_VGA_MEM;
2435 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_MEM].quirks);
2436
2437 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2438 OBJECT(vdev), &vfio_vga_ops,
2439 &vdev->vga->region[QEMU_PCI_VGA_MEM],
2440 "vfio-vga-mmio@0xa0000",
2441 QEMU_PCI_VGA_MEM_SIZE);
2442
2443 vdev->vga->region[QEMU_PCI_VGA_IO_LO].offset = QEMU_PCI_VGA_IO_LO_BASE;
2444 vdev->vga->region[QEMU_PCI_VGA_IO_LO].nr = QEMU_PCI_VGA_IO_LO;
2445 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].quirks);
2446
2447 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2448 OBJECT(vdev), &vfio_vga_ops,
2449 &vdev->vga->region[QEMU_PCI_VGA_IO_LO],
2450 "vfio-vga-io@0x3b0",
2451 QEMU_PCI_VGA_IO_LO_SIZE);
2452
2453 vdev->vga->region[QEMU_PCI_VGA_IO_HI].offset = QEMU_PCI_VGA_IO_HI_BASE;
2454 vdev->vga->region[QEMU_PCI_VGA_IO_HI].nr = QEMU_PCI_VGA_IO_HI;
2455 QLIST_INIT(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].quirks);
2456
2457 memory_region_init_io(&vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem,
2458 OBJECT(vdev), &vfio_vga_ops,
2459 &vdev->vga->region[QEMU_PCI_VGA_IO_HI],
2460 "vfio-vga-io@0x3c0",
2461 QEMU_PCI_VGA_IO_HI_SIZE);
2462
2463 pci_register_vga(&vdev->pdev, &vdev->vga->region[QEMU_PCI_VGA_MEM].mem,
2464 &vdev->vga->region[QEMU_PCI_VGA_IO_LO].mem,
2465 &vdev->vga->region[QEMU_PCI_VGA_IO_HI].mem);
2466
2467 return 0;
2468}
2469
2470static void vfio_populate_device(VFIOPCIDevice *vdev, Error **errp)
2471{
2472 VFIODevice *vbasedev = &vdev->vbasedev;
2473 struct vfio_region_info *reg_info;
2474 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info) };
2475 int i, ret = -1;
2476
2477 /* Sanity check device */
2478 if (!(vbasedev->flags & VFIO_DEVICE_FLAGS_PCI)) {
2479 error_setg(errp, "this isn't a PCI device");
2480 return;
2481 }
2482
2483 if (vbasedev->num_regions < VFIO_PCI_CONFIG_REGION_INDEX + 1) {
2484 error_setg(errp, "unexpected number of io regions %u",
2485 vbasedev->num_regions);
2486 return;
2487 }
2488
2489 if (vbasedev->num_irqs < VFIO_PCI_MSIX_IRQ_INDEX + 1) {
2490 error_setg(errp, "unexpected number of irqs %u", vbasedev->num_irqs);
2491 return;
2492 }
2493
2494 for (i = VFIO_PCI_BAR0_REGION_INDEX; i < VFIO_PCI_ROM_REGION_INDEX; i++) {
2495 char *name = g_strdup_printf("%s BAR %d", vbasedev->name, i);
2496
2497 ret = vfio_region_setup(OBJECT(vdev), vbasedev,
2498 &vdev->bars[i].region, i, name);
2499 g_free(name);
2500
2501 if (ret) {
2502 error_setg_errno(errp, -ret, "failed to get region %d info", i);
2503 return;
2504 }
2505
2506 QLIST_INIT(&vdev->bars[i].quirks);
2507 }
2508
2509 ret = vfio_get_region_info(vbasedev,
2510 VFIO_PCI_CONFIG_REGION_INDEX, &reg_info);
2511 if (ret) {
2512 error_setg_errno(errp, -ret, "failed to get config info");
2513 return;
2514 }
2515
2516 trace_vfio_populate_device_config(vdev->vbasedev.name,
2517 (unsigned long)reg_info->size,
2518 (unsigned long)reg_info->offset,
2519 (unsigned long)reg_info->flags);
2520
2521 vdev->config_size = reg_info->size;
2522 if (vdev->config_size == PCI_CONFIG_SPACE_SIZE) {
2523 vdev->pdev.cap_present &= ~QEMU_PCI_CAP_EXPRESS;
2524 }
2525 vdev->config_offset = reg_info->offset;
2526
2527 g_free(reg_info);
2528
2529 if (vdev->features & VFIO_FEATURE_ENABLE_VGA) {
2530 ret = vfio_populate_vga(vdev, errp);
2531 if (ret) {
2532 error_append_hint(errp, "device does not support "
2533 "requested feature x-vga\n");
2534 return;
2535 }
2536 }
2537
2538 irq_info.index = VFIO_PCI_ERR_IRQ_INDEX;
2539
2540 ret = ioctl(vdev->vbasedev.fd, VFIO_DEVICE_GET_IRQ_INFO, &irq_info);
2541 if (ret) {
2542 /* This can fail for an old kernel or legacy PCI dev */
2543 trace_vfio_populate_device_get_irq_info_failure(strerror(errno));
2544 } else if (irq_info.count == 1) {
2545 vdev->pci_aer = true;
2546 } else {
2547 warn_report(VFIO_MSG_PREFIX
2548 "Could not enable error recovery for the device",
2549 vbasedev->name);
2550 }
2551}
2552
2553static void vfio_put_device(VFIOPCIDevice *vdev)
2554{
2555 g_free(vdev->vbasedev.name);
2556 g_free(vdev->msix);
2557
2558 vfio_put_base_device(&vdev->vbasedev);
2559}
2560
2561static void vfio_err_notifier_handler(void *opaque)
2562{
2563 VFIOPCIDevice *vdev = opaque;
2564
2565 if (!event_notifier_test_and_clear(&vdev->err_notifier)) {
2566 return;
2567 }
2568
2569 /*
2570 * TBD. Retrieve the error details and decide what action
2571 * needs to be taken. One of the actions could be to pass
2572 * the error to the guest and have the guest driver recover
2573 * from the error. This requires that PCIe capabilities be
2574 * exposed to the guest. For now, we just terminate the
2575 * guest to contain the error.
2576 */
2577
2578 error_report("%s(%s) Unrecoverable error detected. Please collect any data possible and then kill the guest", __func__, vdev->vbasedev.name);
2579
2580 vm_stop(RUN_STATE_INTERNAL_ERROR);
2581}
2582
2583/*
2584 * Registers error notifier for devices supporting error recovery.
2585 * If we encounter a failure in this function, we report an error
2586 * and continue after disabling error recovery support for the
2587 * device.
2588 */
2589static void vfio_register_err_notifier(VFIOPCIDevice *vdev)
2590{
2591 Error *err = NULL;
2592 int32_t fd;
2593
2594 if (!vdev->pci_aer) {
2595 return;
2596 }
2597
2598 if (event_notifier_init(&vdev->err_notifier, 0)) {
2599 error_report("vfio: Unable to init event notifier for error detection");
2600 vdev->pci_aer = false;
2601 return;
2602 }
2603
2604 fd = event_notifier_get_fd(&vdev->err_notifier);
2605 qemu_set_fd_handler(fd, vfio_err_notifier_handler, NULL, vdev);
2606
2607 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2608 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2609 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2610 qemu_set_fd_handler(fd, NULL, NULL, vdev);
2611 event_notifier_cleanup(&vdev->err_notifier);
2612 vdev->pci_aer = false;
2613 }
2614}
2615
2616static void vfio_unregister_err_notifier(VFIOPCIDevice *vdev)
2617{
2618 Error *err = NULL;
2619
2620 if (!vdev->pci_aer) {
2621 return;
2622 }
2623
2624 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_ERR_IRQ_INDEX, 0,
2625 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
2626 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2627 }
2628 qemu_set_fd_handler(event_notifier_get_fd(&vdev->err_notifier),
2629 NULL, NULL, vdev);
2630 event_notifier_cleanup(&vdev->err_notifier);
2631}
2632
2633static void vfio_req_notifier_handler(void *opaque)
2634{
2635 VFIOPCIDevice *vdev = opaque;
2636 Error *err = NULL;
2637
2638 if (!event_notifier_test_and_clear(&vdev->req_notifier)) {
2639 return;
2640 }
2641
2642 qdev_unplug(DEVICE(vdev), &err);
2643 if (err) {
2644 warn_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2645 }
2646}
2647
2648static void vfio_register_req_notifier(VFIOPCIDevice *vdev)
2649{
2650 struct vfio_irq_info irq_info = { .argsz = sizeof(irq_info),
2651 .index = VFIO_PCI_REQ_IRQ_INDEX };
2652 Error *err = NULL;
2653 int32_t fd;
2654
2655 if (!(vdev->features & VFIO_FEATURE_ENABLE_REQ)) {
2656 return;
2657 }
2658
2659 if (ioctl(vdev->vbasedev.fd,
2660 VFIO_DEVICE_GET_IRQ_INFO, &irq_info) < 0 || irq_info.count < 1) {
2661 return;
2662 }
2663
2664 if (event_notifier_init(&vdev->req_notifier, 0)) {
2665 error_report("vfio: Unable to init event notifier for device request");
2666 return;
2667 }
2668
2669 fd = event_notifier_get_fd(&vdev->req_notifier);
2670 qemu_set_fd_handler(fd, vfio_req_notifier_handler, NULL, vdev);
2671
2672 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
2673 VFIO_IRQ_SET_ACTION_TRIGGER, fd, &err)) {
2674 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2675 qemu_set_fd_handler(fd, NULL, NULL, vdev);
2676 event_notifier_cleanup(&vdev->req_notifier);
2677 } else {
2678 vdev->req_enabled = true;
2679 }
2680}
2681
2682static void vfio_unregister_req_notifier(VFIOPCIDevice *vdev)
2683{
2684 Error *err = NULL;
2685
2686 if (!vdev->req_enabled) {
2687 return;
2688 }
2689
2690 if (vfio_set_irq_signaling(&vdev->vbasedev, VFIO_PCI_REQ_IRQ_INDEX, 0,
2691 VFIO_IRQ_SET_ACTION_TRIGGER, -1, &err)) {
2692 error_reportf_err(err, VFIO_MSG_PREFIX, vdev->vbasedev.name);
2693 }
2694 qemu_set_fd_handler(event_notifier_get_fd(&vdev->req_notifier),
2695 NULL, NULL, vdev);
2696 event_notifier_cleanup(&vdev->req_notifier);
2697
2698 vdev->req_enabled = false;
2699}
2700
2701static void vfio_realize(PCIDevice *pdev, Error **errp)
2702{
2703 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
2704 VFIODevice *vbasedev_iter;
2705 VFIOGroup *group;
2706 char *tmp, *subsys, group_path[PATH_MAX], *group_name;
2707 Error *err = NULL;
2708 ssize_t len;
2709 struct stat st;
2710 int groupid;
2711 int i, ret;
2712 bool is_mdev;
2713
2714 if (!vdev->vbasedev.sysfsdev) {
2715 if (!(~vdev->host.domain || ~vdev->host.bus ||
2716 ~vdev->host.slot || ~vdev->host.function)) {
2717 error_setg(errp, "No provided host device");
2718 error_append_hint(errp, "Use -device vfio-pci,host=DDDD:BB:DD.F "
2719 "or -device vfio-pci,sysfsdev=PATH_TO_DEVICE\n");
2720 return;
2721 }
2722 vdev->vbasedev.sysfsdev =
2723 g_strdup_printf("/sys/bus/pci/devices/%04x:%02x:%02x.%01x",
2724 vdev->host.domain, vdev->host.bus,
2725 vdev->host.slot, vdev->host.function);
2726 }
2727
2728 if (stat(vdev->vbasedev.sysfsdev, &st) < 0) {
2729 error_setg_errno(errp, errno, "no such host device");
2730 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.sysfsdev);
2731 return;
2732 }
2733
2734 vdev->vbasedev.name = g_path_get_basename(vdev->vbasedev.sysfsdev);
2735 vdev->vbasedev.ops = &vfio_pci_ops;
2736 vdev->vbasedev.type = VFIO_DEVICE_TYPE_PCI;
2737 vdev->vbasedev.dev = DEVICE(vdev);
2738
2739 tmp = g_strdup_printf("%s/iommu_group", vdev->vbasedev.sysfsdev);
2740 len = readlink(tmp, group_path, sizeof(group_path));
2741 g_free(tmp);
2742
2743 if (len <= 0 || len >= sizeof(group_path)) {
2744 error_setg_errno(errp, len < 0 ? errno : ENAMETOOLONG,
2745 "no iommu_group found");
2746 goto error;
2747 }
2748
2749 group_path[len] = 0;
2750
2751 group_name = basename(group_path);
2752 if (sscanf(group_name, "%d", &groupid) != 1) {
2753 error_setg_errno(errp, errno, "failed to read %s", group_path);
2754 goto error;
2755 }
2756
2757 trace_vfio_realize(vdev->vbasedev.name, groupid);
2758
2759 group = vfio_get_group(groupid, pci_device_iommu_address_space(pdev), errp);
2760 if (!group) {
2761 goto error;
2762 }
2763
2764 QLIST_FOREACH(vbasedev_iter, &group->device_list, next) {
2765 if (strcmp(vbasedev_iter->name, vdev->vbasedev.name) == 0) {
2766 error_setg(errp, "device is already attached");
2767 vfio_put_group(group);
2768 goto error;
2769 }
2770 }
2771
2772 /*
2773 * Mediated devices *might* operate compatibly with memory ballooning, but
2774 * we cannot know for certain, it depends on whether the mdev vendor driver
2775 * stays in sync with the active working set of the guest driver. Prevent
2776 * the x-balloon-allowed option unless this is minimally an mdev device.
2777 */
2778 tmp = g_strdup_printf("%s/subsystem", vdev->vbasedev.sysfsdev);
2779 subsys = realpath(tmp, NULL);
2780 g_free(tmp);
2781 is_mdev = subsys && (strcmp(subsys, "/sys/bus/mdev") == 0);
2782 free(subsys);
2783
2784 trace_vfio_mdev(vdev->vbasedev.name, is_mdev);
2785
2786 if (vdev->vbasedev.balloon_allowed && !is_mdev) {
2787 error_setg(errp, "x-balloon-allowed only potentially compatible "
2788 "with mdev devices");
2789 vfio_put_group(group);
2790 goto error;
2791 }
2792
2793 ret = vfio_get_device(group, vdev->vbasedev.name, &vdev->vbasedev, errp);
2794 if (ret) {
2795 vfio_put_group(group);
2796 goto error;
2797 }
2798
2799 vfio_populate_device(vdev, &err);
2800 if (err) {
2801 error_propagate(errp, err);
2802 goto error;
2803 }
2804
2805 /* Get a copy of config space */
2806 ret = pread(vdev->vbasedev.fd, vdev->pdev.config,
2807 MIN(pci_config_size(&vdev->pdev), vdev->config_size),
2808 vdev->config_offset);
2809 if (ret < (int)MIN(pci_config_size(&vdev->pdev), vdev->config_size)) {
2810 ret = ret < 0 ? -errno : -EFAULT;
2811 error_setg_errno(errp, -ret, "failed to read device config space");
2812 goto error;
2813 }
2814
2815 /* vfio emulates a lot for us, but some bits need extra love */
2816 vdev->emulated_config_bits = g_malloc0(vdev->config_size);
2817
2818 /* QEMU can choose to expose the ROM or not */
2819 memset(vdev->emulated_config_bits + PCI_ROM_ADDRESS, 0xff, 4);
2820 /* QEMU can also add or extend BARs */
2821 memset(vdev->emulated_config_bits + PCI_BASE_ADDRESS_0, 0xff, 6 * 4);
2822
2823 /*
2824 * The PCI spec reserves vendor ID 0xffff as an invalid value. The
2825 * device ID is managed by the vendor and need only be a 16-bit value.
2826 * Allow any 16-bit value for subsystem so they can be hidden or changed.
2827 */
2828 if (vdev->vendor_id != PCI_ANY_ID) {
2829 if (vdev->vendor_id >= 0xffff) {
2830 error_setg(errp, "invalid PCI vendor ID provided");
2831 goto error;
2832 }
2833 vfio_add_emulated_word(vdev, PCI_VENDOR_ID, vdev->vendor_id, ~0);
2834 trace_vfio_pci_emulated_vendor_id(vdev->vbasedev.name, vdev->vendor_id);
2835 } else {
2836 vdev->vendor_id = pci_get_word(pdev->config + PCI_VENDOR_ID);
2837 }
2838
2839 if (vdev->device_id != PCI_ANY_ID) {
2840 if (vdev->device_id > 0xffff) {
2841 error_setg(errp, "invalid PCI device ID provided");
2842 goto error;
2843 }
2844 vfio_add_emulated_word(vdev, PCI_DEVICE_ID, vdev->device_id, ~0);
2845 trace_vfio_pci_emulated_device_id(vdev->vbasedev.name, vdev->device_id);
2846 } else {
2847 vdev->device_id = pci_get_word(pdev->config + PCI_DEVICE_ID);
2848 }
2849
2850 if (vdev->sub_vendor_id != PCI_ANY_ID) {
2851 if (vdev->sub_vendor_id > 0xffff) {
2852 error_setg(errp, "invalid PCI subsystem vendor ID provided");
2853 goto error;
2854 }
2855 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_VENDOR_ID,
2856 vdev->sub_vendor_id, ~0);
2857 trace_vfio_pci_emulated_sub_vendor_id(vdev->vbasedev.name,
2858 vdev->sub_vendor_id);
2859 }
2860
2861 if (vdev->sub_device_id != PCI_ANY_ID) {
2862 if (vdev->sub_device_id > 0xffff) {
2863 error_setg(errp, "invalid PCI subsystem device ID provided");
2864 goto error;
2865 }
2866 vfio_add_emulated_word(vdev, PCI_SUBSYSTEM_ID, vdev->sub_device_id, ~0);
2867 trace_vfio_pci_emulated_sub_device_id(vdev->vbasedev.name,
2868 vdev->sub_device_id);
2869 }
2870
2871 /* QEMU can change multi-function devices to single function, or reverse */
2872 vdev->emulated_config_bits[PCI_HEADER_TYPE] =
2873 PCI_HEADER_TYPE_MULTI_FUNCTION;
2874
2875 /* Restore or clear multifunction, this is always controlled by QEMU */
2876 if (vdev->pdev.cap_present & QEMU_PCI_CAP_MULTIFUNCTION) {
2877 vdev->pdev.config[PCI_HEADER_TYPE] |= PCI_HEADER_TYPE_MULTI_FUNCTION;
2878 } else {
2879 vdev->pdev.config[PCI_HEADER_TYPE] &= ~PCI_HEADER_TYPE_MULTI_FUNCTION;
2880 }
2881
2882 /*
2883 * Clear host resource mapping info. If we choose not to register a
2884 * BAR, such as might be the case with the option ROM, we can get
2885 * confusing, unwritable, residual addresses from the host here.
2886 */
2887 memset(&vdev->pdev.config[PCI_BASE_ADDRESS_0], 0, 24);
2888 memset(&vdev->pdev.config[PCI_ROM_ADDRESS], 0, 4);
2889
2890 vfio_pci_size_rom(vdev);
2891
2892 vfio_bars_prepare(vdev);
2893
2894 vfio_msix_early_setup(vdev, &err);
2895 if (err) {
2896 error_propagate(errp, err);
2897 goto error;
2898 }
2899
2900 vfio_bars_register(vdev);
2901
2902 ret = vfio_add_capabilities(vdev, errp);
2903 if (ret) {
2904 goto out_teardown;
2905 }
2906
2907 if (vdev->vga) {
2908 vfio_vga_quirk_setup(vdev);
2909 }
2910
2911 for (i = 0; i < PCI_ROM_SLOT; i++) {
2912 vfio_bar_quirk_setup(vdev, i);
2913 }
2914
2915 if (!vdev->igd_opregion &&
2916 vdev->features & VFIO_FEATURE_ENABLE_IGD_OPREGION) {
2917 struct vfio_region_info *opregion;
2918
2919 if (vdev->pdev.qdev.hotplugged) {
2920 error_setg(errp,
2921 "cannot support IGD OpRegion feature on hotplugged "
2922 "device");
2923 goto out_teardown;
2924 }
2925
2926 ret = vfio_get_dev_region_info(&vdev->vbasedev,
2927 VFIO_REGION_TYPE_PCI_VENDOR_TYPE | PCI_VENDOR_ID_INTEL,
2928 VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION, &opregion);
2929 if (ret) {
2930 error_setg_errno(errp, -ret,
2931 "does not support requested IGD OpRegion feature");
2932 goto out_teardown;
2933 }
2934
2935 ret = vfio_pci_igd_opregion_init(vdev, opregion, errp);
2936 g_free(opregion);
2937 if (ret) {
2938 goto out_teardown;
2939 }
2940 }
2941
2942 /* QEMU emulates all of MSI & MSIX */
2943 if (pdev->cap_present & QEMU_PCI_CAP_MSIX) {
2944 memset(vdev->emulated_config_bits + pdev->msix_cap, 0xff,
2945 MSIX_CAP_LENGTH);
2946 }
2947
2948 if (pdev->cap_present & QEMU_PCI_CAP_MSI) {
2949 memset(vdev->emulated_config_bits + pdev->msi_cap, 0xff,
2950 vdev->msi_cap_size);
2951 }
2952
2953 if (vfio_pci_read_config(&vdev->pdev, PCI_INTERRUPT_PIN, 1)) {
2954 vdev->intx.mmap_timer = timer_new_ms(QEMU_CLOCK_VIRTUAL,
2955 vfio_intx_mmap_enable, vdev);
2956 pci_device_set_intx_routing_notifier(&vdev->pdev, vfio_intx_update);
2957 ret = vfio_intx_enable(vdev, errp);
2958 if (ret) {
2959 goto out_teardown;
2960 }
2961 }
2962
2963 if (vdev->display != ON_OFF_AUTO_OFF) {
2964 ret = vfio_display_probe(vdev, errp);
2965 if (ret) {
2966 goto out_teardown;
2967 }
2968 }
2969 if (vdev->enable_ramfb && vdev->dpy == NULL) {
2970 error_setg(errp, "ramfb=on requires display=on");
2971 goto out_teardown;
2972 }
2973 if (vdev->display_xres || vdev->display_yres) {
2974 if (vdev->dpy == NULL) {
2975 error_setg(errp, "xres and yres properties require display=on");
2976 goto out_teardown;
2977 }
2978 if (vdev->dpy->edid_regs == NULL) {
2979 error_setg(errp, "xres and yres properties need edid support");
2980 goto out_teardown;
2981 }
2982 }
2983
2984 if (vdev->vendor_id == PCI_VENDOR_ID_NVIDIA) {
2985 ret = vfio_pci_nvidia_v100_ram_init(vdev, errp);
2986 if (ret && ret != -ENODEV) {
2987 error_report("Failed to setup NVIDIA V100 GPU RAM");
2988 }
2989 }
2990
2991 if (vdev->vendor_id == PCI_VENDOR_ID_IBM) {
2992 ret = vfio_pci_nvlink2_init(vdev, errp);
2993 if (ret && ret != -ENODEV) {
2994 error_report("Failed to setup NVlink2 bridge");
2995 }
2996 }
2997
2998 vfio_register_err_notifier(vdev);
2999 vfio_register_req_notifier(vdev);
3000 vfio_setup_resetfn_quirk(vdev);
3001
3002 return;
3003
3004out_teardown:
3005 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3006 vfio_teardown_msi(vdev);
3007 vfio_bars_exit(vdev);
3008error:
3009 error_prepend(errp, VFIO_MSG_PREFIX, vdev->vbasedev.name);
3010}
3011
3012static void vfio_instance_finalize(Object *obj)
3013{
3014 VFIOPCIDevice *vdev = PCI_VFIO(obj);
3015 VFIOGroup *group = vdev->vbasedev.group;
3016
3017 vfio_display_finalize(vdev);
3018 vfio_bars_finalize(vdev);
3019 g_free(vdev->emulated_config_bits);
3020 g_free(vdev->rom);
3021 /*
3022 * XXX Leaking igd_opregion is not an oversight, we can't remove the
3023 * fw_cfg entry therefore leaking this allocation seems like the safest
3024 * option.
3025 *
3026 * g_free(vdev->igd_opregion);
3027 */
3028 vfio_put_device(vdev);
3029 vfio_put_group(group);
3030}
3031
3032static void vfio_exitfn(PCIDevice *pdev)
3033{
3034 VFIOPCIDevice *vdev = PCI_VFIO(pdev);
3035
3036 vfio_unregister_req_notifier(vdev);
3037 vfio_unregister_err_notifier(vdev);
3038 pci_device_set_intx_routing_notifier(&vdev->pdev, NULL);
3039 vfio_disable_interrupts(vdev);
3040 if (vdev->intx.mmap_timer) {
3041 timer_free(vdev->intx.mmap_timer);
3042 }
3043 vfio_teardown_msi(vdev);
3044 vfio_bars_exit(vdev);
3045}
3046
3047static void vfio_pci_reset(DeviceState *dev)
3048{
3049 VFIOPCIDevice *vdev = PCI_VFIO(dev);
3050
3051 trace_vfio_pci_reset(vdev->vbasedev.name);
3052
3053 vfio_pci_pre_reset(vdev);
3054
3055 if (vdev->display != ON_OFF_AUTO_OFF) {
3056 vfio_display_reset(vdev);
3057 }
3058
3059 if (vdev->resetfn && !vdev->resetfn(vdev)) {
3060 goto post_reset;
3061 }
3062
3063 if (vdev->vbasedev.reset_works &&
3064 (vdev->has_flr || !vdev->has_pm_reset) &&
3065 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3066 trace_vfio_pci_reset_flr(vdev->vbasedev.name);
3067 goto post_reset;
3068 }
3069
3070 /* See if we can do our own bus reset */
3071 if (!vfio_pci_hot_reset_one(vdev)) {
3072 goto post_reset;
3073 }
3074
3075 /* If nothing else works and the device supports PM reset, use it */
3076 if (vdev->vbasedev.reset_works && vdev->has_pm_reset &&
3077 !ioctl(vdev->vbasedev.fd, VFIO_DEVICE_RESET)) {
3078 trace_vfio_pci_reset_pm(vdev->vbasedev.name);
3079 goto post_reset;
3080 }
3081
3082post_reset:
3083 vfio_pci_post_reset(vdev);
3084}
3085
3086static void vfio_instance_init(Object *obj)
3087{
3088 PCIDevice *pci_dev = PCI_DEVICE(obj);
3089 VFIOPCIDevice *vdev = PCI_VFIO(obj);
3090
3091 device_add_bootindex_property(obj, &vdev->bootindex,
3092 "bootindex", NULL,
3093 &pci_dev->qdev, NULL);
3094 vdev->host.domain = ~0U;
3095 vdev->host.bus = ~0U;
3096 vdev->host.slot = ~0U;
3097 vdev->host.function = ~0U;
3098
3099 vdev->nv_gpudirect_clique = 0xFF;
3100
3101 /* QEMU_PCI_CAP_EXPRESS initialization does not depend on QEMU command
3102 * line, therefore, no need to wait to realize like other devices */
3103 pci_dev->cap_present |= QEMU_PCI_CAP_EXPRESS;
3104}
3105
3106static Property vfio_pci_dev_properties[] = {
3107 DEFINE_PROP_PCI_HOST_DEVADDR("host", VFIOPCIDevice, host),
3108 DEFINE_PROP_STRING("sysfsdev", VFIOPCIDevice, vbasedev.sysfsdev),
3109 DEFINE_PROP_ON_OFF_AUTO("display", VFIOPCIDevice,
3110 display, ON_OFF_AUTO_OFF),
3111 DEFINE_PROP_UINT32("xres", VFIOPCIDevice, display_xres, 0),
3112 DEFINE_PROP_UINT32("yres", VFIOPCIDevice, display_yres, 0),
3113 DEFINE_PROP_UINT32("x-intx-mmap-timeout-ms", VFIOPCIDevice,
3114 intx.mmap_timeout, 1100),
3115 DEFINE_PROP_BIT("x-vga", VFIOPCIDevice, features,
3116 VFIO_FEATURE_ENABLE_VGA_BIT, false),
3117 DEFINE_PROP_BIT("x-req", VFIOPCIDevice, features,
3118 VFIO_FEATURE_ENABLE_REQ_BIT, true),
3119 DEFINE_PROP_BIT("x-igd-opregion", VFIOPCIDevice, features,
3120 VFIO_FEATURE_ENABLE_IGD_OPREGION_BIT, false),
3121 DEFINE_PROP_BOOL("x-no-mmap", VFIOPCIDevice, vbasedev.no_mmap, false),
3122 DEFINE_PROP_BOOL("x-balloon-allowed", VFIOPCIDevice,
3123 vbasedev.balloon_allowed, false),
3124 DEFINE_PROP_BOOL("x-no-kvm-intx", VFIOPCIDevice, no_kvm_intx, false),
3125 DEFINE_PROP_BOOL("x-no-kvm-msi", VFIOPCIDevice, no_kvm_msi, false),
3126 DEFINE_PROP_BOOL("x-no-kvm-msix", VFIOPCIDevice, no_kvm_msix, false),
3127 DEFINE_PROP_BOOL("x-no-geforce-quirks", VFIOPCIDevice,
3128 no_geforce_quirks, false),
3129 DEFINE_PROP_BOOL("x-no-kvm-ioeventfd", VFIOPCIDevice, no_kvm_ioeventfd,
3130 false),
3131 DEFINE_PROP_BOOL("x-no-vfio-ioeventfd", VFIOPCIDevice, no_vfio_ioeventfd,
3132 false),
3133 DEFINE_PROP_UINT32("x-pci-vendor-id", VFIOPCIDevice, vendor_id, PCI_ANY_ID),
3134 DEFINE_PROP_UINT32("x-pci-device-id", VFIOPCIDevice, device_id, PCI_ANY_ID),
3135 DEFINE_PROP_UINT32("x-pci-sub-vendor-id", VFIOPCIDevice,
3136 sub_vendor_id, PCI_ANY_ID),
3137 DEFINE_PROP_UINT32("x-pci-sub-device-id", VFIOPCIDevice,
3138 sub_device_id, PCI_ANY_ID),
3139 DEFINE_PROP_UINT32("x-igd-gms", VFIOPCIDevice, igd_gms, 0),
3140 DEFINE_PROP_UNSIGNED_NODEFAULT("x-nv-gpudirect-clique", VFIOPCIDevice,
3141 nv_gpudirect_clique,
3142 qdev_prop_nv_gpudirect_clique, uint8_t),
3143 DEFINE_PROP_OFF_AUTO_PCIBAR("x-msix-relocation", VFIOPCIDevice, msix_relo,
3144 OFF_AUTOPCIBAR_OFF),
3145 /*
3146 * TODO - support passed fds... is this necessary?
3147 * DEFINE_PROP_STRING("vfiofd", VFIOPCIDevice, vfiofd_name),
3148 * DEFINE_PROP_STRING("vfiogroupfd, VFIOPCIDevice, vfiogroupfd_name),
3149 */
3150 DEFINE_PROP_END_OF_LIST(),
3151};
3152
3153static const VMStateDescription vfio_pci_vmstate = {
3154 .name = "vfio-pci",
3155 .unmigratable = 1,
3156};
3157
3158static void vfio_pci_dev_class_init(ObjectClass *klass, void *data)
3159{
3160 DeviceClass *dc = DEVICE_CLASS(klass);
3161 PCIDeviceClass *pdc = PCI_DEVICE_CLASS(klass);
3162
3163 dc->reset = vfio_pci_reset;
3164 dc->props = vfio_pci_dev_properties;
3165 dc->vmsd = &vfio_pci_vmstate;
3166 dc->desc = "VFIO-based PCI device assignment";
3167 set_bit(DEVICE_CATEGORY_MISC, dc->categories);
3168 pdc->realize = vfio_realize;
3169 pdc->exit = vfio_exitfn;
3170 pdc->config_read = vfio_pci_read_config;
3171 pdc->config_write = vfio_pci_write_config;
3172}
3173
3174static const TypeInfo vfio_pci_dev_info = {
3175 .name = TYPE_VFIO_PCI,
3176 .parent = TYPE_PCI_DEVICE,
3177 .instance_size = sizeof(VFIOPCIDevice),
3178 .class_init = vfio_pci_dev_class_init,
3179 .instance_init = vfio_instance_init,
3180 .instance_finalize = vfio_instance_finalize,
3181 .interfaces = (InterfaceInfo[]) {
3182 { INTERFACE_PCIE_DEVICE },
3183 { INTERFACE_CONVENTIONAL_PCI_DEVICE },
3184 { }
3185 },
3186};
3187
3188static Property vfio_pci_dev_nohotplug_properties[] = {
3189 DEFINE_PROP_BOOL("ramfb", VFIOPCIDevice, enable_ramfb, false),
3190 DEFINE_PROP_END_OF_LIST(),
3191};
3192
3193static void vfio_pci_nohotplug_dev_class_init(ObjectClass *klass, void *data)
3194{
3195 DeviceClass *dc = DEVICE_CLASS(klass);
3196
3197 dc->props = vfio_pci_dev_nohotplug_properties;
3198 dc->hotpluggable = false;
3199}
3200
3201static const TypeInfo vfio_pci_nohotplug_dev_info = {
3202 .name = TYPE_VIFO_PCI_NOHOTPLUG,
3203 .parent = TYPE_VFIO_PCI,
3204 .instance_size = sizeof(VFIOPCIDevice),
3205 .class_init = vfio_pci_nohotplug_dev_class_init,
3206};
3207
3208static void register_vfio_pci_dev_type(void)
3209{
3210 type_register_static(&vfio_pci_dev_info);
3211 type_register_static(&vfio_pci_nohotplug_dev_info);
3212}
3213
3214type_init(register_vfio_pci_dev_type)
3215