1 | /* |
2 | * msi.c |
3 | * |
4 | * Copyright (c) 2010 Isaku Yamahata <yamahata at valinux co jp> |
5 | * VA Linux Systems Japan K.K. |
6 | * |
7 | * This program is free software; you can redistribute it and/or modify |
8 | * it under the terms of the GNU General Public License as published by |
9 | * the Free Software Foundation; either version 2 of the License, or |
10 | * (at your option) any later version. |
11 | |
12 | * This program is distributed in the hope that it will be useful, |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
15 | * GNU General Public License for more details. |
16 | |
17 | * You should have received a copy of the GNU General Public License along |
18 | * with this program; if not, see <http://www.gnu.org/licenses/>. |
19 | */ |
20 | |
21 | #include "qemu/osdep.h" |
22 | #include "hw/pci/msi.h" |
23 | #include "hw/xen/xen.h" |
24 | #include "qemu/range.h" |
25 | #include "qapi/error.h" |
26 | |
27 | /* PCI_MSI_ADDRESS_LO */ |
28 | #define PCI_MSI_ADDRESS_LO_MASK (~0x3) |
29 | |
30 | /* If we get rid of cap allocator, we won't need those. */ |
31 | #define PCI_MSI_32_SIZEOF 0x0a |
32 | #define PCI_MSI_64_SIZEOF 0x0e |
33 | #define PCI_MSI_32M_SIZEOF 0x14 |
34 | #define PCI_MSI_64M_SIZEOF 0x18 |
35 | |
36 | #define PCI_MSI_VECTORS_MAX 32 |
37 | |
38 | /* |
39 | * Flag for interrupt controllers to declare broken MSI/MSI-X support. |
40 | * values: false - broken; true - non-broken. |
41 | * |
42 | * Setting this flag to false will remove MSI/MSI-X capability from all devices. |
43 | * |
44 | * It is preferable for controllers to set this to true (non-broken) even if |
45 | * they do not actually support MSI/MSI-X: guests normally probe the controller |
46 | * type and do not attempt to enable MSI/MSI-X with interrupt controllers not |
47 | * supporting such, so removing the capability is not required, and |
48 | * it seems cleaner to have a given device look the same for all boards. |
49 | * |
50 | * TODO: some existing controllers violate the above rule. Identify and fix them. |
51 | */ |
52 | bool msi_nonbroken; |
53 | |
54 | /* If we get rid of cap allocator, we won't need this. */ |
55 | static inline uint8_t msi_cap_sizeof(uint16_t flags) |
56 | { |
57 | switch (flags & (PCI_MSI_FLAGS_MASKBIT | PCI_MSI_FLAGS_64BIT)) { |
58 | case PCI_MSI_FLAGS_MASKBIT | PCI_MSI_FLAGS_64BIT: |
59 | return PCI_MSI_64M_SIZEOF; |
60 | case PCI_MSI_FLAGS_64BIT: |
61 | return PCI_MSI_64_SIZEOF; |
62 | case PCI_MSI_FLAGS_MASKBIT: |
63 | return PCI_MSI_32M_SIZEOF; |
64 | case 0: |
65 | return PCI_MSI_32_SIZEOF; |
66 | default: |
67 | abort(); |
68 | break; |
69 | } |
70 | return 0; |
71 | } |
72 | |
73 | //#define MSI_DEBUG |
74 | |
75 | #ifdef MSI_DEBUG |
76 | # define MSI_DPRINTF(fmt, ...) \ |
77 | fprintf(stderr, "%s:%d " fmt, __func__, __LINE__, ## __VA_ARGS__) |
78 | #else |
79 | # define MSI_DPRINTF(fmt, ...) do { } while (0) |
80 | #endif |
81 | #define MSI_DEV_PRINTF(dev, fmt, ...) \ |
82 | MSI_DPRINTF("%s:%x " fmt, (dev)->name, (dev)->devfn, ## __VA_ARGS__) |
83 | |
84 | static inline unsigned int msi_nr_vectors(uint16_t flags) |
85 | { |
86 | return 1U << |
87 | ((flags & PCI_MSI_FLAGS_QSIZE) >> ctz32(PCI_MSI_FLAGS_QSIZE)); |
88 | } |
89 | |
90 | static inline uint8_t msi_flags_off(const PCIDevice* dev) |
91 | { |
92 | return dev->msi_cap + PCI_MSI_FLAGS; |
93 | } |
94 | |
95 | static inline uint8_t msi_address_lo_off(const PCIDevice* dev) |
96 | { |
97 | return dev->msi_cap + PCI_MSI_ADDRESS_LO; |
98 | } |
99 | |
100 | static inline uint8_t msi_address_hi_off(const PCIDevice* dev) |
101 | { |
102 | return dev->msi_cap + PCI_MSI_ADDRESS_HI; |
103 | } |
104 | |
105 | static inline uint8_t msi_data_off(const PCIDevice* dev, bool msi64bit) |
106 | { |
107 | return dev->msi_cap + (msi64bit ? PCI_MSI_DATA_64 : PCI_MSI_DATA_32); |
108 | } |
109 | |
110 | static inline uint8_t msi_mask_off(const PCIDevice* dev, bool msi64bit) |
111 | { |
112 | return dev->msi_cap + (msi64bit ? PCI_MSI_MASK_64 : PCI_MSI_MASK_32); |
113 | } |
114 | |
115 | static inline uint8_t msi_pending_off(const PCIDevice* dev, bool msi64bit) |
116 | { |
117 | return dev->msi_cap + (msi64bit ? PCI_MSI_PENDING_64 : PCI_MSI_PENDING_32); |
118 | } |
119 | |
120 | /* |
121 | * Special API for POWER to configure the vectors through |
122 | * a side channel. Should never be used by devices. |
123 | */ |
124 | void msi_set_message(PCIDevice *dev, MSIMessage msg) |
125 | { |
126 | uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev)); |
127 | bool msi64bit = flags & PCI_MSI_FLAGS_64BIT; |
128 | |
129 | if (msi64bit) { |
130 | pci_set_quad(dev->config + msi_address_lo_off(dev), msg.address); |
131 | } else { |
132 | pci_set_long(dev->config + msi_address_lo_off(dev), msg.address); |
133 | } |
134 | pci_set_word(dev->config + msi_data_off(dev, msi64bit), msg.data); |
135 | } |
136 | |
137 | MSIMessage msi_get_message(PCIDevice *dev, unsigned int vector) |
138 | { |
139 | uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev)); |
140 | bool msi64bit = flags & PCI_MSI_FLAGS_64BIT; |
141 | unsigned int nr_vectors = msi_nr_vectors(flags); |
142 | MSIMessage msg; |
143 | |
144 | assert(vector < nr_vectors); |
145 | |
146 | if (msi64bit) { |
147 | msg.address = pci_get_quad(dev->config + msi_address_lo_off(dev)); |
148 | } else { |
149 | msg.address = pci_get_long(dev->config + msi_address_lo_off(dev)); |
150 | } |
151 | |
152 | /* upper bit 31:16 is zero */ |
153 | msg.data = pci_get_word(dev->config + msi_data_off(dev, msi64bit)); |
154 | if (nr_vectors > 1) { |
155 | msg.data &= ~(nr_vectors - 1); |
156 | msg.data |= vector; |
157 | } |
158 | |
159 | return msg; |
160 | } |
161 | |
162 | bool msi_enabled(const PCIDevice *dev) |
163 | { |
164 | return msi_present(dev) && |
165 | (pci_get_word(dev->config + msi_flags_off(dev)) & |
166 | PCI_MSI_FLAGS_ENABLE); |
167 | } |
168 | |
169 | /* |
170 | * Make PCI device @dev MSI-capable. |
171 | * Non-zero @offset puts capability MSI at that offset in PCI config |
172 | * space. |
173 | * @nr_vectors is the number of MSI vectors (1, 2, 4, 8, 16 or 32). |
174 | * If @msi64bit, make the device capable of sending a 64-bit message |
175 | * address. |
176 | * If @msi_per_vector_mask, make the device support per-vector masking. |
177 | * @errp is for returning errors. |
178 | * Return 0 on success; set @errp and return -errno on error. |
179 | * |
180 | * -ENOTSUP means lacking msi support for a msi-capable platform. |
181 | * -EINVAL means capability overlap, happens when @offset is non-zero, |
182 | * also means a programming error, except device assignment, which can check |
183 | * if a real HW is broken. |
184 | */ |
185 | int msi_init(struct PCIDevice *dev, uint8_t offset, |
186 | unsigned int nr_vectors, bool msi64bit, |
187 | bool msi_per_vector_mask, Error **errp) |
188 | { |
189 | unsigned int vectors_order; |
190 | uint16_t flags; |
191 | uint8_t cap_size; |
192 | int config_offset; |
193 | |
194 | if (!msi_nonbroken) { |
195 | error_setg(errp, "MSI is not supported by interrupt controller" ); |
196 | return -ENOTSUP; |
197 | } |
198 | |
199 | MSI_DEV_PRINTF(dev, |
200 | "init offset: 0x%" PRIx8" vector: %" PRId8 |
201 | " 64bit %d mask %d\n" , |
202 | offset, nr_vectors, msi64bit, msi_per_vector_mask); |
203 | |
204 | assert(!(nr_vectors & (nr_vectors - 1))); /* power of 2 */ |
205 | assert(nr_vectors > 0); |
206 | assert(nr_vectors <= PCI_MSI_VECTORS_MAX); |
207 | /* the nr of MSI vectors is up to 32 */ |
208 | vectors_order = ctz32(nr_vectors); |
209 | |
210 | flags = vectors_order << ctz32(PCI_MSI_FLAGS_QMASK); |
211 | if (msi64bit) { |
212 | flags |= PCI_MSI_FLAGS_64BIT; |
213 | } |
214 | if (msi_per_vector_mask) { |
215 | flags |= PCI_MSI_FLAGS_MASKBIT; |
216 | } |
217 | |
218 | cap_size = msi_cap_sizeof(flags); |
219 | config_offset = pci_add_capability(dev, PCI_CAP_ID_MSI, offset, |
220 | cap_size, errp); |
221 | if (config_offset < 0) { |
222 | return config_offset; |
223 | } |
224 | |
225 | dev->msi_cap = config_offset; |
226 | dev->cap_present |= QEMU_PCI_CAP_MSI; |
227 | |
228 | pci_set_word(dev->config + msi_flags_off(dev), flags); |
229 | pci_set_word(dev->wmask + msi_flags_off(dev), |
230 | PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE); |
231 | pci_set_long(dev->wmask + msi_address_lo_off(dev), |
232 | PCI_MSI_ADDRESS_LO_MASK); |
233 | if (msi64bit) { |
234 | pci_set_long(dev->wmask + msi_address_hi_off(dev), 0xffffffff); |
235 | } |
236 | pci_set_word(dev->wmask + msi_data_off(dev, msi64bit), 0xffff); |
237 | |
238 | if (msi_per_vector_mask) { |
239 | /* Make mask bits 0 to nr_vectors - 1 writable. */ |
240 | pci_set_long(dev->wmask + msi_mask_off(dev, msi64bit), |
241 | 0xffffffff >> (PCI_MSI_VECTORS_MAX - nr_vectors)); |
242 | } |
243 | |
244 | return 0; |
245 | } |
246 | |
247 | void msi_uninit(struct PCIDevice *dev) |
248 | { |
249 | uint16_t flags; |
250 | uint8_t cap_size; |
251 | |
252 | if (!msi_present(dev)) { |
253 | return; |
254 | } |
255 | flags = pci_get_word(dev->config + msi_flags_off(dev)); |
256 | cap_size = msi_cap_sizeof(flags); |
257 | pci_del_capability(dev, PCI_CAP_ID_MSI, cap_size); |
258 | dev->cap_present &= ~QEMU_PCI_CAP_MSI; |
259 | |
260 | MSI_DEV_PRINTF(dev, "uninit\n" ); |
261 | } |
262 | |
263 | void msi_reset(PCIDevice *dev) |
264 | { |
265 | uint16_t flags; |
266 | bool msi64bit; |
267 | |
268 | if (!msi_present(dev)) { |
269 | return; |
270 | } |
271 | |
272 | flags = pci_get_word(dev->config + msi_flags_off(dev)); |
273 | flags &= ~(PCI_MSI_FLAGS_QSIZE | PCI_MSI_FLAGS_ENABLE); |
274 | msi64bit = flags & PCI_MSI_FLAGS_64BIT; |
275 | |
276 | pci_set_word(dev->config + msi_flags_off(dev), flags); |
277 | pci_set_long(dev->config + msi_address_lo_off(dev), 0); |
278 | if (msi64bit) { |
279 | pci_set_long(dev->config + msi_address_hi_off(dev), 0); |
280 | } |
281 | pci_set_word(dev->config + msi_data_off(dev, msi64bit), 0); |
282 | if (flags & PCI_MSI_FLAGS_MASKBIT) { |
283 | pci_set_long(dev->config + msi_mask_off(dev, msi64bit), 0); |
284 | pci_set_long(dev->config + msi_pending_off(dev, msi64bit), 0); |
285 | } |
286 | MSI_DEV_PRINTF(dev, "reset\n" ); |
287 | } |
288 | |
289 | bool msi_is_masked(const PCIDevice *dev, unsigned int vector) |
290 | { |
291 | uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev)); |
292 | uint32_t mask, data; |
293 | bool msi64bit = flags & PCI_MSI_FLAGS_64BIT; |
294 | assert(vector < PCI_MSI_VECTORS_MAX); |
295 | |
296 | if (!(flags & PCI_MSI_FLAGS_MASKBIT)) { |
297 | return false; |
298 | } |
299 | |
300 | data = pci_get_word(dev->config + msi_data_off(dev, msi64bit)); |
301 | if (xen_is_pirq_msi(data)) { |
302 | return false; |
303 | } |
304 | |
305 | mask = pci_get_long(dev->config + |
306 | msi_mask_off(dev, flags & PCI_MSI_FLAGS_64BIT)); |
307 | return mask & (1U << vector); |
308 | } |
309 | |
310 | void msi_notify(PCIDevice *dev, unsigned int vector) |
311 | { |
312 | uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev)); |
313 | bool msi64bit = flags & PCI_MSI_FLAGS_64BIT; |
314 | unsigned int nr_vectors = msi_nr_vectors(flags); |
315 | MSIMessage msg; |
316 | |
317 | assert(vector < nr_vectors); |
318 | if (msi_is_masked(dev, vector)) { |
319 | assert(flags & PCI_MSI_FLAGS_MASKBIT); |
320 | pci_long_test_and_set_mask( |
321 | dev->config + msi_pending_off(dev, msi64bit), 1U << vector); |
322 | MSI_DEV_PRINTF(dev, "pending vector 0x%x\n" , vector); |
323 | return; |
324 | } |
325 | |
326 | msg = msi_get_message(dev, vector); |
327 | |
328 | MSI_DEV_PRINTF(dev, |
329 | "notify vector 0x%x" |
330 | " address: 0x%" PRIx64" data: 0x%" PRIx32"\n" , |
331 | vector, msg.address, msg.data); |
332 | msi_send_message(dev, msg); |
333 | } |
334 | |
335 | void msi_send_message(PCIDevice *dev, MSIMessage msg) |
336 | { |
337 | MemTxAttrs attrs = {}; |
338 | |
339 | attrs.requester_id = pci_requester_id(dev); |
340 | address_space_stl_le(&dev->bus_master_as, msg.address, msg.data, |
341 | attrs, NULL); |
342 | } |
343 | |
344 | /* Normally called by pci_default_write_config(). */ |
345 | void msi_write_config(PCIDevice *dev, uint32_t addr, uint32_t val, int len) |
346 | { |
347 | uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev)); |
348 | bool msi64bit = flags & PCI_MSI_FLAGS_64BIT; |
349 | bool msi_per_vector_mask = flags & PCI_MSI_FLAGS_MASKBIT; |
350 | unsigned int nr_vectors; |
351 | uint8_t log_num_vecs; |
352 | uint8_t log_max_vecs; |
353 | unsigned int vector; |
354 | uint32_t pending; |
355 | |
356 | if (!msi_present(dev) || |
357 | !ranges_overlap(addr, len, dev->msi_cap, msi_cap_sizeof(flags))) { |
358 | return; |
359 | } |
360 | |
361 | #ifdef MSI_DEBUG |
362 | MSI_DEV_PRINTF(dev, "addr 0x%" PRIx32" val 0x%" PRIx32" len %d\n" , |
363 | addr, val, len); |
364 | MSI_DEV_PRINTF(dev, "ctrl: 0x%" PRIx16" address: 0x%" PRIx32, |
365 | flags, |
366 | pci_get_long(dev->config + msi_address_lo_off(dev))); |
367 | if (msi64bit) { |
368 | fprintf(stderr, " address-hi: 0x%" PRIx32, |
369 | pci_get_long(dev->config + msi_address_hi_off(dev))); |
370 | } |
371 | fprintf(stderr, " data: 0x%" PRIx16, |
372 | pci_get_word(dev->config + msi_data_off(dev, msi64bit))); |
373 | if (flags & PCI_MSI_FLAGS_MASKBIT) { |
374 | fprintf(stderr, " mask 0x%" PRIx32" pending 0x%" PRIx32, |
375 | pci_get_long(dev->config + msi_mask_off(dev, msi64bit)), |
376 | pci_get_long(dev->config + msi_pending_off(dev, msi64bit))); |
377 | } |
378 | fprintf(stderr, "\n" ); |
379 | #endif |
380 | |
381 | if (!(flags & PCI_MSI_FLAGS_ENABLE)) { |
382 | return; |
383 | } |
384 | |
385 | /* |
386 | * Now MSI is enabled, clear INTx# interrupts. |
387 | * the driver is prohibited from writing enable bit to mask |
388 | * a service request. But the guest OS could do this. |
389 | * So we just discard the interrupts as moderate fallback. |
390 | * |
391 | * 6.8.3.3. Enabling Operation |
392 | * While enabled for MSI or MSI-X operation, a function is prohibited |
393 | * from using its INTx# pin (if implemented) to request |
394 | * service (MSI, MSI-X, and INTx# are mutually exclusive). |
395 | */ |
396 | pci_device_deassert_intx(dev); |
397 | |
398 | /* |
399 | * nr_vectors might be set bigger than capable. So clamp it. |
400 | * This is not legal by spec, so we can do anything we like, |
401 | * just don't crash the host |
402 | */ |
403 | log_num_vecs = |
404 | (flags & PCI_MSI_FLAGS_QSIZE) >> ctz32(PCI_MSI_FLAGS_QSIZE); |
405 | log_max_vecs = |
406 | (flags & PCI_MSI_FLAGS_QMASK) >> ctz32(PCI_MSI_FLAGS_QMASK); |
407 | if (log_num_vecs > log_max_vecs) { |
408 | flags &= ~PCI_MSI_FLAGS_QSIZE; |
409 | flags |= log_max_vecs << ctz32(PCI_MSI_FLAGS_QSIZE); |
410 | pci_set_word(dev->config + msi_flags_off(dev), flags); |
411 | } |
412 | |
413 | if (!msi_per_vector_mask) { |
414 | /* if per vector masking isn't supported, |
415 | there is no pending interrupt. */ |
416 | return; |
417 | } |
418 | |
419 | nr_vectors = msi_nr_vectors(flags); |
420 | |
421 | /* This will discard pending interrupts, if any. */ |
422 | pending = pci_get_long(dev->config + msi_pending_off(dev, msi64bit)); |
423 | pending &= 0xffffffff >> (PCI_MSI_VECTORS_MAX - nr_vectors); |
424 | pci_set_long(dev->config + msi_pending_off(dev, msi64bit), pending); |
425 | |
426 | /* deliver pending interrupts which are unmasked */ |
427 | for (vector = 0; vector < nr_vectors; ++vector) { |
428 | if (msi_is_masked(dev, vector) || !(pending & (1U << vector))) { |
429 | continue; |
430 | } |
431 | |
432 | pci_long_test_and_clear_mask( |
433 | dev->config + msi_pending_off(dev, msi64bit), 1U << vector); |
434 | msi_notify(dev, vector); |
435 | } |
436 | } |
437 | |
438 | unsigned int msi_nr_vectors_allocated(const PCIDevice *dev) |
439 | { |
440 | uint16_t flags = pci_get_word(dev->config + msi_flags_off(dev)); |
441 | return msi_nr_vectors(flags); |
442 | } |
443 | |