1 | /* |
2 | * Copyright (C) 2014-2016 Broadcom Corporation |
3 | * Copyright (c) 2017 Red Hat, Inc. |
4 | * Written by Prem Mallappa, Eric Auger |
5 | * |
6 | * This program is free software; you can redistribute it and/or modify |
7 | * it under the terms of the GNU General Public License version 2 as |
8 | * published by the Free Software Foundation. |
9 | * |
10 | * This program is distributed in the hope that it will be useful, |
11 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 | * GNU General Public License for more details. |
14 | * |
15 | * Author: Prem Mallappa <pmallapp@broadcom.com> |
16 | * |
17 | */ |
18 | |
19 | #include "qemu/osdep.h" |
20 | #include "exec/address-spaces.h" |
21 | #include "trace.h" |
22 | #include "exec/target_page.h" |
23 | #include "hw/core/cpu.h" |
24 | #include "hw/qdev-properties.h" |
25 | #include "qapi/error.h" |
26 | #include "qemu/jhash.h" |
27 | #include "qemu/module.h" |
28 | |
29 | #include "qemu/error-report.h" |
30 | #include "hw/arm/smmu-common.h" |
31 | #include "smmu-internal.h" |
32 | |
33 | /* IOTLB Management */ |
34 | |
35 | inline void smmu_iotlb_inv_all(SMMUState *s) |
36 | { |
37 | trace_smmu_iotlb_inv_all(); |
38 | g_hash_table_remove_all(s->iotlb); |
39 | } |
40 | |
41 | static gboolean smmu_hash_remove_by_asid(gpointer key, gpointer value, |
42 | gpointer user_data) |
43 | { |
44 | uint16_t asid = *(uint16_t *)user_data; |
45 | SMMUIOTLBKey *iotlb_key = (SMMUIOTLBKey *)key; |
46 | |
47 | return iotlb_key->asid == asid; |
48 | } |
49 | |
50 | inline void smmu_iotlb_inv_iova(SMMUState *s, uint16_t asid, dma_addr_t iova) |
51 | { |
52 | SMMUIOTLBKey key = {.asid = asid, .iova = iova}; |
53 | |
54 | trace_smmu_iotlb_inv_iova(asid, iova); |
55 | g_hash_table_remove(s->iotlb, &key); |
56 | } |
57 | |
58 | inline void smmu_iotlb_inv_asid(SMMUState *s, uint16_t asid) |
59 | { |
60 | trace_smmu_iotlb_inv_asid(asid); |
61 | g_hash_table_foreach_remove(s->iotlb, smmu_hash_remove_by_asid, &asid); |
62 | } |
63 | |
64 | /* VMSAv8-64 Translation */ |
65 | |
66 | /** |
67 | * get_pte - Get the content of a page table entry located at |
68 | * @base_addr[@index] |
69 | */ |
70 | static int get_pte(dma_addr_t baseaddr, uint32_t index, uint64_t *pte, |
71 | SMMUPTWEventInfo *info) |
72 | { |
73 | int ret; |
74 | dma_addr_t addr = baseaddr + index * sizeof(*pte); |
75 | |
76 | /* TODO: guarantee 64-bit single-copy atomicity */ |
77 | ret = dma_memory_read(&address_space_memory, addr, |
78 | (uint8_t *)pte, sizeof(*pte)); |
79 | |
80 | if (ret != MEMTX_OK) { |
81 | info->type = SMMU_PTW_ERR_WALK_EABT; |
82 | info->addr = addr; |
83 | return -EINVAL; |
84 | } |
85 | trace_smmu_get_pte(baseaddr, index, addr, *pte); |
86 | return 0; |
87 | } |
88 | |
89 | /* VMSAv8-64 Translation Table Format Descriptor Decoding */ |
90 | |
91 | /** |
92 | * get_page_pte_address - returns the L3 descriptor output address, |
93 | * ie. the page frame |
94 | * ARM ARM spec: Figure D4-17 VMSAv8-64 level 3 descriptor format |
95 | */ |
96 | static inline hwaddr get_page_pte_address(uint64_t pte, int granule_sz) |
97 | { |
98 | return PTE_ADDRESS(pte, granule_sz); |
99 | } |
100 | |
101 | /** |
102 | * get_table_pte_address - return table descriptor output address, |
103 | * ie. address of next level table |
104 | * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats |
105 | */ |
106 | static inline hwaddr get_table_pte_address(uint64_t pte, int granule_sz) |
107 | { |
108 | return PTE_ADDRESS(pte, granule_sz); |
109 | } |
110 | |
111 | /** |
112 | * get_block_pte_address - return block descriptor output address and block size |
113 | * ARM ARM Figure D4-16 VMSAv8-64 level0, level1, and level 2 descriptor formats |
114 | */ |
115 | static inline hwaddr get_block_pte_address(uint64_t pte, int level, |
116 | int granule_sz, uint64_t *bsz) |
117 | { |
118 | int n = level_shift(level, granule_sz); |
119 | |
120 | *bsz = 1ULL << n; |
121 | return PTE_ADDRESS(pte, n); |
122 | } |
123 | |
124 | SMMUTransTableInfo *select_tt(SMMUTransCfg *cfg, dma_addr_t iova) |
125 | { |
126 | bool tbi = extract64(iova, 55, 1) ? TBI1(cfg->tbi) : TBI0(cfg->tbi); |
127 | uint8_t tbi_byte = tbi * 8; |
128 | |
129 | if (cfg->tt[0].tsz && |
130 | !extract64(iova, 64 - cfg->tt[0].tsz, cfg->tt[0].tsz - tbi_byte)) { |
131 | /* there is a ttbr0 region and we are in it (high bits all zero) */ |
132 | return &cfg->tt[0]; |
133 | } else if (cfg->tt[1].tsz && |
134 | !extract64(iova, 64 - cfg->tt[1].tsz, cfg->tt[1].tsz - tbi_byte)) { |
135 | /* there is a ttbr1 region and we are in it (high bits all one) */ |
136 | return &cfg->tt[1]; |
137 | } else if (!cfg->tt[0].tsz) { |
138 | /* ttbr0 region is "everything not in the ttbr1 region" */ |
139 | return &cfg->tt[0]; |
140 | } else if (!cfg->tt[1].tsz) { |
141 | /* ttbr1 region is "everything not in the ttbr0 region" */ |
142 | return &cfg->tt[1]; |
143 | } |
144 | /* in the gap between the two regions, this is a Translation fault */ |
145 | return NULL; |
146 | } |
147 | |
148 | /** |
149 | * smmu_ptw_64 - VMSAv8-64 Walk of the page tables for a given IOVA |
150 | * @cfg: translation config |
151 | * @iova: iova to translate |
152 | * @perm: access type |
153 | * @tlbe: IOMMUTLBEntry (out) |
154 | * @info: handle to an error info |
155 | * |
156 | * Return 0 on success, < 0 on error. In case of error, @info is filled |
157 | * and tlbe->perm is set to IOMMU_NONE. |
158 | * Upon success, @tlbe is filled with translated_addr and entry |
159 | * permission rights. |
160 | */ |
161 | static int smmu_ptw_64(SMMUTransCfg *cfg, |
162 | dma_addr_t iova, IOMMUAccessFlags perm, |
163 | IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) |
164 | { |
165 | dma_addr_t baseaddr, indexmask; |
166 | int stage = cfg->stage; |
167 | SMMUTransTableInfo *tt = select_tt(cfg, iova); |
168 | uint8_t level, granule_sz, inputsize, stride; |
169 | |
170 | if (!tt || tt->disabled) { |
171 | info->type = SMMU_PTW_ERR_TRANSLATION; |
172 | goto error; |
173 | } |
174 | |
175 | granule_sz = tt->granule_sz; |
176 | stride = granule_sz - 3; |
177 | inputsize = 64 - tt->tsz; |
178 | level = 4 - (inputsize - 4) / stride; |
179 | indexmask = (1ULL << (inputsize - (stride * (4 - level)))) - 1; |
180 | baseaddr = extract64(tt->ttb, 0, 48); |
181 | baseaddr &= ~indexmask; |
182 | |
183 | tlbe->iova = iova; |
184 | tlbe->addr_mask = (1 << granule_sz) - 1; |
185 | |
186 | while (level <= 3) { |
187 | uint64_t subpage_size = 1ULL << level_shift(level, granule_sz); |
188 | uint64_t mask = subpage_size - 1; |
189 | uint32_t offset = iova_level_offset(iova, inputsize, level, granule_sz); |
190 | uint64_t pte; |
191 | dma_addr_t pte_addr = baseaddr + offset * sizeof(pte); |
192 | uint8_t ap; |
193 | |
194 | if (get_pte(baseaddr, offset, &pte, info)) { |
195 | goto error; |
196 | } |
197 | trace_smmu_ptw_level(level, iova, subpage_size, |
198 | baseaddr, offset, pte); |
199 | |
200 | if (is_invalid_pte(pte) || is_reserved_pte(pte, level)) { |
201 | trace_smmu_ptw_invalid_pte(stage, level, baseaddr, |
202 | pte_addr, offset, pte); |
203 | info->type = SMMU_PTW_ERR_TRANSLATION; |
204 | goto error; |
205 | } |
206 | |
207 | if (is_page_pte(pte, level)) { |
208 | uint64_t gpa = get_page_pte_address(pte, granule_sz); |
209 | |
210 | ap = PTE_AP(pte); |
211 | if (is_permission_fault(ap, perm)) { |
212 | info->type = SMMU_PTW_ERR_PERMISSION; |
213 | goto error; |
214 | } |
215 | |
216 | tlbe->translated_addr = gpa + (iova & mask); |
217 | tlbe->perm = PTE_AP_TO_PERM(ap); |
218 | trace_smmu_ptw_page_pte(stage, level, iova, |
219 | baseaddr, pte_addr, pte, gpa); |
220 | return 0; |
221 | } |
222 | if (is_block_pte(pte, level)) { |
223 | uint64_t block_size; |
224 | hwaddr gpa = get_block_pte_address(pte, level, granule_sz, |
225 | &block_size); |
226 | |
227 | ap = PTE_AP(pte); |
228 | if (is_permission_fault(ap, perm)) { |
229 | info->type = SMMU_PTW_ERR_PERMISSION; |
230 | goto error; |
231 | } |
232 | |
233 | trace_smmu_ptw_block_pte(stage, level, baseaddr, |
234 | pte_addr, pte, iova, gpa, |
235 | block_size >> 20); |
236 | |
237 | tlbe->translated_addr = gpa + (iova & mask); |
238 | tlbe->perm = PTE_AP_TO_PERM(ap); |
239 | return 0; |
240 | } |
241 | |
242 | /* table pte */ |
243 | ap = PTE_APTABLE(pte); |
244 | |
245 | if (is_permission_fault(ap, perm)) { |
246 | info->type = SMMU_PTW_ERR_PERMISSION; |
247 | goto error; |
248 | } |
249 | baseaddr = get_table_pte_address(pte, granule_sz); |
250 | level++; |
251 | } |
252 | |
253 | info->type = SMMU_PTW_ERR_TRANSLATION; |
254 | |
255 | error: |
256 | tlbe->perm = IOMMU_NONE; |
257 | return -EINVAL; |
258 | } |
259 | |
260 | /** |
261 | * smmu_ptw - Walk the page tables for an IOVA, according to @cfg |
262 | * |
263 | * @cfg: translation configuration |
264 | * @iova: iova to translate |
265 | * @perm: tentative access type |
266 | * @tlbe: returned entry |
267 | * @info: ptw event handle |
268 | * |
269 | * return 0 on success |
270 | */ |
271 | inline int smmu_ptw(SMMUTransCfg *cfg, dma_addr_t iova, IOMMUAccessFlags perm, |
272 | IOMMUTLBEntry *tlbe, SMMUPTWEventInfo *info) |
273 | { |
274 | if (!cfg->aa64) { |
275 | /* |
276 | * This code path is not entered as we check this while decoding |
277 | * the configuration data in the derived SMMU model. |
278 | */ |
279 | g_assert_not_reached(); |
280 | } |
281 | |
282 | return smmu_ptw_64(cfg, iova, perm, tlbe, info); |
283 | } |
284 | |
285 | /** |
286 | * The bus number is used for lookup when SID based invalidation occurs. |
287 | * In that case we lazily populate the SMMUPciBus array from the bus hash |
288 | * table. At the time the SMMUPciBus is created (smmu_find_add_as), the bus |
289 | * numbers may not be always initialized yet. |
290 | */ |
291 | SMMUPciBus *smmu_find_smmu_pcibus(SMMUState *s, uint8_t bus_num) |
292 | { |
293 | SMMUPciBus *smmu_pci_bus = s->smmu_pcibus_by_bus_num[bus_num]; |
294 | |
295 | if (!smmu_pci_bus) { |
296 | GHashTableIter iter; |
297 | |
298 | g_hash_table_iter_init(&iter, s->smmu_pcibus_by_busptr); |
299 | while (g_hash_table_iter_next(&iter, NULL, (void **)&smmu_pci_bus)) { |
300 | if (pci_bus_num(smmu_pci_bus->bus) == bus_num) { |
301 | s->smmu_pcibus_by_bus_num[bus_num] = smmu_pci_bus; |
302 | return smmu_pci_bus; |
303 | } |
304 | } |
305 | } |
306 | return smmu_pci_bus; |
307 | } |
308 | |
309 | static AddressSpace *smmu_find_add_as(PCIBus *bus, void *opaque, int devfn) |
310 | { |
311 | SMMUState *s = opaque; |
312 | SMMUPciBus *sbus = g_hash_table_lookup(s->smmu_pcibus_by_busptr, bus); |
313 | SMMUDevice *sdev; |
314 | static unsigned int index; |
315 | |
316 | if (!sbus) { |
317 | sbus = g_malloc0(sizeof(SMMUPciBus) + |
318 | sizeof(SMMUDevice *) * SMMU_PCI_DEVFN_MAX); |
319 | sbus->bus = bus; |
320 | g_hash_table_insert(s->smmu_pcibus_by_busptr, bus, sbus); |
321 | } |
322 | |
323 | sdev = sbus->pbdev[devfn]; |
324 | if (!sdev) { |
325 | char *name = g_strdup_printf("%s-%d-%d" , s->mrtypename, devfn, index++); |
326 | |
327 | sdev = sbus->pbdev[devfn] = g_new0(SMMUDevice, 1); |
328 | |
329 | sdev->smmu = s; |
330 | sdev->bus = bus; |
331 | sdev->devfn = devfn; |
332 | |
333 | memory_region_init_iommu(&sdev->iommu, sizeof(sdev->iommu), |
334 | s->mrtypename, |
335 | OBJECT(s), name, 1ULL << SMMU_MAX_VA_BITS); |
336 | address_space_init(&sdev->as, |
337 | MEMORY_REGION(&sdev->iommu), name); |
338 | trace_smmu_add_mr(name); |
339 | g_free(name); |
340 | } |
341 | |
342 | return &sdev->as; |
343 | } |
344 | |
345 | IOMMUMemoryRegion *smmu_iommu_mr(SMMUState *s, uint32_t sid) |
346 | { |
347 | uint8_t bus_n, devfn; |
348 | SMMUPciBus *smmu_bus; |
349 | SMMUDevice *smmu; |
350 | |
351 | bus_n = PCI_BUS_NUM(sid); |
352 | smmu_bus = smmu_find_smmu_pcibus(s, bus_n); |
353 | if (smmu_bus) { |
354 | devfn = SMMU_PCI_DEVFN(sid); |
355 | smmu = smmu_bus->pbdev[devfn]; |
356 | if (smmu) { |
357 | return &smmu->iommu; |
358 | } |
359 | } |
360 | return NULL; |
361 | } |
362 | |
363 | static guint smmu_iotlb_key_hash(gconstpointer v) |
364 | { |
365 | SMMUIOTLBKey *key = (SMMUIOTLBKey *)v; |
366 | uint32_t a, b, c; |
367 | |
368 | /* Jenkins hash */ |
369 | a = b = c = JHASH_INITVAL + sizeof(*key); |
370 | a += key->asid; |
371 | b += extract64(key->iova, 0, 32); |
372 | c += extract64(key->iova, 32, 32); |
373 | |
374 | __jhash_mix(a, b, c); |
375 | __jhash_final(a, b, c); |
376 | |
377 | return c; |
378 | } |
379 | |
380 | static gboolean smmu_iotlb_key_equal(gconstpointer v1, gconstpointer v2) |
381 | { |
382 | const SMMUIOTLBKey *k1 = v1; |
383 | const SMMUIOTLBKey *k2 = v2; |
384 | |
385 | return (k1->asid == k2->asid) && (k1->iova == k2->iova); |
386 | } |
387 | |
388 | /* Unmap the whole notifier's range */ |
389 | static void smmu_unmap_notifier_range(IOMMUNotifier *n) |
390 | { |
391 | IOMMUTLBEntry entry; |
392 | |
393 | entry.target_as = &address_space_memory; |
394 | entry.iova = n->start; |
395 | entry.perm = IOMMU_NONE; |
396 | entry.addr_mask = n->end - n->start; |
397 | |
398 | memory_region_notify_one(n, &entry); |
399 | } |
400 | |
401 | /* Unmap all notifiers attached to @mr */ |
402 | inline void smmu_inv_notifiers_mr(IOMMUMemoryRegion *mr) |
403 | { |
404 | IOMMUNotifier *n; |
405 | |
406 | trace_smmu_inv_notifiers_mr(mr->parent_obj.name); |
407 | IOMMU_NOTIFIER_FOREACH(n, mr) { |
408 | smmu_unmap_notifier_range(n); |
409 | } |
410 | } |
411 | |
412 | /* Unmap all notifiers of all mr's */ |
413 | void smmu_inv_notifiers_all(SMMUState *s) |
414 | { |
415 | SMMUDevice *sdev; |
416 | |
417 | QLIST_FOREACH(sdev, &s->devices_with_notifiers, next) { |
418 | smmu_inv_notifiers_mr(&sdev->iommu); |
419 | } |
420 | } |
421 | |
422 | static void smmu_base_realize(DeviceState *dev, Error **errp) |
423 | { |
424 | SMMUState *s = ARM_SMMU(dev); |
425 | SMMUBaseClass *sbc = ARM_SMMU_GET_CLASS(dev); |
426 | Error *local_err = NULL; |
427 | |
428 | sbc->parent_realize(dev, &local_err); |
429 | if (local_err) { |
430 | error_propagate(errp, local_err); |
431 | return; |
432 | } |
433 | s->configs = g_hash_table_new_full(NULL, NULL, NULL, g_free); |
434 | s->iotlb = g_hash_table_new_full(smmu_iotlb_key_hash, smmu_iotlb_key_equal, |
435 | g_free, g_free); |
436 | s->smmu_pcibus_by_busptr = g_hash_table_new(NULL, NULL); |
437 | |
438 | if (s->primary_bus) { |
439 | pci_setup_iommu(s->primary_bus, smmu_find_add_as, s); |
440 | } else { |
441 | error_setg(errp, "SMMU is not attached to any PCI bus!" ); |
442 | } |
443 | } |
444 | |
445 | static void smmu_base_reset(DeviceState *dev) |
446 | { |
447 | SMMUState *s = ARM_SMMU(dev); |
448 | |
449 | g_hash_table_remove_all(s->configs); |
450 | g_hash_table_remove_all(s->iotlb); |
451 | } |
452 | |
453 | static Property smmu_dev_properties[] = { |
454 | DEFINE_PROP_UINT8("bus_num" , SMMUState, bus_num, 0), |
455 | DEFINE_PROP_LINK("primary-bus" , SMMUState, primary_bus, "PCI" , PCIBus *), |
456 | DEFINE_PROP_END_OF_LIST(), |
457 | }; |
458 | |
459 | static void smmu_base_class_init(ObjectClass *klass, void *data) |
460 | { |
461 | DeviceClass *dc = DEVICE_CLASS(klass); |
462 | SMMUBaseClass *sbc = ARM_SMMU_CLASS(klass); |
463 | |
464 | dc->props = smmu_dev_properties; |
465 | device_class_set_parent_realize(dc, smmu_base_realize, |
466 | &sbc->parent_realize); |
467 | dc->reset = smmu_base_reset; |
468 | } |
469 | |
470 | static const TypeInfo smmu_base_info = { |
471 | .name = TYPE_ARM_SMMU, |
472 | .parent = TYPE_SYS_BUS_DEVICE, |
473 | .instance_size = sizeof(SMMUState), |
474 | .class_data = NULL, |
475 | .class_size = sizeof(SMMUBaseClass), |
476 | .class_init = smmu_base_class_init, |
477 | .abstract = true, |
478 | }; |
479 | |
480 | static void smmu_base_register_types(void) |
481 | { |
482 | type_register_static(&smmu_base_info); |
483 | } |
484 | |
485 | type_init(smmu_base_register_types) |
486 | |
487 | |