1 | /* |
2 | * ARM v8.3-PAuth Operations |
3 | * |
4 | * Copyright (c) 2019 Linaro, Ltd. |
5 | * |
6 | * This library is free software; you can redistribute it and/or |
7 | * modify it under the terms of the GNU Lesser General Public |
8 | * License as published by the Free Software Foundation; either |
9 | * version 2 of the License, or (at your option) any later version. |
10 | * |
11 | * This library is distributed in the hope that it will be useful, |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
14 | * Lesser General Public License for more details. |
15 | * |
16 | * You should have received a copy of the GNU Lesser General Public |
17 | * License along with this library; if not, see <http://www.gnu.org/licenses/>. |
18 | */ |
19 | |
20 | #include "qemu/osdep.h" |
21 | #include "cpu.h" |
22 | #include "internals.h" |
23 | #include "exec/exec-all.h" |
24 | #include "exec/cpu_ldst.h" |
25 | #include "exec/helper-proto.h" |
26 | #include "tcg/tcg-gvec-desc.h" |
27 | |
28 | |
29 | static uint64_t pac_cell_shuffle(uint64_t i) |
30 | { |
31 | uint64_t o = 0; |
32 | |
33 | o |= extract64(i, 52, 4); |
34 | o |= extract64(i, 24, 4) << 4; |
35 | o |= extract64(i, 44, 4) << 8; |
36 | o |= extract64(i, 0, 4) << 12; |
37 | |
38 | o |= extract64(i, 28, 4) << 16; |
39 | o |= extract64(i, 48, 4) << 20; |
40 | o |= extract64(i, 4, 4) << 24; |
41 | o |= extract64(i, 40, 4) << 28; |
42 | |
43 | o |= extract64(i, 32, 4) << 32; |
44 | o |= extract64(i, 12, 4) << 36; |
45 | o |= extract64(i, 56, 4) << 40; |
46 | o |= extract64(i, 20, 4) << 44; |
47 | |
48 | o |= extract64(i, 8, 4) << 48; |
49 | o |= extract64(i, 36, 4) << 52; |
50 | o |= extract64(i, 16, 4) << 56; |
51 | o |= extract64(i, 60, 4) << 60; |
52 | |
53 | return o; |
54 | } |
55 | |
56 | static uint64_t pac_cell_inv_shuffle(uint64_t i) |
57 | { |
58 | uint64_t o = 0; |
59 | |
60 | o |= extract64(i, 12, 4); |
61 | o |= extract64(i, 24, 4) << 4; |
62 | o |= extract64(i, 48, 4) << 8; |
63 | o |= extract64(i, 36, 4) << 12; |
64 | |
65 | o |= extract64(i, 56, 4) << 16; |
66 | o |= extract64(i, 44, 4) << 20; |
67 | o |= extract64(i, 4, 4) << 24; |
68 | o |= extract64(i, 16, 4) << 28; |
69 | |
70 | o |= i & MAKE_64BIT_MASK(32, 4); |
71 | o |= extract64(i, 52, 4) << 36; |
72 | o |= extract64(i, 28, 4) << 40; |
73 | o |= extract64(i, 8, 4) << 44; |
74 | |
75 | o |= extract64(i, 20, 4) << 48; |
76 | o |= extract64(i, 0, 4) << 52; |
77 | o |= extract64(i, 40, 4) << 56; |
78 | o |= i & MAKE_64BIT_MASK(60, 4); |
79 | |
80 | return o; |
81 | } |
82 | |
83 | static uint64_t pac_sub(uint64_t i) |
84 | { |
85 | static const uint8_t sub[16] = { |
86 | 0xb, 0x6, 0x8, 0xf, 0xc, 0x0, 0x9, 0xe, |
87 | 0x3, 0x7, 0x4, 0x5, 0xd, 0x2, 0x1, 0xa, |
88 | }; |
89 | uint64_t o = 0; |
90 | int b; |
91 | |
92 | for (b = 0; b < 64; b += 16) { |
93 | o |= (uint64_t)sub[(i >> b) & 0xf] << b; |
94 | } |
95 | return o; |
96 | } |
97 | |
98 | static uint64_t pac_inv_sub(uint64_t i) |
99 | { |
100 | static const uint8_t inv_sub[16] = { |
101 | 0x5, 0xe, 0xd, 0x8, 0xa, 0xb, 0x1, 0x9, |
102 | 0x2, 0x6, 0xf, 0x0, 0x4, 0xc, 0x7, 0x3, |
103 | }; |
104 | uint64_t o = 0; |
105 | int b; |
106 | |
107 | for (b = 0; b < 64; b += 16) { |
108 | o |= (uint64_t)inv_sub[(i >> b) & 0xf] << b; |
109 | } |
110 | return o; |
111 | } |
112 | |
113 | static int rot_cell(int cell, int n) |
114 | { |
115 | /* 4-bit rotate left by n. */ |
116 | cell |= cell << 4; |
117 | return extract32(cell, 4 - n, 4); |
118 | } |
119 | |
120 | static uint64_t pac_mult(uint64_t i) |
121 | { |
122 | uint64_t o = 0; |
123 | int b; |
124 | |
125 | for (b = 0; b < 4 * 4; b += 4) { |
126 | int i0, i4, i8, ic, t0, t1, t2, t3; |
127 | |
128 | i0 = extract64(i, b, 4); |
129 | i4 = extract64(i, b + 4 * 4, 4); |
130 | i8 = extract64(i, b + 8 * 4, 4); |
131 | ic = extract64(i, b + 12 * 4, 4); |
132 | |
133 | t0 = rot_cell(i8, 1) ^ rot_cell(i4, 2) ^ rot_cell(i0, 1); |
134 | t1 = rot_cell(ic, 1) ^ rot_cell(i4, 1) ^ rot_cell(i0, 2); |
135 | t2 = rot_cell(ic, 2) ^ rot_cell(i8, 1) ^ rot_cell(i0, 1); |
136 | t3 = rot_cell(ic, 1) ^ rot_cell(i8, 2) ^ rot_cell(i4, 1); |
137 | |
138 | o |= (uint64_t)t3 << b; |
139 | o |= (uint64_t)t2 << (b + 4 * 4); |
140 | o |= (uint64_t)t1 << (b + 8 * 4); |
141 | o |= (uint64_t)t0 << (b + 12 * 4); |
142 | } |
143 | return o; |
144 | } |
145 | |
146 | static uint64_t tweak_cell_rot(uint64_t cell) |
147 | { |
148 | return (cell >> 1) | (((cell ^ (cell >> 1)) & 1) << 3); |
149 | } |
150 | |
151 | static uint64_t tweak_shuffle(uint64_t i) |
152 | { |
153 | uint64_t o = 0; |
154 | |
155 | o |= extract64(i, 16, 4) << 0; |
156 | o |= extract64(i, 20, 4) << 4; |
157 | o |= tweak_cell_rot(extract64(i, 24, 4)) << 8; |
158 | o |= extract64(i, 28, 4) << 12; |
159 | |
160 | o |= tweak_cell_rot(extract64(i, 44, 4)) << 16; |
161 | o |= extract64(i, 8, 4) << 20; |
162 | o |= extract64(i, 12, 4) << 24; |
163 | o |= tweak_cell_rot(extract64(i, 32, 4)) << 28; |
164 | |
165 | o |= extract64(i, 48, 4) << 32; |
166 | o |= extract64(i, 52, 4) << 36; |
167 | o |= extract64(i, 56, 4) << 40; |
168 | o |= tweak_cell_rot(extract64(i, 60, 4)) << 44; |
169 | |
170 | o |= tweak_cell_rot(extract64(i, 0, 4)) << 48; |
171 | o |= extract64(i, 4, 4) << 52; |
172 | o |= tweak_cell_rot(extract64(i, 40, 4)) << 56; |
173 | o |= tweak_cell_rot(extract64(i, 36, 4)) << 60; |
174 | |
175 | return o; |
176 | } |
177 | |
178 | static uint64_t tweak_cell_inv_rot(uint64_t cell) |
179 | { |
180 | return ((cell << 1) & 0xf) | ((cell & 1) ^ (cell >> 3)); |
181 | } |
182 | |
183 | static uint64_t tweak_inv_shuffle(uint64_t i) |
184 | { |
185 | uint64_t o = 0; |
186 | |
187 | o |= tweak_cell_inv_rot(extract64(i, 48, 4)); |
188 | o |= extract64(i, 52, 4) << 4; |
189 | o |= extract64(i, 20, 4) << 8; |
190 | o |= extract64(i, 24, 4) << 12; |
191 | |
192 | o |= extract64(i, 0, 4) << 16; |
193 | o |= extract64(i, 4, 4) << 20; |
194 | o |= tweak_cell_inv_rot(extract64(i, 8, 4)) << 24; |
195 | o |= extract64(i, 12, 4) << 28; |
196 | |
197 | o |= tweak_cell_inv_rot(extract64(i, 28, 4)) << 32; |
198 | o |= tweak_cell_inv_rot(extract64(i, 60, 4)) << 36; |
199 | o |= tweak_cell_inv_rot(extract64(i, 56, 4)) << 40; |
200 | o |= tweak_cell_inv_rot(extract64(i, 16, 4)) << 44; |
201 | |
202 | o |= extract64(i, 32, 4) << 48; |
203 | o |= extract64(i, 36, 4) << 52; |
204 | o |= extract64(i, 40, 4) << 56; |
205 | o |= tweak_cell_inv_rot(extract64(i, 44, 4)) << 60; |
206 | |
207 | return o; |
208 | } |
209 | |
210 | static uint64_t pauth_computepac(uint64_t data, uint64_t modifier, |
211 | ARMPACKey key) |
212 | { |
213 | static const uint64_t RC[5] = { |
214 | 0x0000000000000000ull, |
215 | 0x13198A2E03707344ull, |
216 | 0xA4093822299F31D0ull, |
217 | 0x082EFA98EC4E6C89ull, |
218 | 0x452821E638D01377ull, |
219 | }; |
220 | const uint64_t alpha = 0xC0AC29B7C97C50DDull; |
221 | /* |
222 | * Note that in the ARM pseudocode, key0 contains bits <127:64> |
223 | * and key1 contains bits <63:0> of the 128-bit key. |
224 | */ |
225 | uint64_t key0 = key.hi, key1 = key.lo; |
226 | uint64_t workingval, runningmod, roundkey, modk0; |
227 | int i; |
228 | |
229 | modk0 = (key0 << 63) | ((key0 >> 1) ^ (key0 >> 63)); |
230 | runningmod = modifier; |
231 | workingval = data ^ key0; |
232 | |
233 | for (i = 0; i <= 4; ++i) { |
234 | roundkey = key1 ^ runningmod; |
235 | workingval ^= roundkey; |
236 | workingval ^= RC[i]; |
237 | if (i > 0) { |
238 | workingval = pac_cell_shuffle(workingval); |
239 | workingval = pac_mult(workingval); |
240 | } |
241 | workingval = pac_sub(workingval); |
242 | runningmod = tweak_shuffle(runningmod); |
243 | } |
244 | roundkey = modk0 ^ runningmod; |
245 | workingval ^= roundkey; |
246 | workingval = pac_cell_shuffle(workingval); |
247 | workingval = pac_mult(workingval); |
248 | workingval = pac_sub(workingval); |
249 | workingval = pac_cell_shuffle(workingval); |
250 | workingval = pac_mult(workingval); |
251 | workingval ^= key1; |
252 | workingval = pac_cell_inv_shuffle(workingval); |
253 | workingval = pac_inv_sub(workingval); |
254 | workingval = pac_mult(workingval); |
255 | workingval = pac_cell_inv_shuffle(workingval); |
256 | workingval ^= key0; |
257 | workingval ^= runningmod; |
258 | for (i = 0; i <= 4; ++i) { |
259 | workingval = pac_inv_sub(workingval); |
260 | if (i < 4) { |
261 | workingval = pac_mult(workingval); |
262 | workingval = pac_cell_inv_shuffle(workingval); |
263 | } |
264 | runningmod = tweak_inv_shuffle(runningmod); |
265 | roundkey = key1 ^ runningmod; |
266 | workingval ^= RC[4 - i]; |
267 | workingval ^= roundkey; |
268 | workingval ^= alpha; |
269 | } |
270 | workingval ^= modk0; |
271 | |
272 | return workingval; |
273 | } |
274 | |
275 | static uint64_t pauth_addpac(CPUARMState *env, uint64_t ptr, uint64_t modifier, |
276 | ARMPACKey *key, bool data) |
277 | { |
278 | ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); |
279 | ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); |
280 | uint64_t pac, ext_ptr, ext, test; |
281 | int bot_bit, top_bit; |
282 | |
283 | /* If tagged pointers are in use, use ptr<55>, otherwise ptr<63>. */ |
284 | if (param.tbi) { |
285 | ext = sextract64(ptr, 55, 1); |
286 | } else { |
287 | ext = sextract64(ptr, 63, 1); |
288 | } |
289 | |
290 | /* Build a pointer with known good extension bits. */ |
291 | top_bit = 64 - 8 * param.tbi; |
292 | bot_bit = 64 - param.tsz; |
293 | ext_ptr = deposit64(ptr, bot_bit, top_bit - bot_bit, ext); |
294 | |
295 | pac = pauth_computepac(ext_ptr, modifier, *key); |
296 | |
297 | /* |
298 | * Check if the ptr has good extension bits and corrupt the |
299 | * pointer authentication code if not. |
300 | */ |
301 | test = sextract64(ptr, bot_bit, top_bit - bot_bit); |
302 | if (test != 0 && test != -1) { |
303 | pac ^= MAKE_64BIT_MASK(top_bit - 1, 1); |
304 | } |
305 | |
306 | /* |
307 | * Preserve the determination between upper and lower at bit 55, |
308 | * and insert pointer authentication code. |
309 | */ |
310 | if (param.tbi) { |
311 | ptr &= ~MAKE_64BIT_MASK(bot_bit, 55 - bot_bit + 1); |
312 | pac &= MAKE_64BIT_MASK(bot_bit, 54 - bot_bit + 1); |
313 | } else { |
314 | ptr &= MAKE_64BIT_MASK(0, bot_bit); |
315 | pac &= ~(MAKE_64BIT_MASK(55, 1) | MAKE_64BIT_MASK(0, bot_bit)); |
316 | } |
317 | ext &= MAKE_64BIT_MASK(55, 1); |
318 | return pac | ext | ptr; |
319 | } |
320 | |
321 | static uint64_t pauth_original_ptr(uint64_t ptr, ARMVAParameters param) |
322 | { |
323 | uint64_t extfield = -param.select; |
324 | int bot_pac_bit = 64 - param.tsz; |
325 | int top_pac_bit = 64 - 8 * param.tbi; |
326 | |
327 | return deposit64(ptr, bot_pac_bit, top_pac_bit - bot_pac_bit, extfield); |
328 | } |
329 | |
330 | static uint64_t pauth_auth(CPUARMState *env, uint64_t ptr, uint64_t modifier, |
331 | ARMPACKey *key, bool data, int keynumber) |
332 | { |
333 | ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); |
334 | ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); |
335 | int bot_bit, top_bit; |
336 | uint64_t pac, orig_ptr, test; |
337 | |
338 | orig_ptr = pauth_original_ptr(ptr, param); |
339 | pac = pauth_computepac(orig_ptr, modifier, *key); |
340 | bot_bit = 64 - param.tsz; |
341 | top_bit = 64 - 8 * param.tbi; |
342 | |
343 | test = (pac ^ ptr) & ~MAKE_64BIT_MASK(55, 1); |
344 | if (unlikely(extract64(test, bot_bit, top_bit - bot_bit))) { |
345 | int error_code = (keynumber << 1) | (keynumber ^ 1); |
346 | if (param.tbi) { |
347 | return deposit64(orig_ptr, 53, 2, error_code); |
348 | } else { |
349 | return deposit64(orig_ptr, 61, 2, error_code); |
350 | } |
351 | } |
352 | return orig_ptr; |
353 | } |
354 | |
355 | static uint64_t pauth_strip(CPUARMState *env, uint64_t ptr, bool data) |
356 | { |
357 | ARMMMUIdx mmu_idx = arm_stage1_mmu_idx(env); |
358 | ARMVAParameters param = aa64_va_parameters(env, ptr, mmu_idx, data); |
359 | |
360 | return pauth_original_ptr(ptr, param); |
361 | } |
362 | |
363 | static void QEMU_NORETURN pauth_trap(CPUARMState *env, int target_el, |
364 | uintptr_t ra) |
365 | { |
366 | raise_exception_ra(env, EXCP_UDEF, syn_pactrap(), target_el, ra); |
367 | } |
368 | |
369 | static void pauth_check_trap(CPUARMState *env, int el, uintptr_t ra) |
370 | { |
371 | if (el < 2 && arm_feature(env, ARM_FEATURE_EL2)) { |
372 | uint64_t hcr = arm_hcr_el2_eff(env); |
373 | bool trap = !(hcr & HCR_API); |
374 | /* FIXME: ARMv8.1-VHE: trap only applies to EL1&0 regime. */ |
375 | /* FIXME: ARMv8.3-NV: HCR_NV trap takes precedence for ERETA[AB]. */ |
376 | if (trap) { |
377 | pauth_trap(env, 2, ra); |
378 | } |
379 | } |
380 | if (el < 3 && arm_feature(env, ARM_FEATURE_EL3)) { |
381 | if (!(env->cp15.scr_el3 & SCR_API)) { |
382 | pauth_trap(env, 3, ra); |
383 | } |
384 | } |
385 | } |
386 | |
387 | static bool pauth_key_enabled(CPUARMState *env, int el, uint32_t bit) |
388 | { |
389 | uint32_t sctlr; |
390 | if (el == 0) { |
391 | /* FIXME: ARMv8.1-VHE S2 translation regime. */ |
392 | sctlr = env->cp15.sctlr_el[1]; |
393 | } else { |
394 | sctlr = env->cp15.sctlr_el[el]; |
395 | } |
396 | return (sctlr & bit) != 0; |
397 | } |
398 | |
399 | uint64_t HELPER(pacia)(CPUARMState *env, uint64_t x, uint64_t y) |
400 | { |
401 | int el = arm_current_el(env); |
402 | if (!pauth_key_enabled(env, el, SCTLR_EnIA)) { |
403 | return x; |
404 | } |
405 | pauth_check_trap(env, el, GETPC()); |
406 | return pauth_addpac(env, x, y, &env->keys.apia, false); |
407 | } |
408 | |
409 | uint64_t HELPER(pacib)(CPUARMState *env, uint64_t x, uint64_t y) |
410 | { |
411 | int el = arm_current_el(env); |
412 | if (!pauth_key_enabled(env, el, SCTLR_EnIB)) { |
413 | return x; |
414 | } |
415 | pauth_check_trap(env, el, GETPC()); |
416 | return pauth_addpac(env, x, y, &env->keys.apib, false); |
417 | } |
418 | |
419 | uint64_t HELPER(pacda)(CPUARMState *env, uint64_t x, uint64_t y) |
420 | { |
421 | int el = arm_current_el(env); |
422 | if (!pauth_key_enabled(env, el, SCTLR_EnDA)) { |
423 | return x; |
424 | } |
425 | pauth_check_trap(env, el, GETPC()); |
426 | return pauth_addpac(env, x, y, &env->keys.apda, true); |
427 | } |
428 | |
429 | uint64_t HELPER(pacdb)(CPUARMState *env, uint64_t x, uint64_t y) |
430 | { |
431 | int el = arm_current_el(env); |
432 | if (!pauth_key_enabled(env, el, SCTLR_EnDB)) { |
433 | return x; |
434 | } |
435 | pauth_check_trap(env, el, GETPC()); |
436 | return pauth_addpac(env, x, y, &env->keys.apdb, true); |
437 | } |
438 | |
439 | uint64_t HELPER(pacga)(CPUARMState *env, uint64_t x, uint64_t y) |
440 | { |
441 | uint64_t pac; |
442 | |
443 | pauth_check_trap(env, arm_current_el(env), GETPC()); |
444 | pac = pauth_computepac(x, y, env->keys.apga); |
445 | |
446 | return pac & 0xffffffff00000000ull; |
447 | } |
448 | |
449 | uint64_t HELPER(autia)(CPUARMState *env, uint64_t x, uint64_t y) |
450 | { |
451 | int el = arm_current_el(env); |
452 | if (!pauth_key_enabled(env, el, SCTLR_EnIA)) { |
453 | return x; |
454 | } |
455 | pauth_check_trap(env, el, GETPC()); |
456 | return pauth_auth(env, x, y, &env->keys.apia, false, 0); |
457 | } |
458 | |
459 | uint64_t HELPER(autib)(CPUARMState *env, uint64_t x, uint64_t y) |
460 | { |
461 | int el = arm_current_el(env); |
462 | if (!pauth_key_enabled(env, el, SCTLR_EnIB)) { |
463 | return x; |
464 | } |
465 | pauth_check_trap(env, el, GETPC()); |
466 | return pauth_auth(env, x, y, &env->keys.apib, false, 1); |
467 | } |
468 | |
469 | uint64_t HELPER(autda)(CPUARMState *env, uint64_t x, uint64_t y) |
470 | { |
471 | int el = arm_current_el(env); |
472 | if (!pauth_key_enabled(env, el, SCTLR_EnDA)) { |
473 | return x; |
474 | } |
475 | pauth_check_trap(env, el, GETPC()); |
476 | return pauth_auth(env, x, y, &env->keys.apda, true, 0); |
477 | } |
478 | |
479 | uint64_t HELPER(autdb)(CPUARMState *env, uint64_t x, uint64_t y) |
480 | { |
481 | int el = arm_current_el(env); |
482 | if (!pauth_key_enabled(env, el, SCTLR_EnDB)) { |
483 | return x; |
484 | } |
485 | pauth_check_trap(env, el, GETPC()); |
486 | return pauth_auth(env, x, y, &env->keys.apdb, true, 1); |
487 | } |
488 | |
489 | uint64_t HELPER(xpaci)(CPUARMState *env, uint64_t a) |
490 | { |
491 | return pauth_strip(env, a, false); |
492 | } |
493 | |
494 | uint64_t HELPER(xpacd)(CPUARMState *env, uint64_t a) |
495 | { |
496 | return pauth_strip(env, a, true); |
497 | } |
498 | |