1/*
2 * Copyright 2019 The OpenSSL Project Authors. All Rights Reserved.
3 *
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
8 */
9
10#ifndef OSSL_CRYPTO_CIPHERMODE_PLATFORM_H
11# define OSSL_CRYPTO_CIPHERMODE_PLATFORM_H
12
13# include "openssl/aes.h"
14
15# ifdef VPAES_ASM
16int vpaes_set_encrypt_key(const unsigned char *userKey, int bits,
17 AES_KEY *key);
18int vpaes_set_decrypt_key(const unsigned char *userKey, int bits,
19 AES_KEY *key);
20void vpaes_encrypt(const unsigned char *in, unsigned char *out,
21 const AES_KEY *key);
22void vpaes_decrypt(const unsigned char *in, unsigned char *out,
23 const AES_KEY *key);
24void vpaes_cbc_encrypt(const unsigned char *in,
25 unsigned char *out,
26 size_t length,
27 const AES_KEY *key, unsigned char *ivec, int enc);
28# endif /* VPAES_ASM */
29
30# ifdef BSAES_ASM
31void bsaes_cbc_encrypt(const unsigned char *in, unsigned char *out,
32 size_t length, const AES_KEY *key,
33 unsigned char ivec[16], int enc);
34void bsaes_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
35 size_t len, const AES_KEY *key,
36 const unsigned char ivec[16]);
37void bsaes_xts_encrypt(const unsigned char *inp, unsigned char *out,
38 size_t len, const AES_KEY *key1,
39 const AES_KEY *key2, const unsigned char iv[16]);
40void bsaes_xts_decrypt(const unsigned char *inp, unsigned char *out,
41 size_t len, const AES_KEY *key1,
42 const AES_KEY *key2, const unsigned char iv[16]);
43# endif /* BSAES_ASM */
44
45# ifdef AES_CTR_ASM
46void AES_ctr32_encrypt(const unsigned char *in, unsigned char *out,
47 size_t blocks, const AES_KEY *key,
48 const unsigned char ivec[AES_BLOCK_SIZE]);
49# endif /* AES_CTR_ASM */
50
51# ifdef AES_XTS_ASM
52void AES_xts_encrypt(const unsigned char *inp, unsigned char *out, size_t len,
53 const AES_KEY *key1, const AES_KEY *key2,
54 const unsigned char iv[16]);
55void AES_xts_decrypt(const unsigned char *inp, unsigned char *out, size_t len,
56 const AES_KEY *key1, const AES_KEY *key2,
57 const unsigned char iv[16]);
58# endif /* AES_XTS_ASM */
59
60# if defined(OPENSSL_CPUID_OBJ)
61# if (defined(__powerpc__) || defined(__ppc__) || defined(_ARCH_PPC))
62# include "ppc_arch.h"
63# ifdef VPAES_ASM
64# define VPAES_CAPABLE (OPENSSL_ppccap_P & PPC_ALTIVEC)
65# endif
66# define HWAES_CAPABLE (OPENSSL_ppccap_P & PPC_CRYPTO207)
67# define HWAES_set_encrypt_key aes_p8_set_encrypt_key
68# define HWAES_set_decrypt_key aes_p8_set_decrypt_key
69# define HWAES_encrypt aes_p8_encrypt
70# define HWAES_decrypt aes_p8_decrypt
71# define HWAES_cbc_encrypt aes_p8_cbc_encrypt
72# define HWAES_ctr32_encrypt_blocks aes_p8_ctr32_encrypt_blocks
73# define HWAES_xts_encrypt aes_p8_xts_encrypt
74# define HWAES_xts_decrypt aes_p8_xts_decrypt
75# endif /* PPC */
76
77# if (defined(__arm__) || defined(__arm) || defined(__aarch64__))
78# include "arm_arch.h"
79# if __ARM_MAX_ARCH__>=7
80# if defined(BSAES_ASM)
81# define BSAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
82# endif
83# if defined(VPAES_ASM)
84# define VPAES_CAPABLE (OPENSSL_armcap_P & ARMV7_NEON)
85# endif
86# define HWAES_CAPABLE (OPENSSL_armcap_P & ARMV8_AES)
87# define HWAES_set_encrypt_key aes_v8_set_encrypt_key
88# define HWAES_set_decrypt_key aes_v8_set_decrypt_key
89# define HWAES_encrypt aes_v8_encrypt
90# define HWAES_decrypt aes_v8_decrypt
91# define HWAES_cbc_encrypt aes_v8_cbc_encrypt
92# define HWAES_ecb_encrypt aes_v8_ecb_encrypt
93# define HWAES_ctr32_encrypt_blocks aes_v8_ctr32_encrypt_blocks
94# endif
95# endif
96# endif /* OPENSSL_CPUID_OBJ */
97
98# if defined(AES_ASM) && !defined(I386_ONLY) && ( \
99 ((defined(__i386) || defined(__i386__) || \
100 defined(_M_IX86)) && defined(OPENSSL_IA32_SSE2))|| \
101 defined(__x86_64) || defined(__x86_64__) || \
102 defined(_M_AMD64) || defined(_M_X64) )
103
104/* AES-NI section */
105
106# define AESNI_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(57-32)))
107# ifdef VPAES_ASM
108# define VPAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
109# endif
110# ifdef BSAES_ASM
111# define BSAES_CAPABLE (OPENSSL_ia32cap_P[1]&(1<<(41-32)))
112# endif
113
114int aesni_set_encrypt_key(const unsigned char *userKey, int bits,
115 AES_KEY *key);
116int aesni_set_decrypt_key(const unsigned char *userKey, int bits,
117 AES_KEY *key);
118
119void aesni_encrypt(const unsigned char *in, unsigned char *out,
120 const AES_KEY *key);
121void aesni_decrypt(const unsigned char *in, unsigned char *out,
122 const AES_KEY *key);
123
124void aesni_ecb_encrypt(const unsigned char *in,
125 unsigned char *out,
126 size_t length, const AES_KEY *key, int enc);
127void aesni_cbc_encrypt(const unsigned char *in,
128 unsigned char *out,
129 size_t length,
130 const AES_KEY *key, unsigned char *ivec, int enc);
131# ifndef OPENSSL_NO_OCB
132void aesni_ocb_encrypt(const unsigned char *in, unsigned char *out,
133 size_t blocks, const void *key,
134 size_t start_block_num,
135 unsigned char offset_i[16],
136 const unsigned char L_[][16],
137 unsigned char checksum[16]);
138void aesni_ocb_decrypt(const unsigned char *in, unsigned char *out,
139 size_t blocks, const void *key,
140 size_t start_block_num,
141 unsigned char offset_i[16],
142 const unsigned char L_[][16],
143 unsigned char checksum[16]);
144# endif /* OPENSSL_NO_OCB */
145
146void aesni_ctr32_encrypt_blocks(const unsigned char *in,
147 unsigned char *out,
148 size_t blocks,
149 const void *key, const unsigned char *ivec);
150
151void aesni_xts_encrypt(const unsigned char *in,
152 unsigned char *out,
153 size_t length,
154 const AES_KEY *key1, const AES_KEY *key2,
155 const unsigned char iv[16]);
156
157void aesni_xts_decrypt(const unsigned char *in,
158 unsigned char *out,
159 size_t length,
160 const AES_KEY *key1, const AES_KEY *key2,
161 const unsigned char iv[16]);
162
163void aesni_ccm64_encrypt_blocks(const unsigned char *in,
164 unsigned char *out,
165 size_t blocks,
166 const void *key,
167 const unsigned char ivec[16],
168 unsigned char cmac[16]);
169
170void aesni_ccm64_decrypt_blocks(const unsigned char *in,
171 unsigned char *out,
172 size_t blocks,
173 const void *key,
174 const unsigned char ivec[16],
175 unsigned char cmac[16]);
176
177# if defined(__x86_64) || defined(__x86_64__) || defined(_M_AMD64) || defined(_M_X64)
178size_t aesni_gcm_encrypt(const unsigned char *in, unsigned char *out, size_t len,
179 const void *key, unsigned char ivec[16], u64 *Xi);
180size_t aesni_gcm_decrypt(const unsigned char *in, unsigned char *out, size_t len,
181 const void *key, unsigned char ivec[16], u64 *Xi);
182void gcm_ghash_avx(u64 Xi[2], const u128 Htable[16], const u8 *in, size_t len);
183
184# define AES_GCM_ASM(ctx) (ctx->ctr == aesni_ctr32_encrypt_blocks && \
185 ctx->gcm.ghash == gcm_ghash_avx)
186# endif
187
188
189# elif defined(AES_ASM) && (defined(__sparc) || defined(__sparc__))
190
191/* Fujitsu SPARC64 X support */
192extern unsigned int OPENSSL_sparcv9cap_P[];
193# include "sparc_arch.h"
194
195# ifndef OPENSSL_NO_CAMELLIA
196# define SPARC_CMLL_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_CAMELLIA)
197# include "openssl/camellia.h"
198
199void cmll_t4_set_key(const unsigned char *key, int bits, CAMELLIA_KEY *ks);
200void cmll_t4_encrypt(const unsigned char *in, unsigned char *out,
201 const CAMELLIA_KEY *key);
202void cmll_t4_decrypt(const unsigned char *in, unsigned char *out,
203 const CAMELLIA_KEY *key);
204
205void cmll128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
206 size_t len, const CAMELLIA_KEY *key,
207 unsigned char *ivec);
208void cmll128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
209 size_t len, const CAMELLIA_KEY *key,
210 unsigned char *ivec);
211void cmll256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
212 size_t len, const CAMELLIA_KEY *key,
213 unsigned char *ivec);
214void cmll256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
215 size_t len, const CAMELLIA_KEY *key,
216 unsigned char *ivec);
217void cmll128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
218 size_t blocks, const CAMELLIA_KEY *key,
219 unsigned char *ivec);
220void cmll256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
221 size_t blocks, const CAMELLIA_KEY *key,
222 unsigned char *ivec);
223# endif /* OPENSSL_NO_CAMELLIA */
224
225
226# define SPARC_AES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_AES)
227# define SPARC_DES_CAPABLE (OPENSSL_sparcv9cap_P[1] & CFR_DES)
228# define HWAES_CAPABLE (OPENSSL_sparcv9cap_P[0] & SPARCV9_FJAESX)
229# define HWAES_set_encrypt_key aes_fx_set_encrypt_key
230# define HWAES_set_decrypt_key aes_fx_set_decrypt_key
231# define HWAES_encrypt aes_fx_encrypt
232# define HWAES_decrypt aes_fx_decrypt
233# define HWAES_cbc_encrypt aes_fx_cbc_encrypt
234# define HWAES_ctr32_encrypt_blocks aes_fx_ctr32_encrypt_blocks
235
236void aes_t4_set_encrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
237void aes_t4_set_decrypt_key(const unsigned char *key, int bits, AES_KEY *ks);
238void aes_t4_encrypt(const unsigned char *in, unsigned char *out,
239 const AES_KEY *key);
240void aes_t4_decrypt(const unsigned char *in, unsigned char *out,
241 const AES_KEY *key);
242/*
243 * Key-length specific subroutines were chosen for following reason.
244 * Each SPARC T4 core can execute up to 8 threads which share core's
245 * resources. Loading as much key material to registers allows to
246 * minimize references to shared memory interface, as well as amount
247 * of instructions in inner loops [much needed on T4]. But then having
248 * non-key-length specific routines would require conditional branches
249 * either in inner loops or on subroutines' entries. Former is hardly
250 * acceptable, while latter means code size increase to size occupied
251 * by multiple key-length specific subroutines, so why fight?
252 */
253void aes128_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
254 size_t len, const AES_KEY *key,
255 unsigned char *ivec);
256void aes128_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
257 size_t len, const AES_KEY *key,
258 unsigned char *ivec);
259void aes192_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
260 size_t len, const AES_KEY *key,
261 unsigned char *ivec);
262void aes192_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
263 size_t len, const AES_KEY *key,
264 unsigned char *ivec);
265void aes256_t4_cbc_encrypt(const unsigned char *in, unsigned char *out,
266 size_t len, const AES_KEY *key,
267 unsigned char *ivec);
268void aes256_t4_cbc_decrypt(const unsigned char *in, unsigned char *out,
269 size_t len, const AES_KEY *key,
270 unsigned char *ivec);
271void aes128_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
272 size_t blocks, const AES_KEY *key,
273 unsigned char *ivec);
274void aes192_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
275 size_t blocks, const AES_KEY *key,
276 unsigned char *ivec);
277void aes256_t4_ctr32_encrypt(const unsigned char *in, unsigned char *out,
278 size_t blocks, const AES_KEY *key,
279 unsigned char *ivec);
280void aes128_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
281 size_t blocks, const AES_KEY *key1,
282 const AES_KEY *key2, const unsigned char *ivec);
283void aes128_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
284 size_t blocks, const AES_KEY *key1,
285 const AES_KEY *key2, const unsigned char *ivec);
286void aes256_t4_xts_encrypt(const unsigned char *in, unsigned char *out,
287 size_t blocks, const AES_KEY *key1,
288 const AES_KEY *key2, const unsigned char *ivec);
289void aes256_t4_xts_decrypt(const unsigned char *in, unsigned char *out,
290 size_t blocks, const AES_KEY *key1,
291 const AES_KEY *key2, const unsigned char *ivec);
292
293# ifndef OPENSSL_NO_DES
294# include <openssl/des.h>
295void des_t4_key_expand(const void *key, DES_key_schedule *ks);
296void des_t4_ede3_cbc_encrypt(const void *inp, void *out, size_t len,
297 const DES_key_schedule ks[3], unsigned char iv[8]);
298void des_t4_ede3_cbc_decrypt(const void *inp, void *out, size_t len,
299 const DES_key_schedule ks[3], unsigned char iv[8]);
300void des_t4_cbc_encrypt(const void *inp, void *out, size_t len,
301 const DES_key_schedule *ks, unsigned char iv[8]);
302void des_t4_cbc_decrypt(const void *inp, void *out, size_t len,
303 const DES_key_schedule *ks, unsigned char iv[8]);
304# endif /* OPENSSL_NO_DES */
305
306# elif defined(OPENSSL_CPUID_OBJ) && defined(__s390__)
307/* IBM S390X support */
308# include "s390x_arch.h"
309
310
311/* Convert key size to function code: [16,24,32] -> [18,19,20]. */
312# define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
313
314/* Most modes of operation need km for partial block processing. */
315# define S390X_aes_128_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
316 S390X_CAPBIT(S390X_AES_128))
317# define S390X_aes_192_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
318 S390X_CAPBIT(S390X_AES_192))
319# define S390X_aes_256_CAPABLE (OPENSSL_s390xcap_P.km[0] & \
320 S390X_CAPBIT(S390X_AES_256))
321
322# define S390X_aes_128_cbc_CAPABLE 1 /* checked by callee */
323# define S390X_aes_192_cbc_CAPABLE 1
324# define S390X_aes_256_cbc_CAPABLE 1
325
326# define S390X_aes_128_ecb_CAPABLE S390X_aes_128_CAPABLE
327# define S390X_aes_192_ecb_CAPABLE S390X_aes_192_CAPABLE
328# define S390X_aes_256_ecb_CAPABLE S390X_aes_256_CAPABLE
329
330# define S390X_aes_128_ofb_CAPABLE (S390X_aes_128_CAPABLE && \
331 (OPENSSL_s390xcap_P.kmo[0] & \
332 S390X_CAPBIT(S390X_AES_128)))
333# define S390X_aes_192_ofb_CAPABLE (S390X_aes_192_CAPABLE && \
334 (OPENSSL_s390xcap_P.kmo[0] & \
335 S390X_CAPBIT(S390X_AES_192)))
336# define S390X_aes_256_ofb_CAPABLE (S390X_aes_256_CAPABLE && \
337 (OPENSSL_s390xcap_P.kmo[0] & \
338 S390X_CAPBIT(S390X_AES_256)))
339
340# define S390X_aes_128_cfb_CAPABLE (S390X_aes_128_CAPABLE && \
341 (OPENSSL_s390xcap_P.kmf[0] & \
342 S390X_CAPBIT(S390X_AES_128)))
343# define S390X_aes_192_cfb_CAPABLE (S390X_aes_192_CAPABLE && \
344 (OPENSSL_s390xcap_P.kmf[0] & \
345 S390X_CAPBIT(S390X_AES_192)))
346# define S390X_aes_256_cfb_CAPABLE (S390X_aes_256_CAPABLE && \
347 (OPENSSL_s390xcap_P.kmf[0] & \
348 S390X_CAPBIT(S390X_AES_256)))
349# define S390X_aes_128_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
350 S390X_CAPBIT(S390X_AES_128))
351# define S390X_aes_192_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
352 S390X_CAPBIT(S390X_AES_192))
353# define S390X_aes_256_cfb8_CAPABLE (OPENSSL_s390xcap_P.kmf[0] & \
354 S390X_CAPBIT(S390X_AES_256))
355# define S390X_aes_128_cfb1_CAPABLE 0
356# define S390X_aes_192_cfb1_CAPABLE 0
357# define S390X_aes_256_cfb1_CAPABLE 0
358
359# define S390X_aes_128_ctr_CAPABLE 1 /* checked by callee */
360# define S390X_aes_192_ctr_CAPABLE 1
361# define S390X_aes_256_ctr_CAPABLE 1
362
363# define S390X_aes_128_xts_CAPABLE 1 /* checked by callee */
364# define S390X_aes_256_xts_CAPABLE 1
365
366# define S390X_aes_128_gcm_CAPABLE (S390X_aes_128_CAPABLE && \
367 (OPENSSL_s390xcap_P.kma[0] & \
368 S390X_CAPBIT(S390X_AES_128)))
369# define S390X_aes_192_gcm_CAPABLE (S390X_aes_192_CAPABLE && \
370 (OPENSSL_s390xcap_P.kma[0] & \
371 S390X_CAPBIT(S390X_AES_192)))
372# define S390X_aes_256_gcm_CAPABLE (S390X_aes_256_CAPABLE && \
373 (OPENSSL_s390xcap_P.kma[0] & \
374 S390X_CAPBIT(S390X_AES_256)))
375
376# define S390X_aes_128_ccm_CAPABLE (S390X_aes_128_CAPABLE && \
377 (OPENSSL_s390xcap_P.kmac[0] & \
378 S390X_CAPBIT(S390X_AES_128)))
379# define S390X_aes_192_ccm_CAPABLE (S390X_aes_192_CAPABLE && \
380 (OPENSSL_s390xcap_P.kmac[0] & \
381 S390X_CAPBIT(S390X_AES_192)))
382# define S390X_aes_256_ccm_CAPABLE (S390X_aes_256_CAPABLE && \
383 (OPENSSL_s390xcap_P.kmac[0] & \
384 S390X_CAPBIT(S390X_AES_256)))
385# define S390X_CCM_AAD_FLAG 0x40
386
387# ifndef OPENSSL_NO_OCB
388# define S390X_aes_128_ocb_CAPABLE 0
389# define S390X_aes_192_ocb_CAPABLE 0
390# define S390X_aes_256_ocb_CAPABLE 0
391# endif /* OPENSSL_NO_OCB */
392
393# ifndef OPENSSL_NO_SIV
394# define S390X_aes_128_siv_CAPABLE 0
395# define S390X_aes_192_siv_CAPABLE 0
396# define S390X_aes_256_siv_CAPABLE 0
397# endif /* OPENSSL_NO_SIV */
398
399/* Convert key size to function code: [16,24,32] -> [18,19,20]. */
400# define S390X_AES_FC(keylen) (S390X_AES_128 + ((((keylen) << 3) - 128) >> 6))
401# endif
402
403# if defined(HWAES_CAPABLE)
404int HWAES_set_encrypt_key(const unsigned char *userKey, const int bits,
405 AES_KEY *key);
406int HWAES_set_decrypt_key(const unsigned char *userKey, const int bits,
407 AES_KEY *key);
408void HWAES_encrypt(const unsigned char *in, unsigned char *out,
409 const AES_KEY *key);
410void HWAES_decrypt(const unsigned char *in, unsigned char *out,
411 const AES_KEY *key);
412void HWAES_cbc_encrypt(const unsigned char *in, unsigned char *out,
413 size_t length, const AES_KEY *key,
414 unsigned char *ivec, const int enc);
415void HWAES_ecb_encrypt(const unsigned char *in, unsigned char *out,
416 size_t length, const AES_KEY *key,
417 const int enc);
418void HWAES_ctr32_encrypt_blocks(const unsigned char *in, unsigned char *out,
419 size_t len, const AES_KEY *key,
420 const unsigned char ivec[16]);
421void HWAES_xts_encrypt(const unsigned char *inp, unsigned char *out,
422 size_t len, const AES_KEY *key1,
423 const AES_KEY *key2, const unsigned char iv[16]);
424void HWAES_xts_decrypt(const unsigned char *inp, unsigned char *out,
425 size_t len, const AES_KEY *key1,
426 const AES_KEY *key2, const unsigned char iv[16]);
427# ifndef OPENSSL_NO_OCB
428# ifdef HWAES_ocb_encrypt
429void HWAES_ocb_encrypt(const unsigned char *in, unsigned char *out,
430 size_t blocks, const void *key,
431 size_t start_block_num,
432 unsigned char offset_i[16],
433 const unsigned char L_[][16],
434 unsigned char checksum[16]);
435# else
436# define HWAES_ocb_encrypt ((ocb128_f)NULL)
437# endif
438# ifdef HWAES_ocb_decrypt
439void HWAES_ocb_decrypt(const unsigned char *in, unsigned char *out,
440 size_t blocks, const void *key,
441 size_t start_block_num,
442 unsigned char offset_i[16],
443 const unsigned char L_[][16],
444 unsigned char checksum[16]);
445# else
446# define HWAES_ocb_decrypt ((ocb128_f)NULL)
447# endif
448# endif /* OPENSSL_NO_OCB */
449
450# endif /* HWAES_CAPABLE */
451
452#endif /* OSSL_CRYPTO_CIPHERMODE_PLATFORM_H */
453