1/* ====================================================================
2 * Copyright (c) 2008 The OpenSSL Project. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in
13 * the documentation and/or other materials provided with the
14 * distribution.
15 *
16 * 3. All advertising materials mentioning features or use of this
17 * software must display the following acknowledgment:
18 * "This product includes software developed by the OpenSSL Project
19 * for use in the OpenSSL Toolkit. (http://www.openssl.org/)"
20 *
21 * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to
22 * endorse or promote products derived from this software without
23 * prior written permission. For written permission, please contact
24 * openssl-core@openssl.org.
25 *
26 * 5. Products derived from this software may not be called "OpenSSL"
27 * nor may "OpenSSL" appear in their names without prior written
28 * permission of the OpenSSL Project.
29 *
30 * 6. Redistributions of any form whatsoever must retain the following
31 * acknowledgment:
32 * "This product includes software developed by the OpenSSL Project
33 * for use in the OpenSSL Toolkit (http://www.openssl.org/)"
34 *
35 * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY
36 * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
37 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
38 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR
39 * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
40 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
41 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
42 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
43 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
44 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
45 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
46 * OF THE POSSIBILITY OF SUCH DAMAGE.
47 * ==================================================================== */
48
49#ifndef OPENSSL_HEADER_MODES_INTERNAL_H
50#define OPENSSL_HEADER_MODES_INTERNAL_H
51
52#include <openssl/base.h>
53
54#include <openssl/aes.h>
55#include <openssl/cpu.h>
56
57#include <stdlib.h>
58#include <string.h>
59
60#include "../../internal.h"
61
62#if defined(__cplusplus)
63extern "C" {
64#endif
65
66
67static inline uint32_t GETU32(const void *in) {
68 uint32_t v;
69 OPENSSL_memcpy(&v, in, sizeof(v));
70 return CRYPTO_bswap4(v);
71}
72
73static inline void PUTU32(void *out, uint32_t v) {
74 v = CRYPTO_bswap4(v);
75 OPENSSL_memcpy(out, &v, sizeof(v));
76}
77
78static inline size_t load_word_le(const void *in) {
79 size_t v;
80 OPENSSL_memcpy(&v, in, sizeof(v));
81 return v;
82}
83
84static inline void store_word_le(void *out, size_t v) {
85 OPENSSL_memcpy(out, &v, sizeof(v));
86}
87
88// block128_f is the type of an AES block cipher implementation.
89//
90// Unlike upstream OpenSSL, it and the other functions in this file hard-code
91// |AES_KEY|. It is undefined in C to call a function pointer with anything
92// other than the original type. Thus we either must match |block128_f| to the
93// type signature of |AES_encrypt| and friends or pass in |void*| wrapper
94// functions.
95//
96// These functions are called exclusively with AES, so we use the former.
97typedef void (*block128_f)(const uint8_t in[16], uint8_t out[16],
98 const AES_KEY *key);
99
100
101// CTR.
102
103// ctr128_f is the type of a function that performs CTR-mode encryption.
104typedef void (*ctr128_f)(const uint8_t *in, uint8_t *out, size_t blocks,
105 const AES_KEY *key, const uint8_t ivec[16]);
106
107// CRYPTO_ctr128_encrypt encrypts (or decrypts, it's the same in CTR mode)
108// |len| bytes from |in| to |out| using |block| in counter mode. There's no
109// requirement that |len| be a multiple of any value and any partial blocks are
110// stored in |ecount_buf| and |*num|, which must be zeroed before the initial
111// call. The counter is a 128-bit, big-endian value in |ivec| and is
112// incremented by this function.
113void CRYPTO_ctr128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
114 const AES_KEY *key, uint8_t ivec[16],
115 uint8_t ecount_buf[16], unsigned *num,
116 block128_f block);
117
118// CRYPTO_ctr128_encrypt_ctr32 acts like |CRYPTO_ctr128_encrypt| but takes
119// |ctr|, a function that performs CTR mode but only deals with the lower 32
120// bits of the counter. This is useful when |ctr| can be an optimised
121// function.
122void CRYPTO_ctr128_encrypt_ctr32(const uint8_t *in, uint8_t *out, size_t len,
123 const AES_KEY *key, uint8_t ivec[16],
124 uint8_t ecount_buf[16], unsigned *num,
125 ctr128_f ctr);
126
127
128// GCM.
129//
130// This API differs from the upstream API slightly. The |GCM128_CONTEXT| does
131// not have a |key| pointer that points to the key as upstream's version does.
132// Instead, every function takes a |key| parameter. This way |GCM128_CONTEXT|
133// can be safely copied. Additionally, |gcm_key| is split into a separate
134// struct.
135
136typedef struct { uint64_t hi,lo; } u128;
137
138// gmult_func multiplies |Xi| by the GCM key and writes the result back to
139// |Xi|.
140typedef void (*gmult_func)(uint64_t Xi[2], const u128 Htable[16]);
141
142// ghash_func repeatedly multiplies |Xi| by the GCM key and adds in blocks from
143// |inp|. The result is written back to |Xi| and the |len| argument must be a
144// multiple of 16.
145typedef void (*ghash_func)(uint64_t Xi[2], const u128 Htable[16],
146 const uint8_t *inp, size_t len);
147
148typedef struct gcm128_key_st {
149 // Note the MOVBE-based, x86-64, GHASH assembly requires |H| and |Htable| to
150 // be the first two elements of this struct. Additionally, some assembly
151 // routines require a 16-byte-aligned |Htable| when hashing data, but not
152 // initialization. |GCM128_KEY| is not itself aligned to simplify embedding in
153 // |EVP_AEAD_CTX|, but |Htable|'s offset must be a multiple of 16.
154 u128 H;
155 u128 Htable[16];
156 gmult_func gmult;
157 ghash_func ghash;
158
159 block128_f block;
160
161 // use_aesni_gcm_crypt is true if this context should use the assembly
162 // functions |aesni_gcm_encrypt| and |aesni_gcm_decrypt| to process data.
163 unsigned use_aesni_gcm_crypt:1;
164} GCM128_KEY;
165
166// GCM128_CONTEXT contains state for a single GCM operation. The structure
167// should be zero-initialized before use.
168typedef struct {
169 // The following 5 names follow names in GCM specification
170 union {
171 uint64_t u[2];
172 uint32_t d[4];
173 uint8_t c[16];
174 size_t t[16 / sizeof(size_t)];
175 } Yi, EKi, EK0, len, Xi;
176
177 // Note that the order of |Xi| and |gcm_key| is fixed by the MOVBE-based,
178 // x86-64, GHASH assembly. Additionally, some assembly routines require
179 // |gcm_key| to be 16-byte aligned. |GCM128_KEY| is not itself aligned to
180 // simplify embedding in |EVP_AEAD_CTX|.
181 alignas(16) GCM128_KEY gcm_key;
182
183 unsigned mres, ares;
184} GCM128_CONTEXT;
185
186#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
187// crypto_gcm_clmul_enabled returns one if the CLMUL implementation of GCM is
188// used.
189int crypto_gcm_clmul_enabled(void);
190#endif
191
192// CRYPTO_ghash_init writes a precomputed table of powers of |gcm_key| to
193// |out_table| and sets |*out_mult| and |*out_hash| to (potentially hardware
194// accelerated) functions for performing operations in the GHASH field. If the
195// AVX implementation was used |*out_is_avx| will be true.
196void CRYPTO_ghash_init(gmult_func *out_mult, ghash_func *out_hash,
197 u128 *out_key, u128 out_table[16], int *out_is_avx,
198 const uint8_t gcm_key[16]);
199
200// CRYPTO_gcm128_init_key initialises |gcm_key| to use |block| (typically AES)
201// with the given key. |block_is_hwaes| is one if |block| is |aes_hw_encrypt|.
202OPENSSL_EXPORT void CRYPTO_gcm128_init_key(GCM128_KEY *gcm_key,
203 const AES_KEY *key, block128_f block,
204 int block_is_hwaes);
205
206// CRYPTO_gcm128_setiv sets the IV (nonce) for |ctx|. The |key| must be the
207// same key that was passed to |CRYPTO_gcm128_init|.
208OPENSSL_EXPORT void CRYPTO_gcm128_setiv(GCM128_CONTEXT *ctx, const AES_KEY *key,
209 const uint8_t *iv, size_t iv_len);
210
211// CRYPTO_gcm128_aad sets the authenticated data for an instance of GCM.
212// This must be called before and data is encrypted. It returns one on success
213// and zero otherwise.
214OPENSSL_EXPORT int CRYPTO_gcm128_aad(GCM128_CONTEXT *ctx, const uint8_t *aad,
215 size_t len);
216
217// CRYPTO_gcm128_encrypt encrypts |len| bytes from |in| to |out|. The |key|
218// must be the same key that was passed to |CRYPTO_gcm128_init|. It returns one
219// on success and zero otherwise.
220OPENSSL_EXPORT int CRYPTO_gcm128_encrypt(GCM128_CONTEXT *ctx,
221 const AES_KEY *key, const uint8_t *in,
222 uint8_t *out, size_t len);
223
224// CRYPTO_gcm128_decrypt decrypts |len| bytes from |in| to |out|. The |key|
225// must be the same key that was passed to |CRYPTO_gcm128_init|. It returns one
226// on success and zero otherwise.
227OPENSSL_EXPORT int CRYPTO_gcm128_decrypt(GCM128_CONTEXT *ctx,
228 const AES_KEY *key, const uint8_t *in,
229 uint8_t *out, size_t len);
230
231// CRYPTO_gcm128_encrypt_ctr32 encrypts |len| bytes from |in| to |out| using
232// a CTR function that only handles the bottom 32 bits of the nonce, like
233// |CRYPTO_ctr128_encrypt_ctr32|. The |key| must be the same key that was
234// passed to |CRYPTO_gcm128_init|. It returns one on success and zero
235// otherwise.
236OPENSSL_EXPORT int CRYPTO_gcm128_encrypt_ctr32(GCM128_CONTEXT *ctx,
237 const AES_KEY *key,
238 const uint8_t *in, uint8_t *out,
239 size_t len, ctr128_f stream);
240
241// CRYPTO_gcm128_decrypt_ctr32 decrypts |len| bytes from |in| to |out| using
242// a CTR function that only handles the bottom 32 bits of the nonce, like
243// |CRYPTO_ctr128_encrypt_ctr32|. The |key| must be the same key that was
244// passed to |CRYPTO_gcm128_init|. It returns one on success and zero
245// otherwise.
246OPENSSL_EXPORT int CRYPTO_gcm128_decrypt_ctr32(GCM128_CONTEXT *ctx,
247 const AES_KEY *key,
248 const uint8_t *in, uint8_t *out,
249 size_t len, ctr128_f stream);
250
251// CRYPTO_gcm128_finish calculates the authenticator and compares it against
252// |len| bytes of |tag|. It returns one on success and zero otherwise.
253OPENSSL_EXPORT int CRYPTO_gcm128_finish(GCM128_CONTEXT *ctx, const uint8_t *tag,
254 size_t len);
255
256// CRYPTO_gcm128_tag calculates the authenticator and copies it into |tag|.
257// The minimum of |len| and 16 bytes are copied into |tag|.
258OPENSSL_EXPORT void CRYPTO_gcm128_tag(GCM128_CONTEXT *ctx, uint8_t *tag,
259 size_t len);
260
261
262// GCM assembly.
263
264#if !defined(OPENSSL_NO_ASM) && \
265 (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
266 defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) || \
267 defined(OPENSSL_PPC64LE))
268#define GHASH_ASM
269#endif
270
271void gcm_init_4bit(u128 Htable[16], const uint64_t H[2]);
272void gcm_gmult_4bit(uint64_t Xi[2], const u128 Htable[16]);
273void gcm_ghash_4bit(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
274 size_t len);
275
276#if defined(GHASH_ASM)
277
278#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
279#define GCM_FUNCREF_4BIT
280void gcm_init_clmul(u128 Htable[16], const uint64_t Xi[2]);
281void gcm_gmult_clmul(uint64_t Xi[2], const u128 Htable[16]);
282void gcm_ghash_clmul(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
283 size_t len);
284
285OPENSSL_INLINE char gcm_ssse3_capable(void) {
286 return (OPENSSL_ia32cap_get()[1] & (1 << (41 - 32))) != 0;
287}
288
289// |gcm_gmult_ssse3| and |gcm_ghash_ssse3| require |Htable| to be
290// 16-byte-aligned, but |gcm_init_ssse3| does not.
291void gcm_init_ssse3(u128 Htable[16], const uint64_t Xi[2]);
292void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]);
293void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
294 size_t len);
295
296#if defined(OPENSSL_X86_64)
297#define GHASH_ASM_X86_64
298void gcm_init_avx(u128 Htable[16], const uint64_t Xi[2]);
299void gcm_gmult_avx(uint64_t Xi[2], const u128 Htable[16]);
300void gcm_ghash_avx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in,
301 size_t len);
302
303#define AESNI_GCM
304size_t aesni_gcm_encrypt(const uint8_t *in, uint8_t *out, size_t len,
305 const AES_KEY *key, uint8_t ivec[16], uint64_t *Xi);
306size_t aesni_gcm_decrypt(const uint8_t *in, uint8_t *out, size_t len,
307 const AES_KEY *key, uint8_t ivec[16], uint64_t *Xi);
308#endif // OPENSSL_X86_64
309
310#if defined(OPENSSL_X86)
311#define GHASH_ASM_X86
312void gcm_gmult_4bit_mmx(uint64_t Xi[2], const u128 Htable[16]);
313void gcm_ghash_4bit_mmx(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
314 size_t len);
315#endif // OPENSSL_X86
316
317#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
318#define GHASH_ASM_ARM
319#define GCM_FUNCREF_4BIT
320
321OPENSSL_INLINE int gcm_pmull_capable(void) {
322 return CRYPTO_is_ARMv8_PMULL_capable();
323}
324
325void gcm_init_v8(u128 Htable[16], const uint64_t Xi[2]);
326void gcm_gmult_v8(uint64_t Xi[2], const u128 Htable[16]);
327void gcm_ghash_v8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
328 size_t len);
329
330OPENSSL_INLINE int gcm_neon_capable(void) { return CRYPTO_is_NEON_capable(); }
331
332void gcm_init_neon(u128 Htable[16], const uint64_t Xi[2]);
333void gcm_gmult_neon(uint64_t Xi[2], const u128 Htable[16]);
334void gcm_ghash_neon(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
335 size_t len);
336
337#elif defined(OPENSSL_PPC64LE)
338#define GHASH_ASM_PPC64LE
339#define GCM_FUNCREF_4BIT
340void gcm_init_p8(u128 Htable[16], const uint64_t Xi[2]);
341void gcm_gmult_p8(uint64_t Xi[2], const u128 Htable[16]);
342void gcm_ghash_p8(uint64_t Xi[2], const u128 Htable[16], const uint8_t *inp,
343 size_t len);
344#endif
345#endif // GHASH_ASM
346
347
348// CCM.
349
350typedef struct ccm128_context {
351 block128_f block;
352 ctr128_f ctr;
353 unsigned M, L;
354} CCM128_CONTEXT;
355
356// CRYPTO_ccm128_init initialises |ctx| to use |block| (typically AES) with the
357// specified |M| and |L| parameters. It returns one on success and zero if |M|
358// or |L| is invalid.
359int CRYPTO_ccm128_init(CCM128_CONTEXT *ctx, const AES_KEY *key,
360 block128_f block, ctr128_f ctr, unsigned M, unsigned L);
361
362// CRYPTO_ccm128_max_input returns the maximum input length accepted by |ctx|.
363size_t CRYPTO_ccm128_max_input(const CCM128_CONTEXT *ctx);
364
365// CRYPTO_ccm128_encrypt encrypts |len| bytes from |in| to |out| writing the tag
366// to |out_tag|. |key| must be the same key that was passed to
367// |CRYPTO_ccm128_init|. It returns one on success and zero otherwise.
368int CRYPTO_ccm128_encrypt(const CCM128_CONTEXT *ctx, const AES_KEY *key,
369 uint8_t *out, uint8_t *out_tag, size_t tag_len,
370 const uint8_t *nonce, size_t nonce_len,
371 const uint8_t *in, size_t len, const uint8_t *aad,
372 size_t aad_len);
373
374// CRYPTO_ccm128_decrypt decrypts |len| bytes from |in| to |out|, writing the
375// expected tag to |out_tag|. |key| must be the same key that was passed to
376// |CRYPTO_ccm128_init|. It returns one on success and zero otherwise.
377int CRYPTO_ccm128_decrypt(const CCM128_CONTEXT *ctx, const AES_KEY *key,
378 uint8_t *out, uint8_t *out_tag, size_t tag_len,
379 const uint8_t *nonce, size_t nonce_len,
380 const uint8_t *in, size_t len, const uint8_t *aad,
381 size_t aad_len);
382
383
384// CBC.
385
386// cbc128_f is the type of a function that performs CBC-mode encryption.
387typedef void (*cbc128_f)(const uint8_t *in, uint8_t *out, size_t len,
388 const AES_KEY *key, uint8_t ivec[16], int enc);
389
390// CRYPTO_cbc128_encrypt encrypts |len| bytes from |in| to |out| using the
391// given IV and block cipher in CBC mode. The input need not be a multiple of
392// 128 bits long, but the output will round up to the nearest 128 bit multiple,
393// zero padding the input if needed. The IV will be updated on return.
394void CRYPTO_cbc128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
395 const AES_KEY *key, uint8_t ivec[16],
396 block128_f block);
397
398// CRYPTO_cbc128_decrypt decrypts |len| bytes from |in| to |out| using the
399// given IV and block cipher in CBC mode. If |len| is not a multiple of 128
400// bits then only that many bytes will be written, but a multiple of 128 bits
401// is always read from |in|. The IV will be updated on return.
402void CRYPTO_cbc128_decrypt(const uint8_t *in, uint8_t *out, size_t len,
403 const AES_KEY *key, uint8_t ivec[16],
404 block128_f block);
405
406
407// OFB.
408
409// CRYPTO_ofb128_encrypt encrypts (or decrypts, it's the same with OFB mode)
410// |len| bytes from |in| to |out| using |block| in OFB mode. There's no
411// requirement that |len| be a multiple of any value and any partial blocks are
412// stored in |ivec| and |*num|, the latter must be zero before the initial
413// call.
414void CRYPTO_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
415 const AES_KEY *key, uint8_t ivec[16], unsigned *num,
416 block128_f block);
417
418
419// CFB.
420
421// CRYPTO_cfb128_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
422// from |in| to |out| using |block| in CFB mode. There's no requirement that
423// |len| be a multiple of any value and any partial blocks are stored in |ivec|
424// and |*num|, the latter must be zero before the initial call.
425void CRYPTO_cfb128_encrypt(const uint8_t *in, uint8_t *out, size_t len,
426 const AES_KEY *key, uint8_t ivec[16], unsigned *num,
427 int enc, block128_f block);
428
429// CRYPTO_cfb128_8_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
430// from |in| to |out| using |block| in CFB-8 mode. Prior to the first call
431// |num| should be set to zero.
432void CRYPTO_cfb128_8_encrypt(const uint8_t *in, uint8_t *out, size_t len,
433 const AES_KEY *key, uint8_t ivec[16],
434 unsigned *num, int enc, block128_f block);
435
436// CRYPTO_cfb128_1_encrypt encrypts (or decrypts, if |enc| is zero) |len| bytes
437// from |in| to |out| using |block| in CFB-1 mode. Prior to the first call
438// |num| should be set to zero.
439void CRYPTO_cfb128_1_encrypt(const uint8_t *in, uint8_t *out, size_t bits,
440 const AES_KEY *key, uint8_t ivec[16],
441 unsigned *num, int enc, block128_f block);
442
443size_t CRYPTO_cts128_encrypt_block(const uint8_t *in, uint8_t *out, size_t len,
444 const AES_KEY *key, uint8_t ivec[16],
445 block128_f block);
446
447
448// POLYVAL.
449//
450// POLYVAL is a polynomial authenticator that operates over a field very
451// similar to the one that GHASH uses. See
452// https://tools.ietf.org/html/draft-irtf-cfrg-gcmsiv-02#section-3.
453
454typedef union {
455 uint64_t u[2];
456 uint8_t c[16];
457} polyval_block;
458
459struct polyval_ctx {
460 // Note that the order of |S|, |H| and |Htable| is fixed by the MOVBE-based,
461 // x86-64, GHASH assembly. Additionally, some assembly routines require
462 // |Htable| to be 16-byte aligned.
463 polyval_block S;
464 u128 H;
465 alignas(16) u128 Htable[16];
466 gmult_func gmult;
467 ghash_func ghash;
468};
469
470// CRYPTO_POLYVAL_init initialises |ctx| using |key|.
471void CRYPTO_POLYVAL_init(struct polyval_ctx *ctx, const uint8_t key[16]);
472
473// CRYPTO_POLYVAL_update_blocks updates the accumulator in |ctx| given the
474// blocks from |in|. Only a whole number of blocks can be processed so |in_len|
475// must be a multiple of 16.
476void CRYPTO_POLYVAL_update_blocks(struct polyval_ctx *ctx, const uint8_t *in,
477 size_t in_len);
478
479// CRYPTO_POLYVAL_finish writes the accumulator from |ctx| to |out|.
480void CRYPTO_POLYVAL_finish(const struct polyval_ctx *ctx, uint8_t out[16]);
481
482
483#if defined(__cplusplus)
484} // extern C
485#endif
486
487#endif // OPENSSL_HEADER_MODES_INTERNAL_H
488