wolfSSL | Mbed

Users » sPymbed » Code » wolfSSL

wolfcrypt/src/aes.c@15:117db924cf7c, 2018-08-18 (annotated)

Committer:: wolfSSL
Date:: Sat Aug 18 22:20:43 2018 +0000
Revision:: 15:117db924cf7c

wolfSSL 3.15.3

Who changed what in which revision?

User	Revision	Line number	New contents of line
wolfSSL	15:117db924cf7c	1	/* aes.c
wolfSSL	15:117db924cf7c	2	*
wolfSSL	15:117db924cf7c	3	* Copyright (C) 2006-2017 wolfSSL Inc.
wolfSSL	15:117db924cf7c	4	*
wolfSSL	15:117db924cf7c	5	* This file is part of wolfSSL.
wolfSSL	15:117db924cf7c	6	*
wolfSSL	15:117db924cf7c	7	* wolfSSL is free software; you can redistribute it and/or modify
wolfSSL	15:117db924cf7c	8	* it under the terms of the GNU General Public License as published by
wolfSSL	15:117db924cf7c	9	* the Free Software Foundation; either version 2 of the License, or
wolfSSL	15:117db924cf7c	10	* (at your option) any later version.
wolfSSL	15:117db924cf7c	11	*
wolfSSL	15:117db924cf7c	12	* wolfSSL is distributed in the hope that it will be useful,
wolfSSL	15:117db924cf7c	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
wolfSSL	15:117db924cf7c	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
wolfSSL	15:117db924cf7c	15	* GNU General Public License for more details.
wolfSSL	15:117db924cf7c	16	*
wolfSSL	15:117db924cf7c	17	* You should have received a copy of the GNU General Public License
wolfSSL	15:117db924cf7c	18	* along with this program; if not, write to the Free Software
wolfSSL	15:117db924cf7c	19	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
wolfSSL	15:117db924cf7c	20	*/
wolfSSL	15:117db924cf7c	21
wolfSSL	15:117db924cf7c	22
wolfSSL	15:117db924cf7c	23	#ifdef HAVE_CONFIG_H
wolfSSL	15:117db924cf7c	24	#include <config.h>
wolfSSL	15:117db924cf7c	25	#endif
wolfSSL	15:117db924cf7c	26
wolfSSL	15:117db924cf7c	27	#include <wolfssl/wolfcrypt/settings.h>
wolfSSL	15:117db924cf7c	28	#include <wolfssl/wolfcrypt/error-crypt.h>
wolfSSL	15:117db924cf7c	29
wolfSSL	15:117db924cf7c	30	#if !defined(NO_AES)
wolfSSL	15:117db924cf7c	31
wolfSSL	15:117db924cf7c	32	#if defined(HAVE_FIPS) && \
wolfSSL	15:117db924cf7c	33	defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2)
wolfSSL	15:117db924cf7c	34
wolfSSL	15:117db924cf7c	35	/* set NO_WRAPPERS before headers, use direct internal f()s not wrappers */
wolfSSL	15:117db924cf7c	36	#define FIPS_NO_WRAPPERS
wolfSSL	15:117db924cf7c	37
wolfSSL	15:117db924cf7c	38	#ifdef USE_WINDOWS_API
wolfSSL	15:117db924cf7c	39	#pragma code_seg(".fipsA$g")
wolfSSL	15:117db924cf7c	40	#pragma const_seg(".fipsB$g")
wolfSSL	15:117db924cf7c	41	#endif
wolfSSL	15:117db924cf7c	42	#endif
wolfSSL	15:117db924cf7c	43
wolfSSL	15:117db924cf7c	44	#include <wolfssl/wolfcrypt/aes.h>
wolfSSL	15:117db924cf7c	45	#include <wolfssl/wolfcrypt/cpuid.h>
wolfSSL	15:117db924cf7c	46
wolfSSL	15:117db924cf7c	47
wolfSSL	15:117db924cf7c	48	/* fips wrapper calls, user can call direct */
wolfSSL	15:117db924cf7c	49	#if defined(HAVE_FIPS) && \
wolfSSL	15:117db924cf7c	50	(!defined(HAVE_FIPS_VERSION) \|\| (HAVE_FIPS_VERSION < 2))
wolfSSL	15:117db924cf7c	51
wolfSSL	15:117db924cf7c	52	int wc_AesSetKey(Aes* aes, const byte* key, word32 len, const byte* iv,
wolfSSL	15:117db924cf7c	53	int dir)
wolfSSL	15:117db924cf7c	54	{
wolfSSL	15:117db924cf7c	55	if (aes == NULL \|\| !( (len == 16) \|\| (len == 24) \|\| (len == 32)) ) {
wolfSSL	15:117db924cf7c	56	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	57	}
wolfSSL	15:117db924cf7c	58
wolfSSL	15:117db924cf7c	59	return AesSetKey_fips(aes, key, len, iv, dir);
wolfSSL	15:117db924cf7c	60	}
wolfSSL	15:117db924cf7c	61	int wc_AesSetIV(Aes* aes, const byte* iv)
wolfSSL	15:117db924cf7c	62	{
wolfSSL	15:117db924cf7c	63	if (aes == NULL) {
wolfSSL	15:117db924cf7c	64	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	65	}
wolfSSL	15:117db924cf7c	66
wolfSSL	15:117db924cf7c	67	return AesSetIV_fips(aes, iv);
wolfSSL	15:117db924cf7c	68	}
wolfSSL	15:117db924cf7c	69	#ifdef HAVE_AES_CBC
wolfSSL	15:117db924cf7c	70	int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	71	{
wolfSSL	15:117db924cf7c	72	if (aes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	73	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	74	}
wolfSSL	15:117db924cf7c	75
wolfSSL	15:117db924cf7c	76	return AesCbcEncrypt_fips(aes, out, in, sz);
wolfSSL	15:117db924cf7c	77	}
wolfSSL	15:117db924cf7c	78	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	79	int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	80	{
wolfSSL	15:117db924cf7c	81	if (aes == NULL \|\| out == NULL \|\| in == NULL
wolfSSL	15:117db924cf7c	82	\|\| sz % AES_BLOCK_SIZE != 0) {
wolfSSL	15:117db924cf7c	83	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	84	}
wolfSSL	15:117db924cf7c	85
wolfSSL	15:117db924cf7c	86	return AesCbcDecrypt_fips(aes, out, in, sz);
wolfSSL	15:117db924cf7c	87	}
wolfSSL	15:117db924cf7c	88	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	89	#endif /* HAVE_AES_CBC */
wolfSSL	15:117db924cf7c	90
wolfSSL	15:117db924cf7c	91	/* AES-CTR */
wolfSSL	15:117db924cf7c	92	#ifdef WOLFSSL_AES_COUNTER
wolfSSL	15:117db924cf7c	93	int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	94	{
wolfSSL	15:117db924cf7c	95	if (aes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	96	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	97	}
wolfSSL	15:117db924cf7c	98
wolfSSL	15:117db924cf7c	99	return AesCtrEncrypt(aes, out, in, sz);
wolfSSL	15:117db924cf7c	100	}
wolfSSL	15:117db924cf7c	101	#endif
wolfSSL	15:117db924cf7c	102
wolfSSL	15:117db924cf7c	103	/* AES-DIRECT */
wolfSSL	15:117db924cf7c	104	#if defined(WOLFSSL_AES_DIRECT)
wolfSSL	15:117db924cf7c	105	void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
wolfSSL	15:117db924cf7c	106	{
wolfSSL	15:117db924cf7c	107	AesEncryptDirect(aes, out, in);
wolfSSL	15:117db924cf7c	108	}
wolfSSL	15:117db924cf7c	109
wolfSSL	15:117db924cf7c	110	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	111	void wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
wolfSSL	15:117db924cf7c	112	{
wolfSSL	15:117db924cf7c	113	AesDecryptDirect(aes, out, in);
wolfSSL	15:117db924cf7c	114	}
wolfSSL	15:117db924cf7c	115	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	116
wolfSSL	15:117db924cf7c	117	int wc_AesSetKeyDirect(Aes* aes, const byte* key, word32 len,
wolfSSL	15:117db924cf7c	118	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	119	{
wolfSSL	15:117db924cf7c	120	return AesSetKeyDirect(aes, key, len, iv, dir);
wolfSSL	15:117db924cf7c	121	}
wolfSSL	15:117db924cf7c	122	#endif /* WOLFSSL_AES_DIRECT */
wolfSSL	15:117db924cf7c	123
wolfSSL	15:117db924cf7c	124	/* AES-GCM */
wolfSSL	15:117db924cf7c	125	#ifdef HAVE_AESGCM
wolfSSL	15:117db924cf7c	126	int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
wolfSSL	15:117db924cf7c	127	{
wolfSSL	15:117db924cf7c	128	if (aes == NULL \|\| !( (len == 16) \|\| (len == 24) \|\| (len == 32)) ) {
wolfSSL	15:117db924cf7c	129	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	130	}
wolfSSL	15:117db924cf7c	131
wolfSSL	15:117db924cf7c	132	return AesGcmSetKey_fips(aes, key, len);
wolfSSL	15:117db924cf7c	133	}
wolfSSL	15:117db924cf7c	134	int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	135	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	136	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	137	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	138	{
wolfSSL	15:117db924cf7c	139	if (aes == NULL \|\| authTagSz > AES_BLOCK_SIZE
wolfSSL	15:117db924cf7c	140	\|\| authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ \|\|
wolfSSL	15:117db924cf7c	141	ivSz > AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	142	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	143	}
wolfSSL	15:117db924cf7c	144
wolfSSL	15:117db924cf7c	145	return AesGcmEncrypt_fips(aes, out, in, sz, iv, ivSz, authTag,
wolfSSL	15:117db924cf7c	146	authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	147	}
wolfSSL	15:117db924cf7c	148
wolfSSL	15:117db924cf7c	149	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	150	int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	151	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	152	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	153	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	154	{
wolfSSL	15:117db924cf7c	155	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| iv == NULL
wolfSSL	15:117db924cf7c	156	\|\| authTag == NULL \|\| authTagSz > AES_BLOCK_SIZE \|\|
wolfSSL	15:117db924cf7c	157	ivSz > AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	158	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	159	}
wolfSSL	15:117db924cf7c	160
wolfSSL	15:117db924cf7c	161	return AesGcmDecrypt_fips(aes, out, in, sz, iv, ivSz, authTag,
wolfSSL	15:117db924cf7c	162	authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	163	}
wolfSSL	15:117db924cf7c	164	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	165
wolfSSL	15:117db924cf7c	166	int wc_GmacSetKey(Gmac* gmac, const byte* key, word32 len)
wolfSSL	15:117db924cf7c	167	{
wolfSSL	15:117db924cf7c	168	if (gmac == NULL \|\| key == NULL \|\| !((len == 16) \|\|
wolfSSL	15:117db924cf7c	169	(len == 24) \|\| (len == 32)) ) {
wolfSSL	15:117db924cf7c	170	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	171	}
wolfSSL	15:117db924cf7c	172
wolfSSL	15:117db924cf7c	173	return GmacSetKey(gmac, key, len);
wolfSSL	15:117db924cf7c	174	}
wolfSSL	15:117db924cf7c	175	int wc_GmacUpdate(Gmac* gmac, const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	176	const byte* authIn, word32 authInSz,
wolfSSL	15:117db924cf7c	177	byte* authTag, word32 authTagSz)
wolfSSL	15:117db924cf7c	178	{
wolfSSL	15:117db924cf7c	179	if (gmac == NULL \|\| authTagSz > AES_BLOCK_SIZE \|\|
wolfSSL	15:117db924cf7c	180	authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
wolfSSL	15:117db924cf7c	181	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	182	}
wolfSSL	15:117db924cf7c	183
wolfSSL	15:117db924cf7c	184	return GmacUpdate(gmac, iv, ivSz, authIn, authInSz,
wolfSSL	15:117db924cf7c	185	authTag, authTagSz);
wolfSSL	15:117db924cf7c	186	}
wolfSSL	15:117db924cf7c	187	#endif /* HAVE_AESGCM */
wolfSSL	15:117db924cf7c	188
wolfSSL	15:117db924cf7c	189	/* AES-CCM */
wolfSSL	15:117db924cf7c	190	#if defined(HAVE_AESCCM) && \
wolfSSL	15:117db924cf7c	191	defined(HAVE_FIPS_VERSION) && (HAVE_FIPS_VERSION >= 2)
wolfSSL	15:117db924cf7c	192	int wc_AesCcmSetKey(Aes* aes, const byte* key, word32 keySz)
wolfSSL	15:117db924cf7c	193	{
wolfSSL	15:117db924cf7c	194	return AesCcmSetKey(aes, key, keySz);
wolfSSL	15:117db924cf7c	195	}
wolfSSL	15:117db924cf7c	196	int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
wolfSSL	15:117db924cf7c	197	const byte* nonce, word32 nonceSz,
wolfSSL	15:117db924cf7c	198	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	199	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	200	{
wolfSSL	15:117db924cf7c	201	/* sanity check on arguments */
wolfSSL	15:117db924cf7c	202	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| nonce == NULL
wolfSSL	15:117db924cf7c	203	\|\| authTag == NULL \|\| nonceSz < 7 \|\| nonceSz > 13)
wolfSSL	15:117db924cf7c	204	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	205
wolfSSL	15:117db924cf7c	206	AesCcmEncrypt(aes, out, in, inSz, nonce, nonceSz, authTag,
wolfSSL	15:117db924cf7c	207	authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	208	return 0;
wolfSSL	15:117db924cf7c	209	}
wolfSSL	15:117db924cf7c	210
wolfSSL	15:117db924cf7c	211	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	212	int wc_AesCcmDecrypt(Aes* aes, byte* out,
wolfSSL	15:117db924cf7c	213	const byte* in, word32 inSz,
wolfSSL	15:117db924cf7c	214	const byte* nonce, word32 nonceSz,
wolfSSL	15:117db924cf7c	215	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	216	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	217	{
wolfSSL	15:117db924cf7c	218
wolfSSL	15:117db924cf7c	219	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| nonce == NULL
wolfSSL	15:117db924cf7c	220	\|\| authTag == NULL \|\| nonceSz < 7 \|\| nonceSz > 13) {
wolfSSL	15:117db924cf7c	221	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	222	}
wolfSSL	15:117db924cf7c	223
wolfSSL	15:117db924cf7c	224	return AesCcmDecrypt(aes, out, in, inSz, nonce, nonceSz,
wolfSSL	15:117db924cf7c	225	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	226	}
wolfSSL	15:117db924cf7c	227	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	228	#endif /* HAVE_AESCCM && HAVE_FIPS_VERSION 2 */
wolfSSL	15:117db924cf7c	229
wolfSSL	15:117db924cf7c	230	int wc_AesInit(Aes* aes, void* h, int i)
wolfSSL	15:117db924cf7c	231	{
wolfSSL	15:117db924cf7c	232	(void)aes;
wolfSSL	15:117db924cf7c	233	(void)h;
wolfSSL	15:117db924cf7c	234	(void)i;
wolfSSL	15:117db924cf7c	235	/* FIPS doesn't support:
wolfSSL	15:117db924cf7c	236	return AesInit(aes, h, i); */
wolfSSL	15:117db924cf7c	237	return 0;
wolfSSL	15:117db924cf7c	238	}
wolfSSL	15:117db924cf7c	239	void wc_AesFree(Aes* aes)
wolfSSL	15:117db924cf7c	240	{
wolfSSL	15:117db924cf7c	241	(void)aes;
wolfSSL	15:117db924cf7c	242	/* FIPS doesn't support:
wolfSSL	15:117db924cf7c	243	AesFree(aes); */
wolfSSL	15:117db924cf7c	244	}
wolfSSL	15:117db924cf7c	245
wolfSSL	15:117db924cf7c	246	#else /* else build without fips, or for FIPS v2 */
wolfSSL	15:117db924cf7c	247
wolfSSL	15:117db924cf7c	248
wolfSSL	15:117db924cf7c	249	#if defined(WOLFSSL_TI_CRYPT)
wolfSSL	15:117db924cf7c	250	#include <wolfcrypt/src/port/ti/ti-aes.c>
wolfSSL	15:117db924cf7c	251	#else
wolfSSL	15:117db924cf7c	252
wolfSSL	15:117db924cf7c	253	#include <wolfssl/wolfcrypt/logging.h>
wolfSSL	15:117db924cf7c	254
wolfSSL	15:117db924cf7c	255	#ifdef NO_INLINE
wolfSSL	15:117db924cf7c	256	#include <wolfssl/wolfcrypt/misc.h>
wolfSSL	15:117db924cf7c	257	#else
wolfSSL	15:117db924cf7c	258	#define WOLFSSL_MISC_INCLUDED
wolfSSL	15:117db924cf7c	259	#include <wolfcrypt/src/misc.c>
wolfSSL	15:117db924cf7c	260	#endif
wolfSSL	15:117db924cf7c	261
wolfSSL	15:117db924cf7c	262	#if !defined(WOLFSSL_ARMASM)
wolfSSL	15:117db924cf7c	263
wolfSSL	15:117db924cf7c	264	#ifdef WOLFSSL_IMX6_CAAM_BLOB
wolfSSL	15:117db924cf7c	265	/* case of possibly not using hardware acceleration for AES but using key
wolfSSL	15:117db924cf7c	266	blobs */
wolfSSL	15:117db924cf7c	267	#include <wolfssl/wolfcrypt/port/caam/wolfcaam.h>
wolfSSL	15:117db924cf7c	268	#endif
wolfSSL	15:117db924cf7c	269
wolfSSL	15:117db924cf7c	270	#ifdef DEBUG_AESNI
wolfSSL	15:117db924cf7c	271	#include <stdio.h>
wolfSSL	15:117db924cf7c	272	#endif
wolfSSL	15:117db924cf7c	273
wolfSSL	15:117db924cf7c	274	#ifdef _MSC_VER
wolfSSL	15:117db924cf7c	275	/* 4127 warning constant while(1) */
wolfSSL	15:117db924cf7c	276	#pragma warning(disable: 4127)
wolfSSL	15:117db924cf7c	277	#endif
wolfSSL	15:117db924cf7c	278
wolfSSL	15:117db924cf7c	279
wolfSSL	15:117db924cf7c	280	/* Define AES implementation includes and functions */
wolfSSL	15:117db924cf7c	281	#if defined(STM32_CRYPTO)
wolfSSL	15:117db924cf7c	282	/* STM32F2/F4 hardware AES support for CBC, CTR modes */
wolfSSL	15:117db924cf7c	283
wolfSSL	15:117db924cf7c	284	#ifdef WOLFSSL_STM32L4
wolfSSL	15:117db924cf7c	285	#define CRYP AES
wolfSSL	15:117db924cf7c	286	#endif
wolfSSL	15:117db924cf7c	287
wolfSSL	15:117db924cf7c	288	/* CRYPT_AES_GCM starts the IV with 2 */
wolfSSL	15:117db924cf7c	289	#define STM32_GCM_IV_START 2
wolfSSL	15:117db924cf7c	290
wolfSSL	15:117db924cf7c	291	#if defined(WOLFSSL_AES_DIRECT) \|\| defined(HAVE_AESGCM) \|\| defined(HAVE_AESCCM)
wolfSSL	15:117db924cf7c	292	static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	293	{
wolfSSL	15:117db924cf7c	294	int ret = 0;
wolfSSL	15:117db924cf7c	295	#ifdef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	296	CRYP_HandleTypeDef hcryp;
wolfSSL	15:117db924cf7c	297
wolfSSL	15:117db924cf7c	298	XMEMSET(&hcryp, 0, sizeof(CRYP_HandleTypeDef));
wolfSSL	15:117db924cf7c	299	switch(aes->rounds) {
wolfSSL	15:117db924cf7c	300	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	301	hcryp.Init.KeySize = CRYP_KEYSIZE_128B;
wolfSSL	15:117db924cf7c	302	break;
wolfSSL	15:117db924cf7c	303	#ifdef CRYP_KEYSIZE_192B
wolfSSL	15:117db924cf7c	304	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	305	hcryp.Init.KeySize = CRYP_KEYSIZE_192B;
wolfSSL	15:117db924cf7c	306	break;
wolfSSL	15:117db924cf7c	307	#endif
wolfSSL	15:117db924cf7c	308	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	309	hcryp.Init.KeySize = CRYP_KEYSIZE_256B;
wolfSSL	15:117db924cf7c	310	break;
wolfSSL	15:117db924cf7c	311	default:
wolfSSL	15:117db924cf7c	312	break;
wolfSSL	15:117db924cf7c	313	}
wolfSSL	15:117db924cf7c	314	hcryp.Instance = CRYP;
wolfSSL	15:117db924cf7c	315	hcryp.Init.DataType = CRYP_DATATYPE_8B;
wolfSSL	15:117db924cf7c	316	hcryp.Init.pKey = (uint8_t*)aes->key;
wolfSSL	15:117db924cf7c	317
wolfSSL	15:117db924cf7c	318	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	319
wolfSSL	15:117db924cf7c	320	if (HAL_CRYP_AESECB_Encrypt(&hcryp, (uint8_t*)inBlock, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	321	outBlock, STM32_HAL_TIMEOUT) != HAL_OK) {
wolfSSL	15:117db924cf7c	322	ret = WC_TIMEOUT_E;
wolfSSL	15:117db924cf7c	323	}
wolfSSL	15:117db924cf7c	324
wolfSSL	15:117db924cf7c	325	HAL_CRYP_DeInit(&hcryp);
wolfSSL	15:117db924cf7c	326	#else
wolfSSL	15:117db924cf7c	327	word32 *enc_key;
wolfSSL	15:117db924cf7c	328	CRYP_InitTypeDef AES_CRYP_InitStructure;
wolfSSL	15:117db924cf7c	329	CRYP_KeyInitTypeDef AES_CRYP_KeyInitStructure;
wolfSSL	15:117db924cf7c	330
wolfSSL	15:117db924cf7c	331	enc_key = aes->key;
wolfSSL	15:117db924cf7c	332
wolfSSL	15:117db924cf7c	333	/* crypto structure initialization */
wolfSSL	15:117db924cf7c	334	CRYP_KeyStructInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	335	CRYP_StructInit(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	336
wolfSSL	15:117db924cf7c	337	/* reset registers to their default values */
wolfSSL	15:117db924cf7c	338	CRYP_DeInit();
wolfSSL	15:117db924cf7c	339
wolfSSL	15:117db924cf7c	340	/* load key into correct registers */
wolfSSL	15:117db924cf7c	341	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	342	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	343	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_128b;
wolfSSL	15:117db924cf7c	344	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[0];
wolfSSL	15:117db924cf7c	345	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[1];
wolfSSL	15:117db924cf7c	346	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[2];
wolfSSL	15:117db924cf7c	347	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[3];
wolfSSL	15:117db924cf7c	348	break;
wolfSSL	15:117db924cf7c	349
wolfSSL	15:117db924cf7c	350	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	351	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_192b;
wolfSSL	15:117db924cf7c	352	AES_CRYP_KeyInitStructure.CRYP_Key1Left = enc_key[0];
wolfSSL	15:117db924cf7c	353	AES_CRYP_KeyInitStructure.CRYP_Key1Right = enc_key[1];
wolfSSL	15:117db924cf7c	354	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[2];
wolfSSL	15:117db924cf7c	355	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[3];
wolfSSL	15:117db924cf7c	356	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[4];
wolfSSL	15:117db924cf7c	357	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[5];
wolfSSL	15:117db924cf7c	358	break;
wolfSSL	15:117db924cf7c	359
wolfSSL	15:117db924cf7c	360	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	361	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_256b;
wolfSSL	15:117db924cf7c	362	AES_CRYP_KeyInitStructure.CRYP_Key0Left = enc_key[0];
wolfSSL	15:117db924cf7c	363	AES_CRYP_KeyInitStructure.CRYP_Key0Right = enc_key[1];
wolfSSL	15:117db924cf7c	364	AES_CRYP_KeyInitStructure.CRYP_Key1Left = enc_key[2];
wolfSSL	15:117db924cf7c	365	AES_CRYP_KeyInitStructure.CRYP_Key1Right = enc_key[3];
wolfSSL	15:117db924cf7c	366	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[4];
wolfSSL	15:117db924cf7c	367	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[5];
wolfSSL	15:117db924cf7c	368	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[6];
wolfSSL	15:117db924cf7c	369	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[7];
wolfSSL	15:117db924cf7c	370	break;
wolfSSL	15:117db924cf7c	371
wolfSSL	15:117db924cf7c	372	default:
wolfSSL	15:117db924cf7c	373	break;
wolfSSL	15:117db924cf7c	374	}
wolfSSL	15:117db924cf7c	375	CRYP_KeyInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	376
wolfSSL	15:117db924cf7c	377	/* set direction, mode, and datatype */
wolfSSL	15:117db924cf7c	378	AES_CRYP_InitStructure.CRYP_AlgoDir = CRYP_AlgoDir_Encrypt;
wolfSSL	15:117db924cf7c	379	AES_CRYP_InitStructure.CRYP_AlgoMode = CRYP_AlgoMode_AES_ECB;
wolfSSL	15:117db924cf7c	380	AES_CRYP_InitStructure.CRYP_DataType = CRYP_DataType_8b;
wolfSSL	15:117db924cf7c	381	CRYP_Init(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	382
wolfSSL	15:117db924cf7c	383	/* enable crypto processor */
wolfSSL	15:117db924cf7c	384	CRYP_Cmd(ENABLE);
wolfSSL	15:117db924cf7c	385
wolfSSL	15:117db924cf7c	386	/* flush IN/OUT FIFOs */
wolfSSL	15:117db924cf7c	387	CRYP_FIFOFlush();
wolfSSL	15:117db924cf7c	388
wolfSSL	15:117db924cf7c	389	CRYP_DataIn((uint32_t)&inBlock[0]);
wolfSSL	15:117db924cf7c	390	CRYP_DataIn((uint32_t)&inBlock[4]);
wolfSSL	15:117db924cf7c	391	CRYP_DataIn((uint32_t)&inBlock[8]);
wolfSSL	15:117db924cf7c	392	CRYP_DataIn((uint32_t)&inBlock[12]);
wolfSSL	15:117db924cf7c	393
wolfSSL	15:117db924cf7c	394	/* wait until the complete message has been processed */
wolfSSL	15:117db924cf7c	395	while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
wolfSSL	15:117db924cf7c	396
wolfSSL	15:117db924cf7c	397	(uint32_t)&outBlock[0] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	398	(uint32_t)&outBlock[4] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	399	(uint32_t)&outBlock[8] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	400	(uint32_t)&outBlock[12] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	401
wolfSSL	15:117db924cf7c	402	/* disable crypto processor */
wolfSSL	15:117db924cf7c	403	CRYP_Cmd(DISABLE);
wolfSSL	15:117db924cf7c	404	#endif /* WOLFSSL_STM32_CUBEMX */
wolfSSL	15:117db924cf7c	405	return ret;
wolfSSL	15:117db924cf7c	406	}
wolfSSL	15:117db924cf7c	407	#endif /* WOLFSSL_AES_DIRECT \|\| HAVE_AESGCM \|\| HAVE_AESCCM */
wolfSSL	15:117db924cf7c	408
wolfSSL	15:117db924cf7c	409	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	410	#if defined(WOLFSSL_AES_DIRECT) \|\| defined(HAVE_AESCCM)
wolfSSL	15:117db924cf7c	411	static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	412	{
wolfSSL	15:117db924cf7c	413	int ret = 0;
wolfSSL	15:117db924cf7c	414	#ifdef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	415	CRYP_HandleTypeDef hcryp;
wolfSSL	15:117db924cf7c	416
wolfSSL	15:117db924cf7c	417	XMEMSET(&hcryp, 0, sizeof(CRYP_HandleTypeDef));
wolfSSL	15:117db924cf7c	418	switch(aes->rounds) {
wolfSSL	15:117db924cf7c	419	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	420	hcryp.Init.KeySize = CRYP_KEYSIZE_128B;
wolfSSL	15:117db924cf7c	421	break;
wolfSSL	15:117db924cf7c	422	#ifdef CRYP_KEYSIZE_192B
wolfSSL	15:117db924cf7c	423	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	424	hcryp.Init.KeySize = CRYP_KEYSIZE_192B;
wolfSSL	15:117db924cf7c	425	break;
wolfSSL	15:117db924cf7c	426	#endif
wolfSSL	15:117db924cf7c	427	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	428	hcryp.Init.KeySize = CRYP_KEYSIZE_256B;
wolfSSL	15:117db924cf7c	429	break;
wolfSSL	15:117db924cf7c	430	default:
wolfSSL	15:117db924cf7c	431	break;
wolfSSL	15:117db924cf7c	432	}
wolfSSL	15:117db924cf7c	433	hcryp.Instance = CRYP;
wolfSSL	15:117db924cf7c	434	hcryp.Init.DataType = CRYP_DATATYPE_8B;
wolfSSL	15:117db924cf7c	435	hcryp.Init.pKey = (uint8_t*)aes->key;
wolfSSL	15:117db924cf7c	436
wolfSSL	15:117db924cf7c	437	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	438
wolfSSL	15:117db924cf7c	439	if (HAL_CRYP_AESECB_Decrypt(&hcryp, (uint8_t*)inBlock, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	440	outBlock, STM32_HAL_TIMEOUT) != HAL_OK) {
wolfSSL	15:117db924cf7c	441	ret = WC_TIMEOUT_E;
wolfSSL	15:117db924cf7c	442	}
wolfSSL	15:117db924cf7c	443
wolfSSL	15:117db924cf7c	444	HAL_CRYP_DeInit(&hcryp);
wolfSSL	15:117db924cf7c	445	#else
wolfSSL	15:117db924cf7c	446	word32 *enc_key;
wolfSSL	15:117db924cf7c	447	CRYP_InitTypeDef AES_CRYP_InitStructure;
wolfSSL	15:117db924cf7c	448	CRYP_KeyInitTypeDef AES_CRYP_KeyInitStructure;
wolfSSL	15:117db924cf7c	449
wolfSSL	15:117db924cf7c	450	enc_key = aes->key;
wolfSSL	15:117db924cf7c	451
wolfSSL	15:117db924cf7c	452	/* crypto structure initialization */
wolfSSL	15:117db924cf7c	453	CRYP_KeyStructInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	454	CRYP_StructInit(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	455
wolfSSL	15:117db924cf7c	456	/* reset registers to their default values */
wolfSSL	15:117db924cf7c	457	CRYP_DeInit();
wolfSSL	15:117db924cf7c	458
wolfSSL	15:117db924cf7c	459	/* load key into correct registers */
wolfSSL	15:117db924cf7c	460	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	461	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	462	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_128b;
wolfSSL	15:117db924cf7c	463	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[0];
wolfSSL	15:117db924cf7c	464	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[1];
wolfSSL	15:117db924cf7c	465	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[2];
wolfSSL	15:117db924cf7c	466	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[3];
wolfSSL	15:117db924cf7c	467	break;
wolfSSL	15:117db924cf7c	468
wolfSSL	15:117db924cf7c	469	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	470	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_192b;
wolfSSL	15:117db924cf7c	471	AES_CRYP_KeyInitStructure.CRYP_Key1Left = enc_key[0];
wolfSSL	15:117db924cf7c	472	AES_CRYP_KeyInitStructure.CRYP_Key1Right = enc_key[1];
wolfSSL	15:117db924cf7c	473	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[2];
wolfSSL	15:117db924cf7c	474	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[3];
wolfSSL	15:117db924cf7c	475	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[4];
wolfSSL	15:117db924cf7c	476	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[5];
wolfSSL	15:117db924cf7c	477	break;
wolfSSL	15:117db924cf7c	478
wolfSSL	15:117db924cf7c	479	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	480	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_256b;
wolfSSL	15:117db924cf7c	481	AES_CRYP_KeyInitStructure.CRYP_Key0Left = enc_key[0];
wolfSSL	15:117db924cf7c	482	AES_CRYP_KeyInitStructure.CRYP_Key0Right = enc_key[1];
wolfSSL	15:117db924cf7c	483	AES_CRYP_KeyInitStructure.CRYP_Key1Left = enc_key[2];
wolfSSL	15:117db924cf7c	484	AES_CRYP_KeyInitStructure.CRYP_Key1Right = enc_key[3];
wolfSSL	15:117db924cf7c	485	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[4];
wolfSSL	15:117db924cf7c	486	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[5];
wolfSSL	15:117db924cf7c	487	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[6];
wolfSSL	15:117db924cf7c	488	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[7];
wolfSSL	15:117db924cf7c	489	break;
wolfSSL	15:117db924cf7c	490
wolfSSL	15:117db924cf7c	491	default:
wolfSSL	15:117db924cf7c	492	break;
wolfSSL	15:117db924cf7c	493	}
wolfSSL	15:117db924cf7c	494	CRYP_KeyInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	495
wolfSSL	15:117db924cf7c	496	/* set direction, key, and datatype */
wolfSSL	15:117db924cf7c	497	AES_CRYP_InitStructure.CRYP_AlgoDir = CRYP_AlgoDir_Decrypt;
wolfSSL	15:117db924cf7c	498	AES_CRYP_InitStructure.CRYP_AlgoMode = CRYP_AlgoMode_AES_Key;
wolfSSL	15:117db924cf7c	499	AES_CRYP_InitStructure.CRYP_DataType = CRYP_DataType_8b;
wolfSSL	15:117db924cf7c	500	CRYP_Init(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	501
wolfSSL	15:117db924cf7c	502	/* enable crypto processor */
wolfSSL	15:117db924cf7c	503	CRYP_Cmd(ENABLE);
wolfSSL	15:117db924cf7c	504
wolfSSL	15:117db924cf7c	505	/* wait until decrypt key has been intialized */
wolfSSL	15:117db924cf7c	506	while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
wolfSSL	15:117db924cf7c	507
wolfSSL	15:117db924cf7c	508	/* set direction, mode, and datatype */
wolfSSL	15:117db924cf7c	509	AES_CRYP_InitStructure.CRYP_AlgoDir = CRYP_AlgoDir_Decrypt;
wolfSSL	15:117db924cf7c	510	AES_CRYP_InitStructure.CRYP_AlgoMode = CRYP_AlgoMode_AES_ECB;
wolfSSL	15:117db924cf7c	511	AES_CRYP_InitStructure.CRYP_DataType = CRYP_DataType_8b;
wolfSSL	15:117db924cf7c	512	CRYP_Init(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	513
wolfSSL	15:117db924cf7c	514	/* enable crypto processor */
wolfSSL	15:117db924cf7c	515	CRYP_Cmd(ENABLE);
wolfSSL	15:117db924cf7c	516
wolfSSL	15:117db924cf7c	517	/* flush IN/OUT FIFOs */
wolfSSL	15:117db924cf7c	518	CRYP_FIFOFlush();
wolfSSL	15:117db924cf7c	519
wolfSSL	15:117db924cf7c	520	CRYP_DataIn((uint32_t)&inBlock[0]);
wolfSSL	15:117db924cf7c	521	CRYP_DataIn((uint32_t)&inBlock[4]);
wolfSSL	15:117db924cf7c	522	CRYP_DataIn((uint32_t)&inBlock[8]);
wolfSSL	15:117db924cf7c	523	CRYP_DataIn((uint32_t)&inBlock[12]);
wolfSSL	15:117db924cf7c	524
wolfSSL	15:117db924cf7c	525	/* wait until the complete message has been processed */
wolfSSL	15:117db924cf7c	526	while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
wolfSSL	15:117db924cf7c	527
wolfSSL	15:117db924cf7c	528	(uint32_t)&outBlock[0] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	529	(uint32_t)&outBlock[4] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	530	(uint32_t)&outBlock[8] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	531	(uint32_t)&outBlock[12] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	532
wolfSSL	15:117db924cf7c	533	/* disable crypto processor */
wolfSSL	15:117db924cf7c	534	CRYP_Cmd(DISABLE);
wolfSSL	15:117db924cf7c	535	#endif /* WOLFSSL_STM32_CUBEMX */
wolfSSL	15:117db924cf7c	536	return ret;
wolfSSL	15:117db924cf7c	537	}
wolfSSL	15:117db924cf7c	538	#endif /* WOLFSSL_AES_DIRECT \|\| HAVE_AESCCM */
wolfSSL	15:117db924cf7c	539	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	540
wolfSSL	15:117db924cf7c	541	#elif defined(HAVE_COLDFIRE_SEC)
wolfSSL	15:117db924cf7c	542	/* Freescale Coldfire SEC support for CBC mode.
wolfSSL	15:117db924cf7c	543	* NOTE: no support for AES-CTR/GCM/CCM/Direct */
wolfSSL	15:117db924cf7c	544	#include <wolfssl/wolfcrypt/types.h>
wolfSSL	15:117db924cf7c	545	#include "sec.h"
wolfSSL	15:117db924cf7c	546	#include "mcf5475_sec.h"
wolfSSL	15:117db924cf7c	547	#include "mcf5475_siu.h"
wolfSSL	15:117db924cf7c	548	#elif defined(FREESCALE_LTC)
wolfSSL	15:117db924cf7c	549	#include "fsl_ltc.h"
wolfSSL	15:117db924cf7c	550	#if defined(FREESCALE_LTC_AES_GCM)
wolfSSL	15:117db924cf7c	551	#undef NEED_AES_TABLES
wolfSSL	15:117db924cf7c	552	#undef GCM_TABLE
wolfSSL	15:117db924cf7c	553	#else
wolfSSL	15:117db924cf7c	554	/* if LTC doesn't have GCM, use software with LTC AES ECB mode */
wolfSSL	15:117db924cf7c	555	static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	556	{
wolfSSL	15:117db924cf7c	557	wc_AesEncryptDirect(aes, outBlock, inBlock);
wolfSSL	15:117db924cf7c	558	return 0;
wolfSSL	15:117db924cf7c	559	}
wolfSSL	15:117db924cf7c	560	static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	561	{
wolfSSL	15:117db924cf7c	562	wc_AesDecryptDirect(aes, outBlock, inBlock);
wolfSSL	15:117db924cf7c	563	return 0;
wolfSSL	15:117db924cf7c	564	}
wolfSSL	15:117db924cf7c	565	#endif
wolfSSL	15:117db924cf7c	566	#elif defined(FREESCALE_MMCAU)
wolfSSL	15:117db924cf7c	567	/* Freescale mmCAU hardware AES support for Direct, CBC, CCM, GCM modes
wolfSSL	15:117db924cf7c	568	* through the CAU/mmCAU library. Documentation located in
wolfSSL	15:117db924cf7c	569	* ColdFire/ColdFire+ CAU and Kinetis mmCAU Software Library User
wolfSSL	15:117db924cf7c	570	* Guide (See note in README). */
wolfSSL	15:117db924cf7c	571	#ifdef FREESCALE_MMCAU_CLASSIC
wolfSSL	15:117db924cf7c	572	/* MMCAU 1.4 library used with non-KSDK / classic MQX builds */
wolfSSL	15:117db924cf7c	573	#include "cau_api.h"
wolfSSL	15:117db924cf7c	574	#else
wolfSSL	15:117db924cf7c	575	#include "fsl_mmcau.h"
wolfSSL	15:117db924cf7c	576	#endif
wolfSSL	15:117db924cf7c	577
wolfSSL	15:117db924cf7c	578	static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	579	{
wolfSSL	15:117db924cf7c	580	int ret;
wolfSSL	15:117db924cf7c	581
wolfSSL	15:117db924cf7c	582	#ifdef FREESCALE_MMCAU_CLASSIC
wolfSSL	15:117db924cf7c	583	if ((wolfssl_word)outBlock % WOLFSSL_MMCAU_ALIGNMENT) {
wolfSSL	15:117db924cf7c	584	WOLFSSL_MSG("Bad cau_aes_encrypt alignment");
wolfSSL	15:117db924cf7c	585	return BAD_ALIGN_E;
wolfSSL	15:117db924cf7c	586	}
wolfSSL	15:117db924cf7c	587	#endif
wolfSSL	15:117db924cf7c	588
wolfSSL	15:117db924cf7c	589	ret = wolfSSL_CryptHwMutexLock();
wolfSSL	15:117db924cf7c	590	if(ret == 0) {
wolfSSL	15:117db924cf7c	591	#ifdef FREESCALE_MMCAU_CLASSIC
wolfSSL	15:117db924cf7c	592	cau_aes_encrypt(inBlock, (byte*)aes->key, aes->rounds, outBlock);
wolfSSL	15:117db924cf7c	593	#else
wolfSSL	15:117db924cf7c	594	MMCAU_AES_EncryptEcb(inBlock, (byte*)aes->key, aes->rounds,
wolfSSL	15:117db924cf7c	595	outBlock);
wolfSSL	15:117db924cf7c	596	#endif
wolfSSL	15:117db924cf7c	597	wolfSSL_CryptHwMutexUnLock();
wolfSSL	15:117db924cf7c	598	}
wolfSSL	15:117db924cf7c	599	return ret;
wolfSSL	15:117db924cf7c	600	}
wolfSSL	15:117db924cf7c	601	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	602	static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	603	{
wolfSSL	15:117db924cf7c	604	int ret;
wolfSSL	15:117db924cf7c	605
wolfSSL	15:117db924cf7c	606	#ifdef FREESCALE_MMCAU_CLASSIC
wolfSSL	15:117db924cf7c	607	if ((wolfssl_word)outBlock % WOLFSSL_MMCAU_ALIGNMENT) {
wolfSSL	15:117db924cf7c	608	WOLFSSL_MSG("Bad cau_aes_decrypt alignment");
wolfSSL	15:117db924cf7c	609	return BAD_ALIGN_E;
wolfSSL	15:117db924cf7c	610	}
wolfSSL	15:117db924cf7c	611	#endif
wolfSSL	15:117db924cf7c	612
wolfSSL	15:117db924cf7c	613	ret = wolfSSL_CryptHwMutexLock();
wolfSSL	15:117db924cf7c	614	if(ret == 0) {
wolfSSL	15:117db924cf7c	615	#ifdef FREESCALE_MMCAU_CLASSIC
wolfSSL	15:117db924cf7c	616	cau_aes_decrypt(inBlock, (byte*)aes->key, aes->rounds, outBlock);
wolfSSL	15:117db924cf7c	617	#else
wolfSSL	15:117db924cf7c	618	MMCAU_AES_DecryptEcb(inBlock, (byte*)aes->key, aes->rounds,
wolfSSL	15:117db924cf7c	619	outBlock);
wolfSSL	15:117db924cf7c	620	#endif
wolfSSL	15:117db924cf7c	621	wolfSSL_CryptHwMutexUnLock();
wolfSSL	15:117db924cf7c	622	}
wolfSSL	15:117db924cf7c	623	return ret;
wolfSSL	15:117db924cf7c	624	}
wolfSSL	15:117db924cf7c	625	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	626
wolfSSL	15:117db924cf7c	627	#elif defined(WOLFSSL_PIC32MZ_CRYPT)
wolfSSL	15:117db924cf7c	628
wolfSSL	15:117db924cf7c	629	#include <wolfssl/wolfcrypt/port/pic32/pic32mz-crypt.h>
wolfSSL	15:117db924cf7c	630
wolfSSL	15:117db924cf7c	631	#if defined(HAVE_AESGCM) \|\| defined(WOLFSSL_AES_DIRECT)
wolfSSL	15:117db924cf7c	632	static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	633	{
wolfSSL	15:117db924cf7c	634	return wc_Pic32AesCrypt(aes->key, aes->keylen, NULL, 0,
wolfSSL	15:117db924cf7c	635	outBlock, inBlock, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	636	PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RECB);
wolfSSL	15:117db924cf7c	637	}
wolfSSL	15:117db924cf7c	638	#endif
wolfSSL	15:117db924cf7c	639
wolfSSL	15:117db924cf7c	640	#if defined(HAVE_AES_DECRYPT) && defined(WOLFSSL_AES_DIRECT)
wolfSSL	15:117db924cf7c	641	static int wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	642	{
wolfSSL	15:117db924cf7c	643	return wc_Pic32AesCrypt(aes->key, aes->keylen, NULL, 0,
wolfSSL	15:117db924cf7c	644	outBlock, inBlock, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	645	PIC32_DECRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RECB);
wolfSSL	15:117db924cf7c	646	}
wolfSSL	15:117db924cf7c	647	#endif
wolfSSL	15:117db924cf7c	648
wolfSSL	15:117db924cf7c	649	#elif defined(WOLFSSL_NRF51_AES)
wolfSSL	15:117db924cf7c	650	/* Use built-in AES hardware - AES 128 ECB Encrypt Only */
wolfSSL	15:117db924cf7c	651	#include "wolfssl/wolfcrypt/port/nrf51.h"
wolfSSL	15:117db924cf7c	652
wolfSSL	15:117db924cf7c	653	static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	654	{
wolfSSL	15:117db924cf7c	655	return nrf51_aes_encrypt(inBlock, (byte*)aes->key, aes->rounds, outBlock);
wolfSSL	15:117db924cf7c	656	}
wolfSSL	15:117db924cf7c	657
wolfSSL	15:117db924cf7c	658	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	659	#error nRF51 AES Hardware does not support decrypt
wolfSSL	15:117db924cf7c	660	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	661
wolfSSL	15:117db924cf7c	662
wolfSSL	15:117db924cf7c	663	#elif defined(WOLFSSL_AESNI)
wolfSSL	15:117db924cf7c	664
wolfSSL	15:117db924cf7c	665	#define NEED_AES_TABLES
wolfSSL	15:117db924cf7c	666
wolfSSL	15:117db924cf7c	667	/* Each platform needs to query info type 1 from cpuid to see if aesni is
wolfSSL	15:117db924cf7c	668	* supported. Also, let's setup a macro for proper linkage w/o ABI conflicts
wolfSSL	15:117db924cf7c	669	*/
wolfSSL	15:117db924cf7c	670
wolfSSL	15:117db924cf7c	671	#ifndef AESNI_ALIGN
wolfSSL	15:117db924cf7c	672	#define AESNI_ALIGN 16
wolfSSL	15:117db924cf7c	673	#endif
wolfSSL	15:117db924cf7c	674
wolfSSL	15:117db924cf7c	675	#ifndef _MSC_VER
wolfSSL	15:117db924cf7c	676	#define XASM_LINK(f) asm(f)
wolfSSL	15:117db924cf7c	677	#else
wolfSSL	15:117db924cf7c	678	#define XASM_LINK(f)
wolfSSL	15:117db924cf7c	679	#endif /* _MSC_VER */
wolfSSL	15:117db924cf7c	680
wolfSSL	15:117db924cf7c	681	static int checkAESNI = 0;
wolfSSL	15:117db924cf7c	682	static int haveAESNI = 0;
wolfSSL	15:117db924cf7c	683	static word32 intel_flags = 0;
wolfSSL	15:117db924cf7c	684
wolfSSL	15:117db924cf7c	685	static int Check_CPU_support_AES(void)
wolfSSL	15:117db924cf7c	686	{
wolfSSL	15:117db924cf7c	687	intel_flags = cpuid_get_flags();
wolfSSL	15:117db924cf7c	688
wolfSSL	15:117db924cf7c	689	return IS_INTEL_AESNI(intel_flags) != 0;
wolfSSL	15:117db924cf7c	690	}
wolfSSL	15:117db924cf7c	691
wolfSSL	15:117db924cf7c	692
wolfSSL	15:117db924cf7c	693	/* tell C compiler these are asm functions in case any mix up of ABI underscore
wolfSSL	15:117db924cf7c	694	prefix between clang/gcc/llvm etc */
wolfSSL	15:117db924cf7c	695	#ifdef HAVE_AES_CBC
wolfSSL	15:117db924cf7c	696	void AES_CBC_encrypt(const unsigned char* in, unsigned char* out,
wolfSSL	15:117db924cf7c	697	unsigned char* ivec, unsigned long length,
wolfSSL	15:117db924cf7c	698	const unsigned char* KS, int nr)
wolfSSL	15:117db924cf7c	699	XASM_LINK("AES_CBC_encrypt");
wolfSSL	15:117db924cf7c	700
wolfSSL	15:117db924cf7c	701	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	702	#if defined(WOLFSSL_AESNI_BY4)
wolfSSL	15:117db924cf7c	703	void AES_CBC_decrypt_by4(const unsigned char* in, unsigned char* out,
wolfSSL	15:117db924cf7c	704	unsigned char* ivec, unsigned long length,
wolfSSL	15:117db924cf7c	705	const unsigned char* KS, int nr)
wolfSSL	15:117db924cf7c	706	XASM_LINK("AES_CBC_decrypt_by4");
wolfSSL	15:117db924cf7c	707	#elif defined(WOLFSSL_AESNI_BY6)
wolfSSL	15:117db924cf7c	708	void AES_CBC_decrypt_by6(const unsigned char* in, unsigned char* out,
wolfSSL	15:117db924cf7c	709	unsigned char* ivec, unsigned long length,
wolfSSL	15:117db924cf7c	710	const unsigned char* KS, int nr)
wolfSSL	15:117db924cf7c	711	XASM_LINK("AES_CBC_decrypt_by6");
wolfSSL	15:117db924cf7c	712	#else /* WOLFSSL_AESNI_BYx */
wolfSSL	15:117db924cf7c	713	void AES_CBC_decrypt_by8(const unsigned char* in, unsigned char* out,
wolfSSL	15:117db924cf7c	714	unsigned char* ivec, unsigned long length,
wolfSSL	15:117db924cf7c	715	const unsigned char* KS, int nr)
wolfSSL	15:117db924cf7c	716	XASM_LINK("AES_CBC_decrypt_by8");
wolfSSL	15:117db924cf7c	717	#endif /* WOLFSSL_AESNI_BYx */
wolfSSL	15:117db924cf7c	718	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	719	#endif /* HAVE_AES_CBC */
wolfSSL	15:117db924cf7c	720
wolfSSL	15:117db924cf7c	721	void AES_ECB_encrypt(const unsigned char* in, unsigned char* out,
wolfSSL	15:117db924cf7c	722	unsigned long length, const unsigned char* KS, int nr)
wolfSSL	15:117db924cf7c	723	XASM_LINK("AES_ECB_encrypt");
wolfSSL	15:117db924cf7c	724
wolfSSL	15:117db924cf7c	725	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	726	void AES_ECB_decrypt(const unsigned char* in, unsigned char* out,
wolfSSL	15:117db924cf7c	727	unsigned long length, const unsigned char* KS, int nr)
wolfSSL	15:117db924cf7c	728	XASM_LINK("AES_ECB_decrypt");
wolfSSL	15:117db924cf7c	729	#endif
wolfSSL	15:117db924cf7c	730
wolfSSL	15:117db924cf7c	731	void AES_128_Key_Expansion(const unsigned char* userkey,
wolfSSL	15:117db924cf7c	732	unsigned char* key_schedule)
wolfSSL	15:117db924cf7c	733	XASM_LINK("AES_128_Key_Expansion");
wolfSSL	15:117db924cf7c	734
wolfSSL	15:117db924cf7c	735	void AES_192_Key_Expansion(const unsigned char* userkey,
wolfSSL	15:117db924cf7c	736	unsigned char* key_schedule)
wolfSSL	15:117db924cf7c	737	XASM_LINK("AES_192_Key_Expansion");
wolfSSL	15:117db924cf7c	738
wolfSSL	15:117db924cf7c	739	void AES_256_Key_Expansion(const unsigned char* userkey,
wolfSSL	15:117db924cf7c	740	unsigned char* key_schedule)
wolfSSL	15:117db924cf7c	741	XASM_LINK("AES_256_Key_Expansion");
wolfSSL	15:117db924cf7c	742
wolfSSL	15:117db924cf7c	743
wolfSSL	15:117db924cf7c	744	static int AES_set_encrypt_key(const unsigned char *userKey, const int bits,
wolfSSL	15:117db924cf7c	745	Aes* aes)
wolfSSL	15:117db924cf7c	746	{
wolfSSL	15:117db924cf7c	747	int ret;
wolfSSL	15:117db924cf7c	748
wolfSSL	15:117db924cf7c	749	if (!userKey \|\| !aes)
wolfSSL	15:117db924cf7c	750	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	751
wolfSSL	15:117db924cf7c	752	switch (bits) {
wolfSSL	15:117db924cf7c	753	case 128:
wolfSSL	15:117db924cf7c	754	AES_128_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 10;
wolfSSL	15:117db924cf7c	755	return 0;
wolfSSL	15:117db924cf7c	756	case 192:
wolfSSL	15:117db924cf7c	757	AES_192_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 12;
wolfSSL	15:117db924cf7c	758	return 0;
wolfSSL	15:117db924cf7c	759	case 256:
wolfSSL	15:117db924cf7c	760	AES_256_Key_Expansion (userKey,(byte*)aes->key); aes->rounds = 14;
wolfSSL	15:117db924cf7c	761	return 0;
wolfSSL	15:117db924cf7c	762	default:
wolfSSL	15:117db924cf7c	763	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	764	}
wolfSSL	15:117db924cf7c	765
wolfSSL	15:117db924cf7c	766	return ret;
wolfSSL	15:117db924cf7c	767	}
wolfSSL	15:117db924cf7c	768
wolfSSL	15:117db924cf7c	769	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	770	static int AES_set_decrypt_key(const unsigned char* userKey,
wolfSSL	15:117db924cf7c	771	const int bits, Aes* aes)
wolfSSL	15:117db924cf7c	772	{
wolfSSL	15:117db924cf7c	773	int nr;
wolfSSL	15:117db924cf7c	774	Aes temp_key;
wolfSSL	15:117db924cf7c	775	__m128i Key_Schedule = (__m128i)aes->key;
wolfSSL	15:117db924cf7c	776	__m128i Temp_Key_Schedule = (__m128i)temp_key.key;
wolfSSL	15:117db924cf7c	777
wolfSSL	15:117db924cf7c	778	if (!userKey \|\| !aes)
wolfSSL	15:117db924cf7c	779	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	780
wolfSSL	15:117db924cf7c	781	if (AES_set_encrypt_key(userKey,bits,&temp_key) == BAD_FUNC_ARG)
wolfSSL	15:117db924cf7c	782	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	783
wolfSSL	15:117db924cf7c	784	nr = temp_key.rounds;
wolfSSL	15:117db924cf7c	785	aes->rounds = nr;
wolfSSL	15:117db924cf7c	786
wolfSSL	15:117db924cf7c	787	Key_Schedule[nr] = Temp_Key_Schedule[0];
wolfSSL	15:117db924cf7c	788	Key_Schedule[nr-1] = _mm_aesimc_si128(Temp_Key_Schedule[1]);
wolfSSL	15:117db924cf7c	789	Key_Schedule[nr-2] = _mm_aesimc_si128(Temp_Key_Schedule[2]);
wolfSSL	15:117db924cf7c	790	Key_Schedule[nr-3] = _mm_aesimc_si128(Temp_Key_Schedule[3]);
wolfSSL	15:117db924cf7c	791	Key_Schedule[nr-4] = _mm_aesimc_si128(Temp_Key_Schedule[4]);
wolfSSL	15:117db924cf7c	792	Key_Schedule[nr-5] = _mm_aesimc_si128(Temp_Key_Schedule[5]);
wolfSSL	15:117db924cf7c	793	Key_Schedule[nr-6] = _mm_aesimc_si128(Temp_Key_Schedule[6]);
wolfSSL	15:117db924cf7c	794	Key_Schedule[nr-7] = _mm_aesimc_si128(Temp_Key_Schedule[7]);
wolfSSL	15:117db924cf7c	795	Key_Schedule[nr-8] = _mm_aesimc_si128(Temp_Key_Schedule[8]);
wolfSSL	15:117db924cf7c	796	Key_Schedule[nr-9] = _mm_aesimc_si128(Temp_Key_Schedule[9]);
wolfSSL	15:117db924cf7c	797
wolfSSL	15:117db924cf7c	798	if (nr>10) {
wolfSSL	15:117db924cf7c	799	Key_Schedule[nr-10] = _mm_aesimc_si128(Temp_Key_Schedule[10]);
wolfSSL	15:117db924cf7c	800	Key_Schedule[nr-11] = _mm_aesimc_si128(Temp_Key_Schedule[11]);
wolfSSL	15:117db924cf7c	801	}
wolfSSL	15:117db924cf7c	802
wolfSSL	15:117db924cf7c	803	if (nr>12) {
wolfSSL	15:117db924cf7c	804	Key_Schedule[nr-12] = _mm_aesimc_si128(Temp_Key_Schedule[12]);
wolfSSL	15:117db924cf7c	805	Key_Schedule[nr-13] = _mm_aesimc_si128(Temp_Key_Schedule[13]);
wolfSSL	15:117db924cf7c	806	}
wolfSSL	15:117db924cf7c	807
wolfSSL	15:117db924cf7c	808	Key_Schedule[0] = Temp_Key_Schedule[nr];
wolfSSL	15:117db924cf7c	809
wolfSSL	15:117db924cf7c	810	return 0;
wolfSSL	15:117db924cf7c	811	}
wolfSSL	15:117db924cf7c	812	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	813
wolfSSL	15:117db924cf7c	814	#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES)
wolfSSL	15:117db924cf7c	815	static int wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	816	{
wolfSSL	15:117db924cf7c	817	wc_AesEncryptDirect(aes, outBlock, inBlock);
wolfSSL	15:117db924cf7c	818	return 0;
wolfSSL	15:117db924cf7c	819	}
wolfSSL	15:117db924cf7c	820	#else
wolfSSL	15:117db924cf7c	821
wolfSSL	15:117db924cf7c	822	/* using wolfCrypt software AES implementation */
wolfSSL	15:117db924cf7c	823	#define NEED_AES_TABLES
wolfSSL	15:117db924cf7c	824	#endif
wolfSSL	15:117db924cf7c	825
wolfSSL	15:117db924cf7c	826
wolfSSL	15:117db924cf7c	827
wolfSSL	15:117db924cf7c	828	#ifdef NEED_AES_TABLES
wolfSSL	15:117db924cf7c	829
wolfSSL	15:117db924cf7c	830	static const word32 rcon[] = {
wolfSSL	15:117db924cf7c	831	0x01000000, 0x02000000, 0x04000000, 0x08000000,
wolfSSL	15:117db924cf7c	832	0x10000000, 0x20000000, 0x40000000, 0x80000000,
wolfSSL	15:117db924cf7c	833	0x1B000000, 0x36000000,
wolfSSL	15:117db924cf7c	834	/* for 128-bit blocks, Rijndael never uses more than 10 rcon values */
wolfSSL	15:117db924cf7c	835	};
wolfSSL	15:117db924cf7c	836
wolfSSL	15:117db924cf7c	837	static const word32 Te[4][256] = {
wolfSSL	15:117db924cf7c	838	{
wolfSSL	15:117db924cf7c	839	0xc66363a5U, 0xf87c7c84U, 0xee777799U, 0xf67b7b8dU,
wolfSSL	15:117db924cf7c	840	0xfff2f20dU, 0xd66b6bbdU, 0xde6f6fb1U, 0x91c5c554U,
wolfSSL	15:117db924cf7c	841	0x60303050U, 0x02010103U, 0xce6767a9U, 0x562b2b7dU,
wolfSSL	15:117db924cf7c	842	0xe7fefe19U, 0xb5d7d762U, 0x4dababe6U, 0xec76769aU,
wolfSSL	15:117db924cf7c	843	0x8fcaca45U, 0x1f82829dU, 0x89c9c940U, 0xfa7d7d87U,
wolfSSL	15:117db924cf7c	844	0xeffafa15U, 0xb25959ebU, 0x8e4747c9U, 0xfbf0f00bU,
wolfSSL	15:117db924cf7c	845	0x41adadecU, 0xb3d4d467U, 0x5fa2a2fdU, 0x45afafeaU,
wolfSSL	15:117db924cf7c	846	0x239c9cbfU, 0x53a4a4f7U, 0xe4727296U, 0x9bc0c05bU,
wolfSSL	15:117db924cf7c	847	0x75b7b7c2U, 0xe1fdfd1cU, 0x3d9393aeU, 0x4c26266aU,
wolfSSL	15:117db924cf7c	848	0x6c36365aU, 0x7e3f3f41U, 0xf5f7f702U, 0x83cccc4fU,
wolfSSL	15:117db924cf7c	849	0x6834345cU, 0x51a5a5f4U, 0xd1e5e534U, 0xf9f1f108U,
wolfSSL	15:117db924cf7c	850	0xe2717193U, 0xabd8d873U, 0x62313153U, 0x2a15153fU,
wolfSSL	15:117db924cf7c	851	0x0804040cU, 0x95c7c752U, 0x46232365U, 0x9dc3c35eU,
wolfSSL	15:117db924cf7c	852	0x30181828U, 0x379696a1U, 0x0a05050fU, 0x2f9a9ab5U,
wolfSSL	15:117db924cf7c	853	0x0e070709U, 0x24121236U, 0x1b80809bU, 0xdfe2e23dU,
wolfSSL	15:117db924cf7c	854	0xcdebeb26U, 0x4e272769U, 0x7fb2b2cdU, 0xea75759fU,
wolfSSL	15:117db924cf7c	855	0x1209091bU, 0x1d83839eU, 0x582c2c74U, 0x341a1a2eU,
wolfSSL	15:117db924cf7c	856	0x361b1b2dU, 0xdc6e6eb2U, 0xb45a5aeeU, 0x5ba0a0fbU,
wolfSSL	15:117db924cf7c	857	0xa45252f6U, 0x763b3b4dU, 0xb7d6d661U, 0x7db3b3ceU,
wolfSSL	15:117db924cf7c	858	0x5229297bU, 0xdde3e33eU, 0x5e2f2f71U, 0x13848497U,
wolfSSL	15:117db924cf7c	859	0xa65353f5U, 0xb9d1d168U, 0x00000000U, 0xc1eded2cU,
wolfSSL	15:117db924cf7c	860	0x40202060U, 0xe3fcfc1fU, 0x79b1b1c8U, 0xb65b5bedU,
wolfSSL	15:117db924cf7c	861	0xd46a6abeU, 0x8dcbcb46U, 0x67bebed9U, 0x7239394bU,
wolfSSL	15:117db924cf7c	862	0x944a4adeU, 0x984c4cd4U, 0xb05858e8U, 0x85cfcf4aU,
wolfSSL	15:117db924cf7c	863	0xbbd0d06bU, 0xc5efef2aU, 0x4faaaae5U, 0xedfbfb16U,
wolfSSL	15:117db924cf7c	864	0x864343c5U, 0x9a4d4dd7U, 0x66333355U, 0x11858594U,
wolfSSL	15:117db924cf7c	865	0x8a4545cfU, 0xe9f9f910U, 0x04020206U, 0xfe7f7f81U,
wolfSSL	15:117db924cf7c	866	0xa05050f0U, 0x783c3c44U, 0x259f9fbaU, 0x4ba8a8e3U,
wolfSSL	15:117db924cf7c	867	0xa25151f3U, 0x5da3a3feU, 0x804040c0U, 0x058f8f8aU,
wolfSSL	15:117db924cf7c	868	0x3f9292adU, 0x219d9dbcU, 0x70383848U, 0xf1f5f504U,
wolfSSL	15:117db924cf7c	869	0x63bcbcdfU, 0x77b6b6c1U, 0xafdada75U, 0x42212163U,
wolfSSL	15:117db924cf7c	870	0x20101030U, 0xe5ffff1aU, 0xfdf3f30eU, 0xbfd2d26dU,
wolfSSL	15:117db924cf7c	871	0x81cdcd4cU, 0x180c0c14U, 0x26131335U, 0xc3ecec2fU,
wolfSSL	15:117db924cf7c	872	0xbe5f5fe1U, 0x359797a2U, 0x884444ccU, 0x2e171739U,
wolfSSL	15:117db924cf7c	873	0x93c4c457U, 0x55a7a7f2U, 0xfc7e7e82U, 0x7a3d3d47U,
wolfSSL	15:117db924cf7c	874	0xc86464acU, 0xba5d5de7U, 0x3219192bU, 0xe6737395U,
wolfSSL	15:117db924cf7c	875	0xc06060a0U, 0x19818198U, 0x9e4f4fd1U, 0xa3dcdc7fU,
wolfSSL	15:117db924cf7c	876	0x44222266U, 0x542a2a7eU, 0x3b9090abU, 0x0b888883U,
wolfSSL	15:117db924cf7c	877	0x8c4646caU, 0xc7eeee29U, 0x6bb8b8d3U, 0x2814143cU,
wolfSSL	15:117db924cf7c	878	0xa7dede79U, 0xbc5e5ee2U, 0x160b0b1dU, 0xaddbdb76U,
wolfSSL	15:117db924cf7c	879	0xdbe0e03bU, 0x64323256U, 0x743a3a4eU, 0x140a0a1eU,
wolfSSL	15:117db924cf7c	880	0x924949dbU, 0x0c06060aU, 0x4824246cU, 0xb85c5ce4U,
wolfSSL	15:117db924cf7c	881	0x9fc2c25dU, 0xbdd3d36eU, 0x43acacefU, 0xc46262a6U,
wolfSSL	15:117db924cf7c	882	0x399191a8U, 0x319595a4U, 0xd3e4e437U, 0xf279798bU,
wolfSSL	15:117db924cf7c	883	0xd5e7e732U, 0x8bc8c843U, 0x6e373759U, 0xda6d6db7U,
wolfSSL	15:117db924cf7c	884	0x018d8d8cU, 0xb1d5d564U, 0x9c4e4ed2U, 0x49a9a9e0U,
wolfSSL	15:117db924cf7c	885	0xd86c6cb4U, 0xac5656faU, 0xf3f4f407U, 0xcfeaea25U,
wolfSSL	15:117db924cf7c	886	0xca6565afU, 0xf47a7a8eU, 0x47aeaee9U, 0x10080818U,
wolfSSL	15:117db924cf7c	887	0x6fbabad5U, 0xf0787888U, 0x4a25256fU, 0x5c2e2e72U,
wolfSSL	15:117db924cf7c	888	0x381c1c24U, 0x57a6a6f1U, 0x73b4b4c7U, 0x97c6c651U,
wolfSSL	15:117db924cf7c	889	0xcbe8e823U, 0xa1dddd7cU, 0xe874749cU, 0x3e1f1f21U,
wolfSSL	15:117db924cf7c	890	0x964b4bddU, 0x61bdbddcU, 0x0d8b8b86U, 0x0f8a8a85U,
wolfSSL	15:117db924cf7c	891	0xe0707090U, 0x7c3e3e42U, 0x71b5b5c4U, 0xcc6666aaU,
wolfSSL	15:117db924cf7c	892	0x904848d8U, 0x06030305U, 0xf7f6f601U, 0x1c0e0e12U,
wolfSSL	15:117db924cf7c	893	0xc26161a3U, 0x6a35355fU, 0xae5757f9U, 0x69b9b9d0U,
wolfSSL	15:117db924cf7c	894	0x17868691U, 0x99c1c158U, 0x3a1d1d27U, 0x279e9eb9U,
wolfSSL	15:117db924cf7c	895	0xd9e1e138U, 0xebf8f813U, 0x2b9898b3U, 0x22111133U,
wolfSSL	15:117db924cf7c	896	0xd26969bbU, 0xa9d9d970U, 0x078e8e89U, 0x339494a7U,
wolfSSL	15:117db924cf7c	897	0x2d9b9bb6U, 0x3c1e1e22U, 0x15878792U, 0xc9e9e920U,
wolfSSL	15:117db924cf7c	898	0x87cece49U, 0xaa5555ffU, 0x50282878U, 0xa5dfdf7aU,
wolfSSL	15:117db924cf7c	899	0x038c8c8fU, 0x59a1a1f8U, 0x09898980U, 0x1a0d0d17U,
wolfSSL	15:117db924cf7c	900	0x65bfbfdaU, 0xd7e6e631U, 0x844242c6U, 0xd06868b8U,
wolfSSL	15:117db924cf7c	901	0x824141c3U, 0x299999b0U, 0x5a2d2d77U, 0x1e0f0f11U,
wolfSSL	15:117db924cf7c	902	0x7bb0b0cbU, 0xa85454fcU, 0x6dbbbbd6U, 0x2c16163aU,
wolfSSL	15:117db924cf7c	903	},
wolfSSL	15:117db924cf7c	904	{
wolfSSL	15:117db924cf7c	905	0xa5c66363U, 0x84f87c7cU, 0x99ee7777U, 0x8df67b7bU,
wolfSSL	15:117db924cf7c	906	0x0dfff2f2U, 0xbdd66b6bU, 0xb1de6f6fU, 0x5491c5c5U,
wolfSSL	15:117db924cf7c	907	0x50603030U, 0x03020101U, 0xa9ce6767U, 0x7d562b2bU,
wolfSSL	15:117db924cf7c	908	0x19e7fefeU, 0x62b5d7d7U, 0xe64dababU, 0x9aec7676U,
wolfSSL	15:117db924cf7c	909	0x458fcacaU, 0x9d1f8282U, 0x4089c9c9U, 0x87fa7d7dU,
wolfSSL	15:117db924cf7c	910	0x15effafaU, 0xebb25959U, 0xc98e4747U, 0x0bfbf0f0U,
wolfSSL	15:117db924cf7c	911	0xec41adadU, 0x67b3d4d4U, 0xfd5fa2a2U, 0xea45afafU,
wolfSSL	15:117db924cf7c	912	0xbf239c9cU, 0xf753a4a4U, 0x96e47272U, 0x5b9bc0c0U,
wolfSSL	15:117db924cf7c	913	0xc275b7b7U, 0x1ce1fdfdU, 0xae3d9393U, 0x6a4c2626U,
wolfSSL	15:117db924cf7c	914	0x5a6c3636U, 0x417e3f3fU, 0x02f5f7f7U, 0x4f83ccccU,
wolfSSL	15:117db924cf7c	915	0x5c683434U, 0xf451a5a5U, 0x34d1e5e5U, 0x08f9f1f1U,
wolfSSL	15:117db924cf7c	916	0x93e27171U, 0x73abd8d8U, 0x53623131U, 0x3f2a1515U,
wolfSSL	15:117db924cf7c	917	0x0c080404U, 0x5295c7c7U, 0x65462323U, 0x5e9dc3c3U,
wolfSSL	15:117db924cf7c	918	0x28301818U, 0xa1379696U, 0x0f0a0505U, 0xb52f9a9aU,
wolfSSL	15:117db924cf7c	919	0x090e0707U, 0x36241212U, 0x9b1b8080U, 0x3ddfe2e2U,
wolfSSL	15:117db924cf7c	920	0x26cdebebU, 0x694e2727U, 0xcd7fb2b2U, 0x9fea7575U,
wolfSSL	15:117db924cf7c	921	0x1b120909U, 0x9e1d8383U, 0x74582c2cU, 0x2e341a1aU,
wolfSSL	15:117db924cf7c	922	0x2d361b1bU, 0xb2dc6e6eU, 0xeeb45a5aU, 0xfb5ba0a0U,
wolfSSL	15:117db924cf7c	923	0xf6a45252U, 0x4d763b3bU, 0x61b7d6d6U, 0xce7db3b3U,
wolfSSL	15:117db924cf7c	924	0x7b522929U, 0x3edde3e3U, 0x715e2f2fU, 0x97138484U,
wolfSSL	15:117db924cf7c	925	0xf5a65353U, 0x68b9d1d1U, 0x00000000U, 0x2cc1ededU,
wolfSSL	15:117db924cf7c	926	0x60402020U, 0x1fe3fcfcU, 0xc879b1b1U, 0xedb65b5bU,
wolfSSL	15:117db924cf7c	927	0xbed46a6aU, 0x468dcbcbU, 0xd967bebeU, 0x4b723939U,
wolfSSL	15:117db924cf7c	928	0xde944a4aU, 0xd4984c4cU, 0xe8b05858U, 0x4a85cfcfU,
wolfSSL	15:117db924cf7c	929	0x6bbbd0d0U, 0x2ac5efefU, 0xe54faaaaU, 0x16edfbfbU,
wolfSSL	15:117db924cf7c	930	0xc5864343U, 0xd79a4d4dU, 0x55663333U, 0x94118585U,
wolfSSL	15:117db924cf7c	931	0xcf8a4545U, 0x10e9f9f9U, 0x06040202U, 0x81fe7f7fU,
wolfSSL	15:117db924cf7c	932	0xf0a05050U, 0x44783c3cU, 0xba259f9fU, 0xe34ba8a8U,
wolfSSL	15:117db924cf7c	933	0xf3a25151U, 0xfe5da3a3U, 0xc0804040U, 0x8a058f8fU,
wolfSSL	15:117db924cf7c	934	0xad3f9292U, 0xbc219d9dU, 0x48703838U, 0x04f1f5f5U,
wolfSSL	15:117db924cf7c	935	0xdf63bcbcU, 0xc177b6b6U, 0x75afdadaU, 0x63422121U,
wolfSSL	15:117db924cf7c	936	0x30201010U, 0x1ae5ffffU, 0x0efdf3f3U, 0x6dbfd2d2U,
wolfSSL	15:117db924cf7c	937	0x4c81cdcdU, 0x14180c0cU, 0x35261313U, 0x2fc3ececU,
wolfSSL	15:117db924cf7c	938	0xe1be5f5fU, 0xa2359797U, 0xcc884444U, 0x392e1717U,
wolfSSL	15:117db924cf7c	939	0x5793c4c4U, 0xf255a7a7U, 0x82fc7e7eU, 0x477a3d3dU,
wolfSSL	15:117db924cf7c	940	0xacc86464U, 0xe7ba5d5dU, 0x2b321919U, 0x95e67373U,
wolfSSL	15:117db924cf7c	941	0xa0c06060U, 0x98198181U, 0xd19e4f4fU, 0x7fa3dcdcU,
wolfSSL	15:117db924cf7c	942	0x66442222U, 0x7e542a2aU, 0xab3b9090U, 0x830b8888U,
wolfSSL	15:117db924cf7c	943	0xca8c4646U, 0x29c7eeeeU, 0xd36bb8b8U, 0x3c281414U,
wolfSSL	15:117db924cf7c	944	0x79a7dedeU, 0xe2bc5e5eU, 0x1d160b0bU, 0x76addbdbU,
wolfSSL	15:117db924cf7c	945	0x3bdbe0e0U, 0x56643232U, 0x4e743a3aU, 0x1e140a0aU,
wolfSSL	15:117db924cf7c	946	0xdb924949U, 0x0a0c0606U, 0x6c482424U, 0xe4b85c5cU,
wolfSSL	15:117db924cf7c	947	0x5d9fc2c2U, 0x6ebdd3d3U, 0xef43acacU, 0xa6c46262U,
wolfSSL	15:117db924cf7c	948	0xa8399191U, 0xa4319595U, 0x37d3e4e4U, 0x8bf27979U,
wolfSSL	15:117db924cf7c	949	0x32d5e7e7U, 0x438bc8c8U, 0x596e3737U, 0xb7da6d6dU,
wolfSSL	15:117db924cf7c	950	0x8c018d8dU, 0x64b1d5d5U, 0xd29c4e4eU, 0xe049a9a9U,
wolfSSL	15:117db924cf7c	951	0xb4d86c6cU, 0xfaac5656U, 0x07f3f4f4U, 0x25cfeaeaU,
wolfSSL	15:117db924cf7c	952	0xafca6565U, 0x8ef47a7aU, 0xe947aeaeU, 0x18100808U,
wolfSSL	15:117db924cf7c	953	0xd56fbabaU, 0x88f07878U, 0x6f4a2525U, 0x725c2e2eU,
wolfSSL	15:117db924cf7c	954	0x24381c1cU, 0xf157a6a6U, 0xc773b4b4U, 0x5197c6c6U,
wolfSSL	15:117db924cf7c	955	0x23cbe8e8U, 0x7ca1ddddU, 0x9ce87474U, 0x213e1f1fU,
wolfSSL	15:117db924cf7c	956	0xdd964b4bU, 0xdc61bdbdU, 0x860d8b8bU, 0x850f8a8aU,
wolfSSL	15:117db924cf7c	957	0x90e07070U, 0x427c3e3eU, 0xc471b5b5U, 0xaacc6666U,
wolfSSL	15:117db924cf7c	958	0xd8904848U, 0x05060303U, 0x01f7f6f6U, 0x121c0e0eU,
wolfSSL	15:117db924cf7c	959	0xa3c26161U, 0x5f6a3535U, 0xf9ae5757U, 0xd069b9b9U,
wolfSSL	15:117db924cf7c	960	0x91178686U, 0x5899c1c1U, 0x273a1d1dU, 0xb9279e9eU,
wolfSSL	15:117db924cf7c	961	0x38d9e1e1U, 0x13ebf8f8U, 0xb32b9898U, 0x33221111U,
wolfSSL	15:117db924cf7c	962	0xbbd26969U, 0x70a9d9d9U, 0x89078e8eU, 0xa7339494U,
wolfSSL	15:117db924cf7c	963	0xb62d9b9bU, 0x223c1e1eU, 0x92158787U, 0x20c9e9e9U,
wolfSSL	15:117db924cf7c	964	0x4987ceceU, 0xffaa5555U, 0x78502828U, 0x7aa5dfdfU,
wolfSSL	15:117db924cf7c	965	0x8f038c8cU, 0xf859a1a1U, 0x80098989U, 0x171a0d0dU,
wolfSSL	15:117db924cf7c	966	0xda65bfbfU, 0x31d7e6e6U, 0xc6844242U, 0xb8d06868U,
wolfSSL	15:117db924cf7c	967	0xc3824141U, 0xb0299999U, 0x775a2d2dU, 0x111e0f0fU,
wolfSSL	15:117db924cf7c	968	0xcb7bb0b0U, 0xfca85454U, 0xd66dbbbbU, 0x3a2c1616U,
wolfSSL	15:117db924cf7c	969	},
wolfSSL	15:117db924cf7c	970	{
wolfSSL	15:117db924cf7c	971	0x63a5c663U, 0x7c84f87cU, 0x7799ee77U, 0x7b8df67bU,
wolfSSL	15:117db924cf7c	972	0xf20dfff2U, 0x6bbdd66bU, 0x6fb1de6fU, 0xc55491c5U,
wolfSSL	15:117db924cf7c	973	0x30506030U, 0x01030201U, 0x67a9ce67U, 0x2b7d562bU,
wolfSSL	15:117db924cf7c	974	0xfe19e7feU, 0xd762b5d7U, 0xabe64dabU, 0x769aec76U,
wolfSSL	15:117db924cf7c	975	0xca458fcaU, 0x829d1f82U, 0xc94089c9U, 0x7d87fa7dU,
wolfSSL	15:117db924cf7c	976	0xfa15effaU, 0x59ebb259U, 0x47c98e47U, 0xf00bfbf0U,
wolfSSL	15:117db924cf7c	977	0xadec41adU, 0xd467b3d4U, 0xa2fd5fa2U, 0xafea45afU,
wolfSSL	15:117db924cf7c	978	0x9cbf239cU, 0xa4f753a4U, 0x7296e472U, 0xc05b9bc0U,
wolfSSL	15:117db924cf7c	979	0xb7c275b7U, 0xfd1ce1fdU, 0x93ae3d93U, 0x266a4c26U,
wolfSSL	15:117db924cf7c	980	0x365a6c36U, 0x3f417e3fU, 0xf702f5f7U, 0xcc4f83ccU,
wolfSSL	15:117db924cf7c	981	0x345c6834U, 0xa5f451a5U, 0xe534d1e5U, 0xf108f9f1U,
wolfSSL	15:117db924cf7c	982	0x7193e271U, 0xd873abd8U, 0x31536231U, 0x153f2a15U,
wolfSSL	15:117db924cf7c	983	0x040c0804U, 0xc75295c7U, 0x23654623U, 0xc35e9dc3U,
wolfSSL	15:117db924cf7c	984	0x18283018U, 0x96a13796U, 0x050f0a05U, 0x9ab52f9aU,
wolfSSL	15:117db924cf7c	985	0x07090e07U, 0x12362412U, 0x809b1b80U, 0xe23ddfe2U,
wolfSSL	15:117db924cf7c	986	0xeb26cdebU, 0x27694e27U, 0xb2cd7fb2U, 0x759fea75U,
wolfSSL	15:117db924cf7c	987	0x091b1209U, 0x839e1d83U, 0x2c74582cU, 0x1a2e341aU,
wolfSSL	15:117db924cf7c	988	0x1b2d361bU, 0x6eb2dc6eU, 0x5aeeb45aU, 0xa0fb5ba0U,
wolfSSL	15:117db924cf7c	989	0x52f6a452U, 0x3b4d763bU, 0xd661b7d6U, 0xb3ce7db3U,
wolfSSL	15:117db924cf7c	990	0x297b5229U, 0xe33edde3U, 0x2f715e2fU, 0x84971384U,
wolfSSL	15:117db924cf7c	991	0x53f5a653U, 0xd168b9d1U, 0x00000000U, 0xed2cc1edU,
wolfSSL	15:117db924cf7c	992	0x20604020U, 0xfc1fe3fcU, 0xb1c879b1U, 0x5bedb65bU,
wolfSSL	15:117db924cf7c	993	0x6abed46aU, 0xcb468dcbU, 0xbed967beU, 0x394b7239U,
wolfSSL	15:117db924cf7c	994	0x4ade944aU, 0x4cd4984cU, 0x58e8b058U, 0xcf4a85cfU,
wolfSSL	15:117db924cf7c	995	0xd06bbbd0U, 0xef2ac5efU, 0xaae54faaU, 0xfb16edfbU,
wolfSSL	15:117db924cf7c	996	0x43c58643U, 0x4dd79a4dU, 0x33556633U, 0x85941185U,
wolfSSL	15:117db924cf7c	997	0x45cf8a45U, 0xf910e9f9U, 0x02060402U, 0x7f81fe7fU,
wolfSSL	15:117db924cf7c	998	0x50f0a050U, 0x3c44783cU, 0x9fba259fU, 0xa8e34ba8U,
wolfSSL	15:117db924cf7c	999	0x51f3a251U, 0xa3fe5da3U, 0x40c08040U, 0x8f8a058fU,
wolfSSL	15:117db924cf7c	1000	0x92ad3f92U, 0x9dbc219dU, 0x38487038U, 0xf504f1f5U,
wolfSSL	15:117db924cf7c	1001	0xbcdf63bcU, 0xb6c177b6U, 0xda75afdaU, 0x21634221U,
wolfSSL	15:117db924cf7c	1002	0x10302010U, 0xff1ae5ffU, 0xf30efdf3U, 0xd26dbfd2U,
wolfSSL	15:117db924cf7c	1003	0xcd4c81cdU, 0x0c14180cU, 0x13352613U, 0xec2fc3ecU,
wolfSSL	15:117db924cf7c	1004	0x5fe1be5fU, 0x97a23597U, 0x44cc8844U, 0x17392e17U,
wolfSSL	15:117db924cf7c	1005	0xc45793c4U, 0xa7f255a7U, 0x7e82fc7eU, 0x3d477a3dU,
wolfSSL	15:117db924cf7c	1006	0x64acc864U, 0x5de7ba5dU, 0x192b3219U, 0x7395e673U,
wolfSSL	15:117db924cf7c	1007	0x60a0c060U, 0x81981981U, 0x4fd19e4fU, 0xdc7fa3dcU,
wolfSSL	15:117db924cf7c	1008	0x22664422U, 0x2a7e542aU, 0x90ab3b90U, 0x88830b88U,
wolfSSL	15:117db924cf7c	1009	0x46ca8c46U, 0xee29c7eeU, 0xb8d36bb8U, 0x143c2814U,
wolfSSL	15:117db924cf7c	1010	0xde79a7deU, 0x5ee2bc5eU, 0x0b1d160bU, 0xdb76addbU,
wolfSSL	15:117db924cf7c	1011	0xe03bdbe0U, 0x32566432U, 0x3a4e743aU, 0x0a1e140aU,
wolfSSL	15:117db924cf7c	1012	0x49db9249U, 0x060a0c06U, 0x246c4824U, 0x5ce4b85cU,
wolfSSL	15:117db924cf7c	1013	0xc25d9fc2U, 0xd36ebdd3U, 0xacef43acU, 0x62a6c462U,
wolfSSL	15:117db924cf7c	1014	0x91a83991U, 0x95a43195U, 0xe437d3e4U, 0x798bf279U,
wolfSSL	15:117db924cf7c	1015	0xe732d5e7U, 0xc8438bc8U, 0x37596e37U, 0x6db7da6dU,
wolfSSL	15:117db924cf7c	1016	0x8d8c018dU, 0xd564b1d5U, 0x4ed29c4eU, 0xa9e049a9U,
wolfSSL	15:117db924cf7c	1017	0x6cb4d86cU, 0x56faac56U, 0xf407f3f4U, 0xea25cfeaU,
wolfSSL	15:117db924cf7c	1018	0x65afca65U, 0x7a8ef47aU, 0xaee947aeU, 0x08181008U,
wolfSSL	15:117db924cf7c	1019	0xbad56fbaU, 0x7888f078U, 0x256f4a25U, 0x2e725c2eU,
wolfSSL	15:117db924cf7c	1020	0x1c24381cU, 0xa6f157a6U, 0xb4c773b4U, 0xc65197c6U,
wolfSSL	15:117db924cf7c	1021	0xe823cbe8U, 0xdd7ca1ddU, 0x749ce874U, 0x1f213e1fU,
wolfSSL	15:117db924cf7c	1022	0x4bdd964bU, 0xbddc61bdU, 0x8b860d8bU, 0x8a850f8aU,
wolfSSL	15:117db924cf7c	1023	0x7090e070U, 0x3e427c3eU, 0xb5c471b5U, 0x66aacc66U,
wolfSSL	15:117db924cf7c	1024	0x48d89048U, 0x03050603U, 0xf601f7f6U, 0x0e121c0eU,
wolfSSL	15:117db924cf7c	1025	0x61a3c261U, 0x355f6a35U, 0x57f9ae57U, 0xb9d069b9U,
wolfSSL	15:117db924cf7c	1026	0x86911786U, 0xc15899c1U, 0x1d273a1dU, 0x9eb9279eU,
wolfSSL	15:117db924cf7c	1027	0xe138d9e1U, 0xf813ebf8U, 0x98b32b98U, 0x11332211U,
wolfSSL	15:117db924cf7c	1028	0x69bbd269U, 0xd970a9d9U, 0x8e89078eU, 0x94a73394U,
wolfSSL	15:117db924cf7c	1029	0x9bb62d9bU, 0x1e223c1eU, 0x87921587U, 0xe920c9e9U,
wolfSSL	15:117db924cf7c	1030	0xce4987ceU, 0x55ffaa55U, 0x28785028U, 0xdf7aa5dfU,
wolfSSL	15:117db924cf7c	1031	0x8c8f038cU, 0xa1f859a1U, 0x89800989U, 0x0d171a0dU,
wolfSSL	15:117db924cf7c	1032	0xbfda65bfU, 0xe631d7e6U, 0x42c68442U, 0x68b8d068U,
wolfSSL	15:117db924cf7c	1033	0x41c38241U, 0x99b02999U, 0x2d775a2dU, 0x0f111e0fU,
wolfSSL	15:117db924cf7c	1034	0xb0cb7bb0U, 0x54fca854U, 0xbbd66dbbU, 0x163a2c16U,
wolfSSL	15:117db924cf7c	1035	},
wolfSSL	15:117db924cf7c	1036	{
wolfSSL	15:117db924cf7c	1037	0x6363a5c6U, 0x7c7c84f8U, 0x777799eeU, 0x7b7b8df6U,
wolfSSL	15:117db924cf7c	1038	0xf2f20dffU, 0x6b6bbdd6U, 0x6f6fb1deU, 0xc5c55491U,
wolfSSL	15:117db924cf7c	1039	0x30305060U, 0x01010302U, 0x6767a9ceU, 0x2b2b7d56U,
wolfSSL	15:117db924cf7c	1040	0xfefe19e7U, 0xd7d762b5U, 0xababe64dU, 0x76769aecU,
wolfSSL	15:117db924cf7c	1041	0xcaca458fU, 0x82829d1fU, 0xc9c94089U, 0x7d7d87faU,
wolfSSL	15:117db924cf7c	1042	0xfafa15efU, 0x5959ebb2U, 0x4747c98eU, 0xf0f00bfbU,
wolfSSL	15:117db924cf7c	1043	0xadadec41U, 0xd4d467b3U, 0xa2a2fd5fU, 0xafafea45U,
wolfSSL	15:117db924cf7c	1044	0x9c9cbf23U, 0xa4a4f753U, 0x727296e4U, 0xc0c05b9bU,
wolfSSL	15:117db924cf7c	1045	0xb7b7c275U, 0xfdfd1ce1U, 0x9393ae3dU, 0x26266a4cU,
wolfSSL	15:117db924cf7c	1046	0x36365a6cU, 0x3f3f417eU, 0xf7f702f5U, 0xcccc4f83U,
wolfSSL	15:117db924cf7c	1047	0x34345c68U, 0xa5a5f451U, 0xe5e534d1U, 0xf1f108f9U,
wolfSSL	15:117db924cf7c	1048	0x717193e2U, 0xd8d873abU, 0x31315362U, 0x15153f2aU,
wolfSSL	15:117db924cf7c	1049	0x04040c08U, 0xc7c75295U, 0x23236546U, 0xc3c35e9dU,
wolfSSL	15:117db924cf7c	1050	0x18182830U, 0x9696a137U, 0x05050f0aU, 0x9a9ab52fU,
wolfSSL	15:117db924cf7c	1051	0x0707090eU, 0x12123624U, 0x80809b1bU, 0xe2e23ddfU,
wolfSSL	15:117db924cf7c	1052	0xebeb26cdU, 0x2727694eU, 0xb2b2cd7fU, 0x75759feaU,
wolfSSL	15:117db924cf7c	1053	0x09091b12U, 0x83839e1dU, 0x2c2c7458U, 0x1a1a2e34U,
wolfSSL	15:117db924cf7c	1054	0x1b1b2d36U, 0x6e6eb2dcU, 0x5a5aeeb4U, 0xa0a0fb5bU,
wolfSSL	15:117db924cf7c	1055	0x5252f6a4U, 0x3b3b4d76U, 0xd6d661b7U, 0xb3b3ce7dU,
wolfSSL	15:117db924cf7c	1056	0x29297b52U, 0xe3e33eddU, 0x2f2f715eU, 0x84849713U,
wolfSSL	15:117db924cf7c	1057	0x5353f5a6U, 0xd1d168b9U, 0x00000000U, 0xeded2cc1U,
wolfSSL	15:117db924cf7c	1058	0x20206040U, 0xfcfc1fe3U, 0xb1b1c879U, 0x5b5bedb6U,
wolfSSL	15:117db924cf7c	1059	0x6a6abed4U, 0xcbcb468dU, 0xbebed967U, 0x39394b72U,
wolfSSL	15:117db924cf7c	1060	0x4a4ade94U, 0x4c4cd498U, 0x5858e8b0U, 0xcfcf4a85U,
wolfSSL	15:117db924cf7c	1061	0xd0d06bbbU, 0xefef2ac5U, 0xaaaae54fU, 0xfbfb16edU,
wolfSSL	15:117db924cf7c	1062	0x4343c586U, 0x4d4dd79aU, 0x33335566U, 0x85859411U,
wolfSSL	15:117db924cf7c	1063	0x4545cf8aU, 0xf9f910e9U, 0x02020604U, 0x7f7f81feU,
wolfSSL	15:117db924cf7c	1064	0x5050f0a0U, 0x3c3c4478U, 0x9f9fba25U, 0xa8a8e34bU,
wolfSSL	15:117db924cf7c	1065	0x5151f3a2U, 0xa3a3fe5dU, 0x4040c080U, 0x8f8f8a05U,
wolfSSL	15:117db924cf7c	1066	0x9292ad3fU, 0x9d9dbc21U, 0x38384870U, 0xf5f504f1U,
wolfSSL	15:117db924cf7c	1067	0xbcbcdf63U, 0xb6b6c177U, 0xdada75afU, 0x21216342U,
wolfSSL	15:117db924cf7c	1068	0x10103020U, 0xffff1ae5U, 0xf3f30efdU, 0xd2d26dbfU,
wolfSSL	15:117db924cf7c	1069	0xcdcd4c81U, 0x0c0c1418U, 0x13133526U, 0xecec2fc3U,
wolfSSL	15:117db924cf7c	1070	0x5f5fe1beU, 0x9797a235U, 0x4444cc88U, 0x1717392eU,
wolfSSL	15:117db924cf7c	1071	0xc4c45793U, 0xa7a7f255U, 0x7e7e82fcU, 0x3d3d477aU,
wolfSSL	15:117db924cf7c	1072	0x6464acc8U, 0x5d5de7baU, 0x19192b32U, 0x737395e6U,
wolfSSL	15:117db924cf7c	1073	0x6060a0c0U, 0x81819819U, 0x4f4fd19eU, 0xdcdc7fa3U,
wolfSSL	15:117db924cf7c	1074	0x22226644U, 0x2a2a7e54U, 0x9090ab3bU, 0x8888830bU,
wolfSSL	15:117db924cf7c	1075	0x4646ca8cU, 0xeeee29c7U, 0xb8b8d36bU, 0x14143c28U,
wolfSSL	15:117db924cf7c	1076	0xdede79a7U, 0x5e5ee2bcU, 0x0b0b1d16U, 0xdbdb76adU,
wolfSSL	15:117db924cf7c	1077	0xe0e03bdbU, 0x32325664U, 0x3a3a4e74U, 0x0a0a1e14U,
wolfSSL	15:117db924cf7c	1078	0x4949db92U, 0x06060a0cU, 0x24246c48U, 0x5c5ce4b8U,
wolfSSL	15:117db924cf7c	1079	0xc2c25d9fU, 0xd3d36ebdU, 0xacacef43U, 0x6262a6c4U,
wolfSSL	15:117db924cf7c	1080	0x9191a839U, 0x9595a431U, 0xe4e437d3U, 0x79798bf2U,
wolfSSL	15:117db924cf7c	1081	0xe7e732d5U, 0xc8c8438bU, 0x3737596eU, 0x6d6db7daU,
wolfSSL	15:117db924cf7c	1082	0x8d8d8c01U, 0xd5d564b1U, 0x4e4ed29cU, 0xa9a9e049U,
wolfSSL	15:117db924cf7c	1083	0x6c6cb4d8U, 0x5656faacU, 0xf4f407f3U, 0xeaea25cfU,
wolfSSL	15:117db924cf7c	1084	0x6565afcaU, 0x7a7a8ef4U, 0xaeaee947U, 0x08081810U,
wolfSSL	15:117db924cf7c	1085	0xbabad56fU, 0x787888f0U, 0x25256f4aU, 0x2e2e725cU,
wolfSSL	15:117db924cf7c	1086	0x1c1c2438U, 0xa6a6f157U, 0xb4b4c773U, 0xc6c65197U,
wolfSSL	15:117db924cf7c	1087	0xe8e823cbU, 0xdddd7ca1U, 0x74749ce8U, 0x1f1f213eU,
wolfSSL	15:117db924cf7c	1088	0x4b4bdd96U, 0xbdbddc61U, 0x8b8b860dU, 0x8a8a850fU,
wolfSSL	15:117db924cf7c	1089	0x707090e0U, 0x3e3e427cU, 0xb5b5c471U, 0x6666aaccU,
wolfSSL	15:117db924cf7c	1090	0x4848d890U, 0x03030506U, 0xf6f601f7U, 0x0e0e121cU,
wolfSSL	15:117db924cf7c	1091	0x6161a3c2U, 0x35355f6aU, 0x5757f9aeU, 0xb9b9d069U,
wolfSSL	15:117db924cf7c	1092	0x86869117U, 0xc1c15899U, 0x1d1d273aU, 0x9e9eb927U,
wolfSSL	15:117db924cf7c	1093	0xe1e138d9U, 0xf8f813ebU, 0x9898b32bU, 0x11113322U,
wolfSSL	15:117db924cf7c	1094	0x6969bbd2U, 0xd9d970a9U, 0x8e8e8907U, 0x9494a733U,
wolfSSL	15:117db924cf7c	1095	0x9b9bb62dU, 0x1e1e223cU, 0x87879215U, 0xe9e920c9U,
wolfSSL	15:117db924cf7c	1096	0xcece4987U, 0x5555ffaaU, 0x28287850U, 0xdfdf7aa5U,
wolfSSL	15:117db924cf7c	1097	0x8c8c8f03U, 0xa1a1f859U, 0x89898009U, 0x0d0d171aU,
wolfSSL	15:117db924cf7c	1098	0xbfbfda65U, 0xe6e631d7U, 0x4242c684U, 0x6868b8d0U,
wolfSSL	15:117db924cf7c	1099	0x4141c382U, 0x9999b029U, 0x2d2d775aU, 0x0f0f111eU,
wolfSSL	15:117db924cf7c	1100	0xb0b0cb7bU, 0x5454fca8U, 0xbbbbd66dU, 0x16163a2cU,
wolfSSL	15:117db924cf7c	1101	}
wolfSSL	15:117db924cf7c	1102	};
wolfSSL	15:117db924cf7c	1103
wolfSSL	15:117db924cf7c	1104	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	1105	static const word32 Td[4][256] = {
wolfSSL	15:117db924cf7c	1106	{
wolfSSL	15:117db924cf7c	1107	0x51f4a750U, 0x7e416553U, 0x1a17a4c3U, 0x3a275e96U,
wolfSSL	15:117db924cf7c	1108	0x3bab6bcbU, 0x1f9d45f1U, 0xacfa58abU, 0x4be30393U,
wolfSSL	15:117db924cf7c	1109	0x2030fa55U, 0xad766df6U, 0x88cc7691U, 0xf5024c25U,
wolfSSL	15:117db924cf7c	1110	0x4fe5d7fcU, 0xc52acbd7U, 0x26354480U, 0xb562a38fU,
wolfSSL	15:117db924cf7c	1111	0xdeb15a49U, 0x25ba1b67U, 0x45ea0e98U, 0x5dfec0e1U,
wolfSSL	15:117db924cf7c	1112	0xc32f7502U, 0x814cf012U, 0x8d4697a3U, 0x6bd3f9c6U,
wolfSSL	15:117db924cf7c	1113	0x038f5fe7U, 0x15929c95U, 0xbf6d7aebU, 0x955259daU,
wolfSSL	15:117db924cf7c	1114	0xd4be832dU, 0x587421d3U, 0x49e06929U, 0x8ec9c844U,
wolfSSL	15:117db924cf7c	1115	0x75c2896aU, 0xf48e7978U, 0x99583e6bU, 0x27b971ddU,
wolfSSL	15:117db924cf7c	1116	0xbee14fb6U, 0xf088ad17U, 0xc920ac66U, 0x7dce3ab4U,
wolfSSL	15:117db924cf7c	1117	0x63df4a18U, 0xe51a3182U, 0x97513360U, 0x62537f45U,
wolfSSL	15:117db924cf7c	1118	0xb16477e0U, 0xbb6bae84U, 0xfe81a01cU, 0xf9082b94U,
wolfSSL	15:117db924cf7c	1119	0x70486858U, 0x8f45fd19U, 0x94de6c87U, 0x527bf8b7U,
wolfSSL	15:117db924cf7c	1120	0xab73d323U, 0x724b02e2U, 0xe31f8f57U, 0x6655ab2aU,
wolfSSL	15:117db924cf7c	1121	0xb2eb2807U, 0x2fb5c203U, 0x86c57b9aU, 0xd33708a5U,
wolfSSL	15:117db924cf7c	1122	0x302887f2U, 0x23bfa5b2U, 0x02036abaU, 0xed16825cU,
wolfSSL	15:117db924cf7c	1123	0x8acf1c2bU, 0xa779b492U, 0xf307f2f0U, 0x4e69e2a1U,
wolfSSL	15:117db924cf7c	1124	0x65daf4cdU, 0x0605bed5U, 0xd134621fU, 0xc4a6fe8aU,
wolfSSL	15:117db924cf7c	1125	0x342e539dU, 0xa2f355a0U, 0x058ae132U, 0xa4f6eb75U,
wolfSSL	15:117db924cf7c	1126	0x0b83ec39U, 0x4060efaaU, 0x5e719f06U, 0xbd6e1051U,
wolfSSL	15:117db924cf7c	1127	0x3e218af9U, 0x96dd063dU, 0xdd3e05aeU, 0x4de6bd46U,
wolfSSL	15:117db924cf7c	1128	0x91548db5U, 0x71c45d05U, 0x0406d46fU, 0x605015ffU,
wolfSSL	15:117db924cf7c	1129	0x1998fb24U, 0xd6bde997U, 0x894043ccU, 0x67d99e77U,
wolfSSL	15:117db924cf7c	1130	0xb0e842bdU, 0x07898b88U, 0xe7195b38U, 0x79c8eedbU,
wolfSSL	15:117db924cf7c	1131	0xa17c0a47U, 0x7c420fe9U, 0xf8841ec9U, 0x00000000U,
wolfSSL	15:117db924cf7c	1132	0x09808683U, 0x322bed48U, 0x1e1170acU, 0x6c5a724eU,
wolfSSL	15:117db924cf7c	1133	0xfd0efffbU, 0x0f853856U, 0x3daed51eU, 0x362d3927U,
wolfSSL	15:117db924cf7c	1134	0x0a0fd964U, 0x685ca621U, 0x9b5b54d1U, 0x24362e3aU,
wolfSSL	15:117db924cf7c	1135	0x0c0a67b1U, 0x9357e70fU, 0xb4ee96d2U, 0x1b9b919eU,
wolfSSL	15:117db924cf7c	1136	0x80c0c54fU, 0x61dc20a2U, 0x5a774b69U, 0x1c121a16U,
wolfSSL	15:117db924cf7c	1137	0xe293ba0aU, 0xc0a02ae5U, 0x3c22e043U, 0x121b171dU,
wolfSSL	15:117db924cf7c	1138	0x0e090d0bU, 0xf28bc7adU, 0x2db6a8b9U, 0x141ea9c8U,
wolfSSL	15:117db924cf7c	1139	0x57f11985U, 0xaf75074cU, 0xee99ddbbU, 0xa37f60fdU,
wolfSSL	15:117db924cf7c	1140	0xf701269fU, 0x5c72f5bcU, 0x44663bc5U, 0x5bfb7e34U,
wolfSSL	15:117db924cf7c	1141	0x8b432976U, 0xcb23c6dcU, 0xb6edfc68U, 0xb8e4f163U,
wolfSSL	15:117db924cf7c	1142	0xd731dccaU, 0x42638510U, 0x13972240U, 0x84c61120U,
wolfSSL	15:117db924cf7c	1143	0x854a247dU, 0xd2bb3df8U, 0xaef93211U, 0xc729a16dU,
wolfSSL	15:117db924cf7c	1144	0x1d9e2f4bU, 0xdcb230f3U, 0x0d8652ecU, 0x77c1e3d0U,
wolfSSL	15:117db924cf7c	1145	0x2bb3166cU, 0xa970b999U, 0x119448faU, 0x47e96422U,
wolfSSL	15:117db924cf7c	1146	0xa8fc8cc4U, 0xa0f03f1aU, 0x567d2cd8U, 0x223390efU,
wolfSSL	15:117db924cf7c	1147	0x87494ec7U, 0xd938d1c1U, 0x8ccaa2feU, 0x98d40b36U,
wolfSSL	15:117db924cf7c	1148	0xa6f581cfU, 0xa57ade28U, 0xdab78e26U, 0x3fadbfa4U,
wolfSSL	15:117db924cf7c	1149	0x2c3a9de4U, 0x5078920dU, 0x6a5fcc9bU, 0x547e4662U,
wolfSSL	15:117db924cf7c	1150	0xf68d13c2U, 0x90d8b8e8U, 0x2e39f75eU, 0x82c3aff5U,
wolfSSL	15:117db924cf7c	1151	0x9f5d80beU, 0x69d0937cU, 0x6fd52da9U, 0xcf2512b3U,
wolfSSL	15:117db924cf7c	1152	0xc8ac993bU, 0x10187da7U, 0xe89c636eU, 0xdb3bbb7bU,
wolfSSL	15:117db924cf7c	1153	0xcd267809U, 0x6e5918f4U, 0xec9ab701U, 0x834f9aa8U,
wolfSSL	15:117db924cf7c	1154	0xe6956e65U, 0xaaffe67eU, 0x21bccf08U, 0xef15e8e6U,
wolfSSL	15:117db924cf7c	1155	0xbae79bd9U, 0x4a6f36ceU, 0xea9f09d4U, 0x29b07cd6U,
wolfSSL	15:117db924cf7c	1156	0x31a4b2afU, 0x2a3f2331U, 0xc6a59430U, 0x35a266c0U,
wolfSSL	15:117db924cf7c	1157	0x744ebc37U, 0xfc82caa6U, 0xe090d0b0U, 0x33a7d815U,
wolfSSL	15:117db924cf7c	1158	0xf104984aU, 0x41ecdaf7U, 0x7fcd500eU, 0x1791f62fU,
wolfSSL	15:117db924cf7c	1159	0x764dd68dU, 0x43efb04dU, 0xccaa4d54U, 0xe49604dfU,
wolfSSL	15:117db924cf7c	1160	0x9ed1b5e3U, 0x4c6a881bU, 0xc12c1fb8U, 0x4665517fU,
wolfSSL	15:117db924cf7c	1161	0x9d5eea04U, 0x018c355dU, 0xfa877473U, 0xfb0b412eU,
wolfSSL	15:117db924cf7c	1162	0xb3671d5aU, 0x92dbd252U, 0xe9105633U, 0x6dd64713U,
wolfSSL	15:117db924cf7c	1163	0x9ad7618cU, 0x37a10c7aU, 0x59f8148eU, 0xeb133c89U,
wolfSSL	15:117db924cf7c	1164	0xcea927eeU, 0xb761c935U, 0xe11ce5edU, 0x7a47b13cU,
wolfSSL	15:117db924cf7c	1165	0x9cd2df59U, 0x55f2733fU, 0x1814ce79U, 0x73c737bfU,
wolfSSL	15:117db924cf7c	1166	0x53f7cdeaU, 0x5ffdaa5bU, 0xdf3d6f14U, 0x7844db86U,
wolfSSL	15:117db924cf7c	1167	0xcaaff381U, 0xb968c43eU, 0x3824342cU, 0xc2a3405fU,
wolfSSL	15:117db924cf7c	1168	0x161dc372U, 0xbce2250cU, 0x283c498bU, 0xff0d9541U,
wolfSSL	15:117db924cf7c	1169	0x39a80171U, 0x080cb3deU, 0xd8b4e49cU, 0x6456c190U,
wolfSSL	15:117db924cf7c	1170	0x7bcb8461U, 0xd532b670U, 0x486c5c74U, 0xd0b85742U,
wolfSSL	15:117db924cf7c	1171	},
wolfSSL	15:117db924cf7c	1172	{
wolfSSL	15:117db924cf7c	1173	0x5051f4a7U, 0x537e4165U, 0xc31a17a4U, 0x963a275eU,
wolfSSL	15:117db924cf7c	1174	0xcb3bab6bU, 0xf11f9d45U, 0xabacfa58U, 0x934be303U,
wolfSSL	15:117db924cf7c	1175	0x552030faU, 0xf6ad766dU, 0x9188cc76U, 0x25f5024cU,
wolfSSL	15:117db924cf7c	1176	0xfc4fe5d7U, 0xd7c52acbU, 0x80263544U, 0x8fb562a3U,
wolfSSL	15:117db924cf7c	1177	0x49deb15aU, 0x6725ba1bU, 0x9845ea0eU, 0xe15dfec0U,
wolfSSL	15:117db924cf7c	1178	0x02c32f75U, 0x12814cf0U, 0xa38d4697U, 0xc66bd3f9U,
wolfSSL	15:117db924cf7c	1179	0xe7038f5fU, 0x9515929cU, 0xebbf6d7aU, 0xda955259U,
wolfSSL	15:117db924cf7c	1180	0x2dd4be83U, 0xd3587421U, 0x2949e069U, 0x448ec9c8U,
wolfSSL	15:117db924cf7c	1181	0x6a75c289U, 0x78f48e79U, 0x6b99583eU, 0xdd27b971U,
wolfSSL	15:117db924cf7c	1182	0xb6bee14fU, 0x17f088adU, 0x66c920acU, 0xb47dce3aU,
wolfSSL	15:117db924cf7c	1183	0x1863df4aU, 0x82e51a31U, 0x60975133U, 0x4562537fU,
wolfSSL	15:117db924cf7c	1184	0xe0b16477U, 0x84bb6baeU, 0x1cfe81a0U, 0x94f9082bU,
wolfSSL	15:117db924cf7c	1185	0x58704868U, 0x198f45fdU, 0x8794de6cU, 0xb7527bf8U,
wolfSSL	15:117db924cf7c	1186	0x23ab73d3U, 0xe2724b02U, 0x57e31f8fU, 0x2a6655abU,
wolfSSL	15:117db924cf7c	1187	0x07b2eb28U, 0x032fb5c2U, 0x9a86c57bU, 0xa5d33708U,
wolfSSL	15:117db924cf7c	1188	0xf2302887U, 0xb223bfa5U, 0xba02036aU, 0x5ced1682U,
wolfSSL	15:117db924cf7c	1189	0x2b8acf1cU, 0x92a779b4U, 0xf0f307f2U, 0xa14e69e2U,
wolfSSL	15:117db924cf7c	1190	0xcd65daf4U, 0xd50605beU, 0x1fd13462U, 0x8ac4a6feU,
wolfSSL	15:117db924cf7c	1191	0x9d342e53U, 0xa0a2f355U, 0x32058ae1U, 0x75a4f6ebU,
wolfSSL	15:117db924cf7c	1192	0x390b83ecU, 0xaa4060efU, 0x065e719fU, 0x51bd6e10U,
wolfSSL	15:117db924cf7c	1193	0xf93e218aU, 0x3d96dd06U, 0xaedd3e05U, 0x464de6bdU,
wolfSSL	15:117db924cf7c	1194	0xb591548dU, 0x0571c45dU, 0x6f0406d4U, 0xff605015U,
wolfSSL	15:117db924cf7c	1195	0x241998fbU, 0x97d6bde9U, 0xcc894043U, 0x7767d99eU,
wolfSSL	15:117db924cf7c	1196	0xbdb0e842U, 0x8807898bU, 0x38e7195bU, 0xdb79c8eeU,
wolfSSL	15:117db924cf7c	1197	0x47a17c0aU, 0xe97c420fU, 0xc9f8841eU, 0x00000000U,
wolfSSL	15:117db924cf7c	1198	0x83098086U, 0x48322bedU, 0xac1e1170U, 0x4e6c5a72U,
wolfSSL	15:117db924cf7c	1199	0xfbfd0effU, 0x560f8538U, 0x1e3daed5U, 0x27362d39U,
wolfSSL	15:117db924cf7c	1200	0x640a0fd9U, 0x21685ca6U, 0xd19b5b54U, 0x3a24362eU,
wolfSSL	15:117db924cf7c	1201	0xb10c0a67U, 0x0f9357e7U, 0xd2b4ee96U, 0x9e1b9b91U,
wolfSSL	15:117db924cf7c	1202	0x4f80c0c5U, 0xa261dc20U, 0x695a774bU, 0x161c121aU,
wolfSSL	15:117db924cf7c	1203	0x0ae293baU, 0xe5c0a02aU, 0x433c22e0U, 0x1d121b17U,
wolfSSL	15:117db924cf7c	1204	0x0b0e090dU, 0xadf28bc7U, 0xb92db6a8U, 0xc8141ea9U,
wolfSSL	15:117db924cf7c	1205	0x8557f119U, 0x4caf7507U, 0xbbee99ddU, 0xfda37f60U,
wolfSSL	15:117db924cf7c	1206	0x9ff70126U, 0xbc5c72f5U, 0xc544663bU, 0x345bfb7eU,
wolfSSL	15:117db924cf7c	1207	0x768b4329U, 0xdccb23c6U, 0x68b6edfcU, 0x63b8e4f1U,
wolfSSL	15:117db924cf7c	1208	0xcad731dcU, 0x10426385U, 0x40139722U, 0x2084c611U,
wolfSSL	15:117db924cf7c	1209	0x7d854a24U, 0xf8d2bb3dU, 0x11aef932U, 0x6dc729a1U,
wolfSSL	15:117db924cf7c	1210	0x4b1d9e2fU, 0xf3dcb230U, 0xec0d8652U, 0xd077c1e3U,
wolfSSL	15:117db924cf7c	1211	0x6c2bb316U, 0x99a970b9U, 0xfa119448U, 0x2247e964U,
wolfSSL	15:117db924cf7c	1212	0xc4a8fc8cU, 0x1aa0f03fU, 0xd8567d2cU, 0xef223390U,
wolfSSL	15:117db924cf7c	1213	0xc787494eU, 0xc1d938d1U, 0xfe8ccaa2U, 0x3698d40bU,
wolfSSL	15:117db924cf7c	1214	0xcfa6f581U, 0x28a57adeU, 0x26dab78eU, 0xa43fadbfU,
wolfSSL	15:117db924cf7c	1215	0xe42c3a9dU, 0x0d507892U, 0x9b6a5fccU, 0x62547e46U,
wolfSSL	15:117db924cf7c	1216	0xc2f68d13U, 0xe890d8b8U, 0x5e2e39f7U, 0xf582c3afU,
wolfSSL	15:117db924cf7c	1217	0xbe9f5d80U, 0x7c69d093U, 0xa96fd52dU, 0xb3cf2512U,
wolfSSL	15:117db924cf7c	1218	0x3bc8ac99U, 0xa710187dU, 0x6ee89c63U, 0x7bdb3bbbU,
wolfSSL	15:117db924cf7c	1219	0x09cd2678U, 0xf46e5918U, 0x01ec9ab7U, 0xa8834f9aU,
wolfSSL	15:117db924cf7c	1220	0x65e6956eU, 0x7eaaffe6U, 0x0821bccfU, 0xe6ef15e8U,
wolfSSL	15:117db924cf7c	1221	0xd9bae79bU, 0xce4a6f36U, 0xd4ea9f09U, 0xd629b07cU,
wolfSSL	15:117db924cf7c	1222	0xaf31a4b2U, 0x312a3f23U, 0x30c6a594U, 0xc035a266U,
wolfSSL	15:117db924cf7c	1223	0x37744ebcU, 0xa6fc82caU, 0xb0e090d0U, 0x1533a7d8U,
wolfSSL	15:117db924cf7c	1224	0x4af10498U, 0xf741ecdaU, 0x0e7fcd50U, 0x2f1791f6U,
wolfSSL	15:117db924cf7c	1225	0x8d764dd6U, 0x4d43efb0U, 0x54ccaa4dU, 0xdfe49604U,
wolfSSL	15:117db924cf7c	1226	0xe39ed1b5U, 0x1b4c6a88U, 0xb8c12c1fU, 0x7f466551U,
wolfSSL	15:117db924cf7c	1227	0x049d5eeaU, 0x5d018c35U, 0x73fa8774U, 0x2efb0b41U,
wolfSSL	15:117db924cf7c	1228	0x5ab3671dU, 0x5292dbd2U, 0x33e91056U, 0x136dd647U,
wolfSSL	15:117db924cf7c	1229	0x8c9ad761U, 0x7a37a10cU, 0x8e59f814U, 0x89eb133cU,
wolfSSL	15:117db924cf7c	1230	0xeecea927U, 0x35b761c9U, 0xede11ce5U, 0x3c7a47b1U,
wolfSSL	15:117db924cf7c	1231	0x599cd2dfU, 0x3f55f273U, 0x791814ceU, 0xbf73c737U,
wolfSSL	15:117db924cf7c	1232	0xea53f7cdU, 0x5b5ffdaaU, 0x14df3d6fU, 0x867844dbU,
wolfSSL	15:117db924cf7c	1233	0x81caaff3U, 0x3eb968c4U, 0x2c382434U, 0x5fc2a340U,
wolfSSL	15:117db924cf7c	1234	0x72161dc3U, 0x0cbce225U, 0x8b283c49U, 0x41ff0d95U,
wolfSSL	15:117db924cf7c	1235	0x7139a801U, 0xde080cb3U, 0x9cd8b4e4U, 0x906456c1U,
wolfSSL	15:117db924cf7c	1236	0x617bcb84U, 0x70d532b6U, 0x74486c5cU, 0x42d0b857U,
wolfSSL	15:117db924cf7c	1237	},
wolfSSL	15:117db924cf7c	1238	{
wolfSSL	15:117db924cf7c	1239	0xa75051f4U, 0x65537e41U, 0xa4c31a17U, 0x5e963a27U,
wolfSSL	15:117db924cf7c	1240	0x6bcb3babU, 0x45f11f9dU, 0x58abacfaU, 0x03934be3U,
wolfSSL	15:117db924cf7c	1241	0xfa552030U, 0x6df6ad76U, 0x769188ccU, 0x4c25f502U,
wolfSSL	15:117db924cf7c	1242	0xd7fc4fe5U, 0xcbd7c52aU, 0x44802635U, 0xa38fb562U,
wolfSSL	15:117db924cf7c	1243	0x5a49deb1U, 0x1b6725baU, 0x0e9845eaU, 0xc0e15dfeU,
wolfSSL	15:117db924cf7c	1244	0x7502c32fU, 0xf012814cU, 0x97a38d46U, 0xf9c66bd3U,
wolfSSL	15:117db924cf7c	1245	0x5fe7038fU, 0x9c951592U, 0x7aebbf6dU, 0x59da9552U,
wolfSSL	15:117db924cf7c	1246	0x832dd4beU, 0x21d35874U, 0x692949e0U, 0xc8448ec9U,
wolfSSL	15:117db924cf7c	1247	0x896a75c2U, 0x7978f48eU, 0x3e6b9958U, 0x71dd27b9U,
wolfSSL	15:117db924cf7c	1248	0x4fb6bee1U, 0xad17f088U, 0xac66c920U, 0x3ab47dceU,
wolfSSL	15:117db924cf7c	1249	0x4a1863dfU, 0x3182e51aU, 0x33609751U, 0x7f456253U,
wolfSSL	15:117db924cf7c	1250	0x77e0b164U, 0xae84bb6bU, 0xa01cfe81U, 0x2b94f908U,
wolfSSL	15:117db924cf7c	1251	0x68587048U, 0xfd198f45U, 0x6c8794deU, 0xf8b7527bU,
wolfSSL	15:117db924cf7c	1252	0xd323ab73U, 0x02e2724bU, 0x8f57e31fU, 0xab2a6655U,
wolfSSL	15:117db924cf7c	1253	0x2807b2ebU, 0xc2032fb5U, 0x7b9a86c5U, 0x08a5d337U,
wolfSSL	15:117db924cf7c	1254	0x87f23028U, 0xa5b223bfU, 0x6aba0203U, 0x825ced16U,
wolfSSL	15:117db924cf7c	1255	0x1c2b8acfU, 0xb492a779U, 0xf2f0f307U, 0xe2a14e69U,
wolfSSL	15:117db924cf7c	1256	0xf4cd65daU, 0xbed50605U, 0x621fd134U, 0xfe8ac4a6U,
wolfSSL	15:117db924cf7c	1257	0x539d342eU, 0x55a0a2f3U, 0xe132058aU, 0xeb75a4f6U,
wolfSSL	15:117db924cf7c	1258	0xec390b83U, 0xefaa4060U, 0x9f065e71U, 0x1051bd6eU,
wolfSSL	15:117db924cf7c	1259
wolfSSL	15:117db924cf7c	1260	0x8af93e21U, 0x063d96ddU, 0x05aedd3eU, 0xbd464de6U,
wolfSSL	15:117db924cf7c	1261	0x8db59154U, 0x5d0571c4U, 0xd46f0406U, 0x15ff6050U,
wolfSSL	15:117db924cf7c	1262	0xfb241998U, 0xe997d6bdU, 0x43cc8940U, 0x9e7767d9U,
wolfSSL	15:117db924cf7c	1263	0x42bdb0e8U, 0x8b880789U, 0x5b38e719U, 0xeedb79c8U,
wolfSSL	15:117db924cf7c	1264	0x0a47a17cU, 0x0fe97c42U, 0x1ec9f884U, 0x00000000U,
wolfSSL	15:117db924cf7c	1265	0x86830980U, 0xed48322bU, 0x70ac1e11U, 0x724e6c5aU,
wolfSSL	15:117db924cf7c	1266	0xfffbfd0eU, 0x38560f85U, 0xd51e3daeU, 0x3927362dU,
wolfSSL	15:117db924cf7c	1267	0xd9640a0fU, 0xa621685cU, 0x54d19b5bU, 0x2e3a2436U,
wolfSSL	15:117db924cf7c	1268	0x67b10c0aU, 0xe70f9357U, 0x96d2b4eeU, 0x919e1b9bU,
wolfSSL	15:117db924cf7c	1269	0xc54f80c0U, 0x20a261dcU, 0x4b695a77U, 0x1a161c12U,
wolfSSL	15:117db924cf7c	1270	0xba0ae293U, 0x2ae5c0a0U, 0xe0433c22U, 0x171d121bU,
wolfSSL	15:117db924cf7c	1271	0x0d0b0e09U, 0xc7adf28bU, 0xa8b92db6U, 0xa9c8141eU,
wolfSSL	15:117db924cf7c	1272	0x198557f1U, 0x074caf75U, 0xddbbee99U, 0x60fda37fU,
wolfSSL	15:117db924cf7c	1273	0x269ff701U, 0xf5bc5c72U, 0x3bc54466U, 0x7e345bfbU,
wolfSSL	15:117db924cf7c	1274	0x29768b43U, 0xc6dccb23U, 0xfc68b6edU, 0xf163b8e4U,
wolfSSL	15:117db924cf7c	1275	0xdccad731U, 0x85104263U, 0x22401397U, 0x112084c6U,
wolfSSL	15:117db924cf7c	1276	0x247d854aU, 0x3df8d2bbU, 0x3211aef9U, 0xa16dc729U,
wolfSSL	15:117db924cf7c	1277	0x2f4b1d9eU, 0x30f3dcb2U, 0x52ec0d86U, 0xe3d077c1U,
wolfSSL	15:117db924cf7c	1278	0x166c2bb3U, 0xb999a970U, 0x48fa1194U, 0x642247e9U,
wolfSSL	15:117db924cf7c	1279	0x8cc4a8fcU, 0x3f1aa0f0U, 0x2cd8567dU, 0x90ef2233U,
wolfSSL	15:117db924cf7c	1280	0x4ec78749U, 0xd1c1d938U, 0xa2fe8ccaU, 0x0b3698d4U,
wolfSSL	15:117db924cf7c	1281	0x81cfa6f5U, 0xde28a57aU, 0x8e26dab7U, 0xbfa43fadU,
wolfSSL	15:117db924cf7c	1282	0x9de42c3aU, 0x920d5078U, 0xcc9b6a5fU, 0x4662547eU,
wolfSSL	15:117db924cf7c	1283	0x13c2f68dU, 0xb8e890d8U, 0xf75e2e39U, 0xaff582c3U,
wolfSSL	15:117db924cf7c	1284	0x80be9f5dU, 0x937c69d0U, 0x2da96fd5U, 0x12b3cf25U,
wolfSSL	15:117db924cf7c	1285	0x993bc8acU, 0x7da71018U, 0x636ee89cU, 0xbb7bdb3bU,
wolfSSL	15:117db924cf7c	1286	0x7809cd26U, 0x18f46e59U, 0xb701ec9aU, 0x9aa8834fU,
wolfSSL	15:117db924cf7c	1287	0x6e65e695U, 0xe67eaaffU, 0xcf0821bcU, 0xe8e6ef15U,
wolfSSL	15:117db924cf7c	1288	0x9bd9bae7U, 0x36ce4a6fU, 0x09d4ea9fU, 0x7cd629b0U,
wolfSSL	15:117db924cf7c	1289	0xb2af31a4U, 0x23312a3fU, 0x9430c6a5U, 0x66c035a2U,
wolfSSL	15:117db924cf7c	1290	0xbc37744eU, 0xcaa6fc82U, 0xd0b0e090U, 0xd81533a7U,
wolfSSL	15:117db924cf7c	1291	0x984af104U, 0xdaf741ecU, 0x500e7fcdU, 0xf62f1791U,
wolfSSL	15:117db924cf7c	1292	0xd68d764dU, 0xb04d43efU, 0x4d54ccaaU, 0x04dfe496U,
wolfSSL	15:117db924cf7c	1293	0xb5e39ed1U, 0x881b4c6aU, 0x1fb8c12cU, 0x517f4665U,
wolfSSL	15:117db924cf7c	1294	0xea049d5eU, 0x355d018cU, 0x7473fa87U, 0x412efb0bU,
wolfSSL	15:117db924cf7c	1295	0x1d5ab367U, 0xd25292dbU, 0x5633e910U, 0x47136dd6U,
wolfSSL	15:117db924cf7c	1296	0x618c9ad7U, 0x0c7a37a1U, 0x148e59f8U, 0x3c89eb13U,
wolfSSL	15:117db924cf7c	1297	0x27eecea9U, 0xc935b761U, 0xe5ede11cU, 0xb13c7a47U,
wolfSSL	15:117db924cf7c	1298	0xdf599cd2U, 0x733f55f2U, 0xce791814U, 0x37bf73c7U,
wolfSSL	15:117db924cf7c	1299	0xcdea53f7U, 0xaa5b5ffdU, 0x6f14df3dU, 0xdb867844U,
wolfSSL	15:117db924cf7c	1300	0xf381caafU, 0xc43eb968U, 0x342c3824U, 0x405fc2a3U,
wolfSSL	15:117db924cf7c	1301	0xc372161dU, 0x250cbce2U, 0x498b283cU, 0x9541ff0dU,
wolfSSL	15:117db924cf7c	1302	0x017139a8U, 0xb3de080cU, 0xe49cd8b4U, 0xc1906456U,
wolfSSL	15:117db924cf7c	1303	0x84617bcbU, 0xb670d532U, 0x5c74486cU, 0x5742d0b8U,
wolfSSL	15:117db924cf7c	1304	},
wolfSSL	15:117db924cf7c	1305	{
wolfSSL	15:117db924cf7c	1306	0xf4a75051U, 0x4165537eU, 0x17a4c31aU, 0x275e963aU,
wolfSSL	15:117db924cf7c	1307	0xab6bcb3bU, 0x9d45f11fU, 0xfa58abacU, 0xe303934bU,
wolfSSL	15:117db924cf7c	1308	0x30fa5520U, 0x766df6adU, 0xcc769188U, 0x024c25f5U,
wolfSSL	15:117db924cf7c	1309	0xe5d7fc4fU, 0x2acbd7c5U, 0x35448026U, 0x62a38fb5U,
wolfSSL	15:117db924cf7c	1310	0xb15a49deU, 0xba1b6725U, 0xea0e9845U, 0xfec0e15dU,
wolfSSL	15:117db924cf7c	1311	0x2f7502c3U, 0x4cf01281U, 0x4697a38dU, 0xd3f9c66bU,
wolfSSL	15:117db924cf7c	1312	0x8f5fe703U, 0x929c9515U, 0x6d7aebbfU, 0x5259da95U,
wolfSSL	15:117db924cf7c	1313	0xbe832dd4U, 0x7421d358U, 0xe0692949U, 0xc9c8448eU,
wolfSSL	15:117db924cf7c	1314	0xc2896a75U, 0x8e7978f4U, 0x583e6b99U, 0xb971dd27U,
wolfSSL	15:117db924cf7c	1315	0xe14fb6beU, 0x88ad17f0U, 0x20ac66c9U, 0xce3ab47dU,
wolfSSL	15:117db924cf7c	1316	0xdf4a1863U, 0x1a3182e5U, 0x51336097U, 0x537f4562U,
wolfSSL	15:117db924cf7c	1317	0x6477e0b1U, 0x6bae84bbU, 0x81a01cfeU, 0x082b94f9U,
wolfSSL	15:117db924cf7c	1318	0x48685870U, 0x45fd198fU, 0xde6c8794U, 0x7bf8b752U,
wolfSSL	15:117db924cf7c	1319	0x73d323abU, 0x4b02e272U, 0x1f8f57e3U, 0x55ab2a66U,
wolfSSL	15:117db924cf7c	1320	0xeb2807b2U, 0xb5c2032fU, 0xc57b9a86U, 0x3708a5d3U,
wolfSSL	15:117db924cf7c	1321	0x2887f230U, 0xbfa5b223U, 0x036aba02U, 0x16825cedU,
wolfSSL	15:117db924cf7c	1322	0xcf1c2b8aU, 0x79b492a7U, 0x07f2f0f3U, 0x69e2a14eU,
wolfSSL	15:117db924cf7c	1323	0xdaf4cd65U, 0x05bed506U, 0x34621fd1U, 0xa6fe8ac4U,
wolfSSL	15:117db924cf7c	1324	0x2e539d34U, 0xf355a0a2U, 0x8ae13205U, 0xf6eb75a4U,
wolfSSL	15:117db924cf7c	1325	0x83ec390bU, 0x60efaa40U, 0x719f065eU, 0x6e1051bdU,
wolfSSL	15:117db924cf7c	1326	0x218af93eU, 0xdd063d96U, 0x3e05aeddU, 0xe6bd464dU,
wolfSSL	15:117db924cf7c	1327	0x548db591U, 0xc45d0571U, 0x06d46f04U, 0x5015ff60U,
wolfSSL	15:117db924cf7c	1328	0x98fb2419U, 0xbde997d6U, 0x4043cc89U, 0xd99e7767U,
wolfSSL	15:117db924cf7c	1329	0xe842bdb0U, 0x898b8807U, 0x195b38e7U, 0xc8eedb79U,
wolfSSL	15:117db924cf7c	1330	0x7c0a47a1U, 0x420fe97cU, 0x841ec9f8U, 0x00000000U,
wolfSSL	15:117db924cf7c	1331	0x80868309U, 0x2bed4832U, 0x1170ac1eU, 0x5a724e6cU,
wolfSSL	15:117db924cf7c	1332	0x0efffbfdU, 0x8538560fU, 0xaed51e3dU, 0x2d392736U,
wolfSSL	15:117db924cf7c	1333	0x0fd9640aU, 0x5ca62168U, 0x5b54d19bU, 0x362e3a24U,
wolfSSL	15:117db924cf7c	1334	0x0a67b10cU, 0x57e70f93U, 0xee96d2b4U, 0x9b919e1bU,
wolfSSL	15:117db924cf7c	1335	0xc0c54f80U, 0xdc20a261U, 0x774b695aU, 0x121a161cU,
wolfSSL	15:117db924cf7c	1336	0x93ba0ae2U, 0xa02ae5c0U, 0x22e0433cU, 0x1b171d12U,
wolfSSL	15:117db924cf7c	1337	0x090d0b0eU, 0x8bc7adf2U, 0xb6a8b92dU, 0x1ea9c814U,
wolfSSL	15:117db924cf7c	1338	0xf1198557U, 0x75074cafU, 0x99ddbbeeU, 0x7f60fda3U,
wolfSSL	15:117db924cf7c	1339	0x01269ff7U, 0x72f5bc5cU, 0x663bc544U, 0xfb7e345bU,
wolfSSL	15:117db924cf7c	1340	0x4329768bU, 0x23c6dccbU, 0xedfc68b6U, 0xe4f163b8U,
wolfSSL	15:117db924cf7c	1341	0x31dccad7U, 0x63851042U, 0x97224013U, 0xc6112084U,
wolfSSL	15:117db924cf7c	1342	0x4a247d85U, 0xbb3df8d2U, 0xf93211aeU, 0x29a16dc7U,
wolfSSL	15:117db924cf7c	1343	0x9e2f4b1dU, 0xb230f3dcU, 0x8652ec0dU, 0xc1e3d077U,
wolfSSL	15:117db924cf7c	1344	0xb3166c2bU, 0x70b999a9U, 0x9448fa11U, 0xe9642247U,
wolfSSL	15:117db924cf7c	1345	0xfc8cc4a8U, 0xf03f1aa0U, 0x7d2cd856U, 0x3390ef22U,
wolfSSL	15:117db924cf7c	1346	0x494ec787U, 0x38d1c1d9U, 0xcaa2fe8cU, 0xd40b3698U,
wolfSSL	15:117db924cf7c	1347	0xf581cfa6U, 0x7ade28a5U, 0xb78e26daU, 0xadbfa43fU,
wolfSSL	15:117db924cf7c	1348	0x3a9de42cU, 0x78920d50U, 0x5fcc9b6aU, 0x7e466254U,
wolfSSL	15:117db924cf7c	1349	0x8d13c2f6U, 0xd8b8e890U, 0x39f75e2eU, 0xc3aff582U,
wolfSSL	15:117db924cf7c	1350	0x5d80be9fU, 0xd0937c69U, 0xd52da96fU, 0x2512b3cfU,
wolfSSL	15:117db924cf7c	1351	0xac993bc8U, 0x187da710U, 0x9c636ee8U, 0x3bbb7bdbU,
wolfSSL	15:117db924cf7c	1352	0x267809cdU, 0x5918f46eU, 0x9ab701ecU, 0x4f9aa883U,
wolfSSL	15:117db924cf7c	1353	0x956e65e6U, 0xffe67eaaU, 0xbccf0821U, 0x15e8e6efU,
wolfSSL	15:117db924cf7c	1354	0xe79bd9baU, 0x6f36ce4aU, 0x9f09d4eaU, 0xb07cd629U,
wolfSSL	15:117db924cf7c	1355	0xa4b2af31U, 0x3f23312aU, 0xa59430c6U, 0xa266c035U,
wolfSSL	15:117db924cf7c	1356	0x4ebc3774U, 0x82caa6fcU, 0x90d0b0e0U, 0xa7d81533U,
wolfSSL	15:117db924cf7c	1357	0x04984af1U, 0xecdaf741U, 0xcd500e7fU, 0x91f62f17U,
wolfSSL	15:117db924cf7c	1358	0x4dd68d76U, 0xefb04d43U, 0xaa4d54ccU, 0x9604dfe4U,
wolfSSL	15:117db924cf7c	1359	0xd1b5e39eU, 0x6a881b4cU, 0x2c1fb8c1U, 0x65517f46U,
wolfSSL	15:117db924cf7c	1360	0x5eea049dU, 0x8c355d01U, 0x877473faU, 0x0b412efbU,
wolfSSL	15:117db924cf7c	1361	0x671d5ab3U, 0xdbd25292U, 0x105633e9U, 0xd647136dU,
wolfSSL	15:117db924cf7c	1362	0xd7618c9aU, 0xa10c7a37U, 0xf8148e59U, 0x133c89ebU,
wolfSSL	15:117db924cf7c	1363	0xa927eeceU, 0x61c935b7U, 0x1ce5ede1U, 0x47b13c7aU,
wolfSSL	15:117db924cf7c	1364	0xd2df599cU, 0xf2733f55U, 0x14ce7918U, 0xc737bf73U,
wolfSSL	15:117db924cf7c	1365	0xf7cdea53U, 0xfdaa5b5fU, 0x3d6f14dfU, 0x44db8678U,
wolfSSL	15:117db924cf7c	1366	0xaff381caU, 0x68c43eb9U, 0x24342c38U, 0xa3405fc2U,
wolfSSL	15:117db924cf7c	1367	0x1dc37216U, 0xe2250cbcU, 0x3c498b28U, 0x0d9541ffU,
wolfSSL	15:117db924cf7c	1368	0xa8017139U, 0x0cb3de08U, 0xb4e49cd8U, 0x56c19064U,
wolfSSL	15:117db924cf7c	1369	0xcb84617bU, 0x32b670d5U, 0x6c5c7448U, 0xb85742d0U,
wolfSSL	15:117db924cf7c	1370	}
wolfSSL	15:117db924cf7c	1371	};
wolfSSL	15:117db924cf7c	1372
wolfSSL	15:117db924cf7c	1373
wolfSSL	15:117db924cf7c	1374	static const byte Td4[256] =
wolfSSL	15:117db924cf7c	1375	{
wolfSSL	15:117db924cf7c	1376	0x52U, 0x09U, 0x6aU, 0xd5U, 0x30U, 0x36U, 0xa5U, 0x38U,
wolfSSL	15:117db924cf7c	1377	0xbfU, 0x40U, 0xa3U, 0x9eU, 0x81U, 0xf3U, 0xd7U, 0xfbU,
wolfSSL	15:117db924cf7c	1378	0x7cU, 0xe3U, 0x39U, 0x82U, 0x9bU, 0x2fU, 0xffU, 0x87U,
wolfSSL	15:117db924cf7c	1379	0x34U, 0x8eU, 0x43U, 0x44U, 0xc4U, 0xdeU, 0xe9U, 0xcbU,
wolfSSL	15:117db924cf7c	1380	0x54U, 0x7bU, 0x94U, 0x32U, 0xa6U, 0xc2U, 0x23U, 0x3dU,
wolfSSL	15:117db924cf7c	1381	0xeeU, 0x4cU, 0x95U, 0x0bU, 0x42U, 0xfaU, 0xc3U, 0x4eU,
wolfSSL	15:117db924cf7c	1382	0x08U, 0x2eU, 0xa1U, 0x66U, 0x28U, 0xd9U, 0x24U, 0xb2U,
wolfSSL	15:117db924cf7c	1383	0x76U, 0x5bU, 0xa2U, 0x49U, 0x6dU, 0x8bU, 0xd1U, 0x25U,
wolfSSL	15:117db924cf7c	1384	0x72U, 0xf8U, 0xf6U, 0x64U, 0x86U, 0x68U, 0x98U, 0x16U,
wolfSSL	15:117db924cf7c	1385	0xd4U, 0xa4U, 0x5cU, 0xccU, 0x5dU, 0x65U, 0xb6U, 0x92U,
wolfSSL	15:117db924cf7c	1386	0x6cU, 0x70U, 0x48U, 0x50U, 0xfdU, 0xedU, 0xb9U, 0xdaU,
wolfSSL	15:117db924cf7c	1387	0x5eU, 0x15U, 0x46U, 0x57U, 0xa7U, 0x8dU, 0x9dU, 0x84U,
wolfSSL	15:117db924cf7c	1388	0x90U, 0xd8U, 0xabU, 0x00U, 0x8cU, 0xbcU, 0xd3U, 0x0aU,
wolfSSL	15:117db924cf7c	1389	0xf7U, 0xe4U, 0x58U, 0x05U, 0xb8U, 0xb3U, 0x45U, 0x06U,
wolfSSL	15:117db924cf7c	1390	0xd0U, 0x2cU, 0x1eU, 0x8fU, 0xcaU, 0x3fU, 0x0fU, 0x02U,
wolfSSL	15:117db924cf7c	1391	0xc1U, 0xafU, 0xbdU, 0x03U, 0x01U, 0x13U, 0x8aU, 0x6bU,
wolfSSL	15:117db924cf7c	1392	0x3aU, 0x91U, 0x11U, 0x41U, 0x4fU, 0x67U, 0xdcU, 0xeaU,
wolfSSL	15:117db924cf7c	1393	0x97U, 0xf2U, 0xcfU, 0xceU, 0xf0U, 0xb4U, 0xe6U, 0x73U,
wolfSSL	15:117db924cf7c	1394	0x96U, 0xacU, 0x74U, 0x22U, 0xe7U, 0xadU, 0x35U, 0x85U,
wolfSSL	15:117db924cf7c	1395	0xe2U, 0xf9U, 0x37U, 0xe8U, 0x1cU, 0x75U, 0xdfU, 0x6eU,
wolfSSL	15:117db924cf7c	1396	0x47U, 0xf1U, 0x1aU, 0x71U, 0x1dU, 0x29U, 0xc5U, 0x89U,
wolfSSL	15:117db924cf7c	1397	0x6fU, 0xb7U, 0x62U, 0x0eU, 0xaaU, 0x18U, 0xbeU, 0x1bU,
wolfSSL	15:117db924cf7c	1398	0xfcU, 0x56U, 0x3eU, 0x4bU, 0xc6U, 0xd2U, 0x79U, 0x20U,
wolfSSL	15:117db924cf7c	1399	0x9aU, 0xdbU, 0xc0U, 0xfeU, 0x78U, 0xcdU, 0x5aU, 0xf4U,
wolfSSL	15:117db924cf7c	1400	0x1fU, 0xddU, 0xa8U, 0x33U, 0x88U, 0x07U, 0xc7U, 0x31U,
wolfSSL	15:117db924cf7c	1401	0xb1U, 0x12U, 0x10U, 0x59U, 0x27U, 0x80U, 0xecU, 0x5fU,
wolfSSL	15:117db924cf7c	1402	0x60U, 0x51U, 0x7fU, 0xa9U, 0x19U, 0xb5U, 0x4aU, 0x0dU,
wolfSSL	15:117db924cf7c	1403	0x2dU, 0xe5U, 0x7aU, 0x9fU, 0x93U, 0xc9U, 0x9cU, 0xefU,
wolfSSL	15:117db924cf7c	1404	0xa0U, 0xe0U, 0x3bU, 0x4dU, 0xaeU, 0x2aU, 0xf5U, 0xb0U,
wolfSSL	15:117db924cf7c	1405	0xc8U, 0xebU, 0xbbU, 0x3cU, 0x83U, 0x53U, 0x99U, 0x61U,
wolfSSL	15:117db924cf7c	1406	0x17U, 0x2bU, 0x04U, 0x7eU, 0xbaU, 0x77U, 0xd6U, 0x26U,
wolfSSL	15:117db924cf7c	1407	0xe1U, 0x69U, 0x14U, 0x63U, 0x55U, 0x21U, 0x0cU, 0x7dU,
wolfSSL	15:117db924cf7c	1408	};
wolfSSL	15:117db924cf7c	1409	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	1410
wolfSSL	15:117db924cf7c	1411	#define GETBYTE(x, y) (word32)((byte)((x) >> (8 * (y))))
wolfSSL	15:117db924cf7c	1412
wolfSSL	15:117db924cf7c	1413
wolfSSL	15:117db924cf7c	1414
wolfSSL	15:117db924cf7c	1415	#if defined(HAVE_AES_CBC) \|\| defined(WOLFSSL_AES_DIRECT) \|\| defined(HAVE_AESGCM)
wolfSSL	15:117db924cf7c	1416
wolfSSL	15:117db924cf7c	1417	#ifndef WC_CACHE_LINE_SZ
wolfSSL	15:117db924cf7c	1418	#if defined(__x86_64__) \|\| defined(_M_X64) \|\| \
wolfSSL	15:117db924cf7c	1419	(defined(__ILP32__) && (__ILP32__ >= 1))
wolfSSL	15:117db924cf7c	1420	#define WC_CACHE_LINE_SZ 64
wolfSSL	15:117db924cf7c	1421	#else
wolfSSL	15:117db924cf7c	1422	/* default cache line size */
wolfSSL	15:117db924cf7c	1423	#define WC_CACHE_LINE_SZ 32
wolfSSL	15:117db924cf7c	1424	#endif
wolfSSL	15:117db924cf7c	1425	#endif
wolfSSL	15:117db924cf7c	1426
wolfSSL	15:117db924cf7c	1427
wolfSSL	15:117db924cf7c	1428	/* load 4 Te Tables into cache by cache line stride */
wolfSSL	15:117db924cf7c	1429	static WC_INLINE word32 PreFetchTe(void)
wolfSSL	15:117db924cf7c	1430	{
wolfSSL	15:117db924cf7c	1431	word32 x = 0;
wolfSSL	15:117db924cf7c	1432	int i,j;
wolfSSL	15:117db924cf7c	1433
wolfSSL	15:117db924cf7c	1434	for (i = 0; i < 4; i++) {
wolfSSL	15:117db924cf7c	1435	/* 256 elements, each one is 4 bytes */
wolfSSL	15:117db924cf7c	1436	for (j = 0; j < 256; j += WC_CACHE_LINE_SZ/4) {
wolfSSL	15:117db924cf7c	1437	x &= Te[i][j];
wolfSSL	15:117db924cf7c	1438	}
wolfSSL	15:117db924cf7c	1439	}
wolfSSL	15:117db924cf7c	1440	return x;
wolfSSL	15:117db924cf7c	1441	}
wolfSSL	15:117db924cf7c	1442
wolfSSL	15:117db924cf7c	1443
wolfSSL	15:117db924cf7c	1444	static void wc_AesEncrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	1445	{
wolfSSL	15:117db924cf7c	1446	word32 s0, s1, s2, s3;
wolfSSL	15:117db924cf7c	1447	word32 t0, t1, t2, t3;
wolfSSL	15:117db924cf7c	1448	word32 r = aes->rounds >> 1;
wolfSSL	15:117db924cf7c	1449	const word32* rk = aes->key;
wolfSSL	15:117db924cf7c	1450
wolfSSL	15:117db924cf7c	1451	if (r > 7 \|\| r == 0) {
wolfSSL	15:117db924cf7c	1452	WOLFSSL_MSG("AesEncrypt encountered improper key, set it up");
wolfSSL	15:117db924cf7c	1453	return; /* stop instead of segfaulting, set up your keys! */
wolfSSL	15:117db924cf7c	1454	}
wolfSSL	15:117db924cf7c	1455
wolfSSL	15:117db924cf7c	1456	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	1457	if (haveAESNI && aes->use_aesni) {
wolfSSL	15:117db924cf7c	1458	#ifdef DEBUG_AESNI
wolfSSL	15:117db924cf7c	1459	printf("about to aes encrypt\n");
wolfSSL	15:117db924cf7c	1460	printf("in = %p\n", inBlock);
wolfSSL	15:117db924cf7c	1461	printf("out = %p\n", outBlock);
wolfSSL	15:117db924cf7c	1462	printf("aes->key = %p\n", aes->key);
wolfSSL	15:117db924cf7c	1463	printf("aes->rounds = %d\n", aes->rounds);
wolfSSL	15:117db924cf7c	1464	printf("sz = %d\n", AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	1465	#endif
wolfSSL	15:117db924cf7c	1466
wolfSSL	15:117db924cf7c	1467	/* check alignment, decrypt doesn't need alignment */
wolfSSL	15:117db924cf7c	1468	if ((wolfssl_word)inBlock % AESNI_ALIGN) {
wolfSSL	15:117db924cf7c	1469	#ifndef NO_WOLFSSL_ALLOC_ALIGN
wolfSSL	15:117db924cf7c	1470	byte* tmp = (byte*)XMALLOC(AES_BLOCK_SIZE + AESNI_ALIGN, aes->heap,
wolfSSL	15:117db924cf7c	1471	DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	1472	byte* tmp_align;
wolfSSL	15:117db924cf7c	1473	if (tmp == NULL) return;
wolfSSL	15:117db924cf7c	1474
wolfSSL	15:117db924cf7c	1475	tmp_align = tmp + (AESNI_ALIGN - ((size_t)tmp % AESNI_ALIGN));
wolfSSL	15:117db924cf7c	1476
wolfSSL	15:117db924cf7c	1477	XMEMCPY(tmp_align, inBlock, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	1478	AES_ECB_encrypt(tmp_align, tmp_align, AES_BLOCK_SIZE, (byte*)aes->key,
wolfSSL	15:117db924cf7c	1479	aes->rounds);
wolfSSL	15:117db924cf7c	1480	XMEMCPY(outBlock, tmp_align, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	1481	XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	1482	return;
wolfSSL	15:117db924cf7c	1483	#else
wolfSSL	15:117db924cf7c	1484	WOLFSSL_MSG("AES-ECB encrypt with bad alignment");
wolfSSL	15:117db924cf7c	1485	return;
wolfSSL	15:117db924cf7c	1486	#endif
wolfSSL	15:117db924cf7c	1487	}
wolfSSL	15:117db924cf7c	1488
wolfSSL	15:117db924cf7c	1489	AES_ECB_encrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key,
wolfSSL	15:117db924cf7c	1490	aes->rounds);
wolfSSL	15:117db924cf7c	1491
wolfSSL	15:117db924cf7c	1492	return;
wolfSSL	15:117db924cf7c	1493	}
wolfSSL	15:117db924cf7c	1494	else {
wolfSSL	15:117db924cf7c	1495	#ifdef DEBUG_AESNI
wolfSSL	15:117db924cf7c	1496	printf("Skipping AES-NI\n");
wolfSSL	15:117db924cf7c	1497	#endif
wolfSSL	15:117db924cf7c	1498	}
wolfSSL	15:117db924cf7c	1499	#endif
wolfSSL	15:117db924cf7c	1500
wolfSSL	15:117db924cf7c	1501	/*
wolfSSL	15:117db924cf7c	1502	* map byte array block to cipher state
wolfSSL	15:117db924cf7c	1503	* and add initial round key:
wolfSSL	15:117db924cf7c	1504	*/
wolfSSL	15:117db924cf7c	1505	XMEMCPY(&s0, inBlock, sizeof(s0));
wolfSSL	15:117db924cf7c	1506	XMEMCPY(&s1, inBlock + sizeof(s0), sizeof(s1));
wolfSSL	15:117db924cf7c	1507	XMEMCPY(&s2, inBlock + 2 * sizeof(s0), sizeof(s2));
wolfSSL	15:117db924cf7c	1508	XMEMCPY(&s3, inBlock + 3 * sizeof(s0), sizeof(s3));
wolfSSL	15:117db924cf7c	1509
wolfSSL	15:117db924cf7c	1510	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	1511	s0 = ByteReverseWord32(s0);
wolfSSL	15:117db924cf7c	1512	s1 = ByteReverseWord32(s1);
wolfSSL	15:117db924cf7c	1513	s2 = ByteReverseWord32(s2);
wolfSSL	15:117db924cf7c	1514	s3 = ByteReverseWord32(s3);
wolfSSL	15:117db924cf7c	1515	#endif
wolfSSL	15:117db924cf7c	1516
wolfSSL	15:117db924cf7c	1517	s0 ^= rk[0];
wolfSSL	15:117db924cf7c	1518	s1 ^= rk[1];
wolfSSL	15:117db924cf7c	1519	s2 ^= rk[2];
wolfSSL	15:117db924cf7c	1520	s3 ^= rk[3];
wolfSSL	15:117db924cf7c	1521
wolfSSL	15:117db924cf7c	1522	s0 \|= PreFetchTe();
wolfSSL	15:117db924cf7c	1523
wolfSSL	15:117db924cf7c	1524	/*
wolfSSL	15:117db924cf7c	1525	* Nr - 1 full rounds:
wolfSSL	15:117db924cf7c	1526	*/
wolfSSL	15:117db924cf7c	1527
wolfSSL	15:117db924cf7c	1528	for (;;) {
wolfSSL	15:117db924cf7c	1529	t0 =
wolfSSL	15:117db924cf7c	1530	Te[0][GETBYTE(s0, 3)] ^
wolfSSL	15:117db924cf7c	1531	Te[1][GETBYTE(s1, 2)] ^
wolfSSL	15:117db924cf7c	1532	Te[2][GETBYTE(s2, 1)] ^
wolfSSL	15:117db924cf7c	1533	Te[3][GETBYTE(s3, 0)] ^
wolfSSL	15:117db924cf7c	1534	rk[4];
wolfSSL	15:117db924cf7c	1535	t1 =
wolfSSL	15:117db924cf7c	1536	Te[0][GETBYTE(s1, 3)] ^
wolfSSL	15:117db924cf7c	1537	Te[1][GETBYTE(s2, 2)] ^
wolfSSL	15:117db924cf7c	1538	Te[2][GETBYTE(s3, 1)] ^
wolfSSL	15:117db924cf7c	1539	Te[3][GETBYTE(s0, 0)] ^
wolfSSL	15:117db924cf7c	1540	rk[5];
wolfSSL	15:117db924cf7c	1541	t2 =
wolfSSL	15:117db924cf7c	1542	Te[0][GETBYTE(s2, 3)] ^
wolfSSL	15:117db924cf7c	1543	Te[1][GETBYTE(s3, 2)] ^
wolfSSL	15:117db924cf7c	1544	Te[2][GETBYTE(s0, 1)] ^
wolfSSL	15:117db924cf7c	1545	Te[3][GETBYTE(s1, 0)] ^
wolfSSL	15:117db924cf7c	1546	rk[6];
wolfSSL	15:117db924cf7c	1547	t3 =
wolfSSL	15:117db924cf7c	1548	Te[0][GETBYTE(s3, 3)] ^
wolfSSL	15:117db924cf7c	1549	Te[1][GETBYTE(s0, 2)] ^
wolfSSL	15:117db924cf7c	1550	Te[2][GETBYTE(s1, 1)] ^
wolfSSL	15:117db924cf7c	1551	Te[3][GETBYTE(s2, 0)] ^
wolfSSL	15:117db924cf7c	1552	rk[7];
wolfSSL	15:117db924cf7c	1553
wolfSSL	15:117db924cf7c	1554	rk += 8;
wolfSSL	15:117db924cf7c	1555	if (--r == 0) {
wolfSSL	15:117db924cf7c	1556	break;
wolfSSL	15:117db924cf7c	1557	}
wolfSSL	15:117db924cf7c	1558
wolfSSL	15:117db924cf7c	1559	s0 =
wolfSSL	15:117db924cf7c	1560	Te[0][GETBYTE(t0, 3)] ^
wolfSSL	15:117db924cf7c	1561	Te[1][GETBYTE(t1, 2)] ^
wolfSSL	15:117db924cf7c	1562	Te[2][GETBYTE(t2, 1)] ^
wolfSSL	15:117db924cf7c	1563	Te[3][GETBYTE(t3, 0)] ^
wolfSSL	15:117db924cf7c	1564	rk[0];
wolfSSL	15:117db924cf7c	1565	s1 =
wolfSSL	15:117db924cf7c	1566	Te[0][GETBYTE(t1, 3)] ^
wolfSSL	15:117db924cf7c	1567	Te[1][GETBYTE(t2, 2)] ^
wolfSSL	15:117db924cf7c	1568	Te[2][GETBYTE(t3, 1)] ^
wolfSSL	15:117db924cf7c	1569	Te[3][GETBYTE(t0, 0)] ^
wolfSSL	15:117db924cf7c	1570	rk[1];
wolfSSL	15:117db924cf7c	1571	s2 =
wolfSSL	15:117db924cf7c	1572	Te[0][GETBYTE(t2, 3)] ^
wolfSSL	15:117db924cf7c	1573	Te[1][GETBYTE(t3, 2)] ^
wolfSSL	15:117db924cf7c	1574	Te[2][GETBYTE(t0, 1)] ^
wolfSSL	15:117db924cf7c	1575	Te[3][GETBYTE(t1, 0)] ^
wolfSSL	15:117db924cf7c	1576	rk[2];
wolfSSL	15:117db924cf7c	1577	s3 =
wolfSSL	15:117db924cf7c	1578	Te[0][GETBYTE(t3, 3)] ^
wolfSSL	15:117db924cf7c	1579	Te[1][GETBYTE(t0, 2)] ^
wolfSSL	15:117db924cf7c	1580	Te[2][GETBYTE(t1, 1)] ^
wolfSSL	15:117db924cf7c	1581	Te[3][GETBYTE(t2, 0)] ^
wolfSSL	15:117db924cf7c	1582	rk[3];
wolfSSL	15:117db924cf7c	1583	}
wolfSSL	15:117db924cf7c	1584
wolfSSL	15:117db924cf7c	1585	/*
wolfSSL	15:117db924cf7c	1586	* apply last round and
wolfSSL	15:117db924cf7c	1587	* map cipher state to byte array block:
wolfSSL	15:117db924cf7c	1588	*/
wolfSSL	15:117db924cf7c	1589
wolfSSL	15:117db924cf7c	1590	s0 =
wolfSSL	15:117db924cf7c	1591	(Te[2][GETBYTE(t0, 3)] & 0xff000000) ^
wolfSSL	15:117db924cf7c	1592	(Te[3][GETBYTE(t1, 2)] & 0x00ff0000) ^
wolfSSL	15:117db924cf7c	1593	(Te[0][GETBYTE(t2, 1)] & 0x0000ff00) ^
wolfSSL	15:117db924cf7c	1594	(Te[1][GETBYTE(t3, 0)] & 0x000000ff) ^
wolfSSL	15:117db924cf7c	1595	rk[0];
wolfSSL	15:117db924cf7c	1596	s1 =
wolfSSL	15:117db924cf7c	1597	(Te[2][GETBYTE(t1, 3)] & 0xff000000) ^
wolfSSL	15:117db924cf7c	1598	(Te[3][GETBYTE(t2, 2)] & 0x00ff0000) ^
wolfSSL	15:117db924cf7c	1599	(Te[0][GETBYTE(t3, 1)] & 0x0000ff00) ^
wolfSSL	15:117db924cf7c	1600	(Te[1][GETBYTE(t0, 0)] & 0x000000ff) ^
wolfSSL	15:117db924cf7c	1601	rk[1];
wolfSSL	15:117db924cf7c	1602	s2 =
wolfSSL	15:117db924cf7c	1603	(Te[2][GETBYTE(t2, 3)] & 0xff000000) ^
wolfSSL	15:117db924cf7c	1604	(Te[3][GETBYTE(t3, 2)] & 0x00ff0000) ^
wolfSSL	15:117db924cf7c	1605	(Te[0][GETBYTE(t0, 1)] & 0x0000ff00) ^
wolfSSL	15:117db924cf7c	1606	(Te[1][GETBYTE(t1, 0)] & 0x000000ff) ^
wolfSSL	15:117db924cf7c	1607	rk[2];
wolfSSL	15:117db924cf7c	1608	s3 =
wolfSSL	15:117db924cf7c	1609	(Te[2][GETBYTE(t3, 3)] & 0xff000000) ^
wolfSSL	15:117db924cf7c	1610	(Te[3][GETBYTE(t0, 2)] & 0x00ff0000) ^
wolfSSL	15:117db924cf7c	1611	(Te[0][GETBYTE(t1, 1)] & 0x0000ff00) ^
wolfSSL	15:117db924cf7c	1612	(Te[1][GETBYTE(t2, 0)] & 0x000000ff) ^
wolfSSL	15:117db924cf7c	1613	rk[3];
wolfSSL	15:117db924cf7c	1614
wolfSSL	15:117db924cf7c	1615	/* write out */
wolfSSL	15:117db924cf7c	1616	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	1617	s0 = ByteReverseWord32(s0);
wolfSSL	15:117db924cf7c	1618	s1 = ByteReverseWord32(s1);
wolfSSL	15:117db924cf7c	1619	s2 = ByteReverseWord32(s2);
wolfSSL	15:117db924cf7c	1620	s3 = ByteReverseWord32(s3);
wolfSSL	15:117db924cf7c	1621	#endif
wolfSSL	15:117db924cf7c	1622
wolfSSL	15:117db924cf7c	1623	XMEMCPY(outBlock, &s0, sizeof(s0));
wolfSSL	15:117db924cf7c	1624	XMEMCPY(outBlock + sizeof(s0), &s1, sizeof(s1));
wolfSSL	15:117db924cf7c	1625	XMEMCPY(outBlock + 2 * sizeof(s0), &s2, sizeof(s2));
wolfSSL	15:117db924cf7c	1626	XMEMCPY(outBlock + 3 * sizeof(s0), &s3, sizeof(s3));
wolfSSL	15:117db924cf7c	1627
wolfSSL	15:117db924cf7c	1628	}
wolfSSL	15:117db924cf7c	1629	#endif /* HAVE_AES_CBC \|\| WOLFSSL_AES_DIRECT \|\| HAVE_AESGCM */
wolfSSL	15:117db924cf7c	1630
wolfSSL	15:117db924cf7c	1631	#if defined(HAVE_AES_DECRYPT)
wolfSSL	15:117db924cf7c	1632	#if defined(HAVE_AES_CBC) \|\| defined(WOLFSSL_AES_DIRECT)
wolfSSL	15:117db924cf7c	1633
wolfSSL	15:117db924cf7c	1634	/* load 4 Td Tables into cache by cache line stride */
wolfSSL	15:117db924cf7c	1635	static WC_INLINE word32 PreFetchTd(void)
wolfSSL	15:117db924cf7c	1636	{
wolfSSL	15:117db924cf7c	1637	word32 x = 0;
wolfSSL	15:117db924cf7c	1638	int i,j;
wolfSSL	15:117db924cf7c	1639
wolfSSL	15:117db924cf7c	1640	for (i = 0; i < 4; i++) {
wolfSSL	15:117db924cf7c	1641	/* 256 elements, each one is 4 bytes */
wolfSSL	15:117db924cf7c	1642	for (j = 0; j < 256; j += WC_CACHE_LINE_SZ/4) {
wolfSSL	15:117db924cf7c	1643	x &= Td[i][j];
wolfSSL	15:117db924cf7c	1644	}
wolfSSL	15:117db924cf7c	1645	}
wolfSSL	15:117db924cf7c	1646	return x;
wolfSSL	15:117db924cf7c	1647	}
wolfSSL	15:117db924cf7c	1648
wolfSSL	15:117db924cf7c	1649	/* load Td Table4 into cache by cache line stride */
wolfSSL	15:117db924cf7c	1650	static WC_INLINE word32 PreFetchTd4(void)
wolfSSL	15:117db924cf7c	1651	{
wolfSSL	15:117db924cf7c	1652	word32 x = 0;
wolfSSL	15:117db924cf7c	1653	int i;
wolfSSL	15:117db924cf7c	1654
wolfSSL	15:117db924cf7c	1655	for (i = 0; i < 256; i += WC_CACHE_LINE_SZ) {
wolfSSL	15:117db924cf7c	1656	x &= (word32)Td4[i];
wolfSSL	15:117db924cf7c	1657	}
wolfSSL	15:117db924cf7c	1658	return x;
wolfSSL	15:117db924cf7c	1659	}
wolfSSL	15:117db924cf7c	1660
wolfSSL	15:117db924cf7c	1661	static void wc_AesDecrypt(Aes* aes, const byte* inBlock, byte* outBlock)
wolfSSL	15:117db924cf7c	1662	{
wolfSSL	15:117db924cf7c	1663	word32 s0, s1, s2, s3;
wolfSSL	15:117db924cf7c	1664	word32 t0, t1, t2, t3;
wolfSSL	15:117db924cf7c	1665	word32 r = aes->rounds >> 1;
wolfSSL	15:117db924cf7c	1666
wolfSSL	15:117db924cf7c	1667	const word32* rk = aes->key;
wolfSSL	15:117db924cf7c	1668	if (r > 7 \|\| r == 0) {
wolfSSL	15:117db924cf7c	1669	WOLFSSL_MSG("AesDecrypt encountered improper key, set it up");
wolfSSL	15:117db924cf7c	1670	return; /* stop instead of segfaulting, set up your keys! */
wolfSSL	15:117db924cf7c	1671	}
wolfSSL	15:117db924cf7c	1672	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	1673	if (haveAESNI && aes->use_aesni) {
wolfSSL	15:117db924cf7c	1674	#ifdef DEBUG_AESNI
wolfSSL	15:117db924cf7c	1675	printf("about to aes decrypt\n");
wolfSSL	15:117db924cf7c	1676	printf("in = %p\n", inBlock);
wolfSSL	15:117db924cf7c	1677	printf("out = %p\n", outBlock);
wolfSSL	15:117db924cf7c	1678	printf("aes->key = %p\n", aes->key);
wolfSSL	15:117db924cf7c	1679	printf("aes->rounds = %d\n", aes->rounds);
wolfSSL	15:117db924cf7c	1680	printf("sz = %d\n", AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	1681	#endif
wolfSSL	15:117db924cf7c	1682
wolfSSL	15:117db924cf7c	1683	/* if input and output same will overwrite input iv */
wolfSSL	15:117db924cf7c	1684	XMEMCPY(aes->tmp, inBlock, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	1685	AES_ECB_decrypt(inBlock, outBlock, AES_BLOCK_SIZE, (byte*)aes->key,
wolfSSL	15:117db924cf7c	1686	aes->rounds);
wolfSSL	15:117db924cf7c	1687	return;
wolfSSL	15:117db924cf7c	1688	}
wolfSSL	15:117db924cf7c	1689	else {
wolfSSL	15:117db924cf7c	1690	#ifdef DEBUG_AESNI
wolfSSL	15:117db924cf7c	1691	printf("Skipping AES-NI\n");
wolfSSL	15:117db924cf7c	1692	#endif
wolfSSL	15:117db924cf7c	1693	}
wolfSSL	15:117db924cf7c	1694	#endif /* WOLFSSL_AESNI */
wolfSSL	15:117db924cf7c	1695
wolfSSL	15:117db924cf7c	1696	/*
wolfSSL	15:117db924cf7c	1697	* map byte array block to cipher state
wolfSSL	15:117db924cf7c	1698	* and add initial round key:
wolfSSL	15:117db924cf7c	1699	*/
wolfSSL	15:117db924cf7c	1700	XMEMCPY(&s0, inBlock, sizeof(s0));
wolfSSL	15:117db924cf7c	1701	XMEMCPY(&s1, inBlock + sizeof(s0), sizeof(s1));
wolfSSL	15:117db924cf7c	1702	XMEMCPY(&s2, inBlock + 2 * sizeof(s0), sizeof(s2));
wolfSSL	15:117db924cf7c	1703	XMEMCPY(&s3, inBlock + 3 * sizeof(s0), sizeof(s3));
wolfSSL	15:117db924cf7c	1704
wolfSSL	15:117db924cf7c	1705	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	1706	s0 = ByteReverseWord32(s0);
wolfSSL	15:117db924cf7c	1707	s1 = ByteReverseWord32(s1);
wolfSSL	15:117db924cf7c	1708	s2 = ByteReverseWord32(s2);
wolfSSL	15:117db924cf7c	1709	s3 = ByteReverseWord32(s3);
wolfSSL	15:117db924cf7c	1710	#endif
wolfSSL	15:117db924cf7c	1711
wolfSSL	15:117db924cf7c	1712	s0 ^= rk[0];
wolfSSL	15:117db924cf7c	1713	s1 ^= rk[1];
wolfSSL	15:117db924cf7c	1714	s2 ^= rk[2];
wolfSSL	15:117db924cf7c	1715	s3 ^= rk[3];
wolfSSL	15:117db924cf7c	1716
wolfSSL	15:117db924cf7c	1717	s0 \|= PreFetchTd();
wolfSSL	15:117db924cf7c	1718
wolfSSL	15:117db924cf7c	1719	/*
wolfSSL	15:117db924cf7c	1720	* Nr - 1 full rounds:
wolfSSL	15:117db924cf7c	1721	*/
wolfSSL	15:117db924cf7c	1722
wolfSSL	15:117db924cf7c	1723	for (;;) {
wolfSSL	15:117db924cf7c	1724	t0 =
wolfSSL	15:117db924cf7c	1725	Td[0][GETBYTE(s0, 3)] ^
wolfSSL	15:117db924cf7c	1726	Td[1][GETBYTE(s3, 2)] ^
wolfSSL	15:117db924cf7c	1727	Td[2][GETBYTE(s2, 1)] ^
wolfSSL	15:117db924cf7c	1728	Td[3][GETBYTE(s1, 0)] ^
wolfSSL	15:117db924cf7c	1729	rk[4];
wolfSSL	15:117db924cf7c	1730	t1 =
wolfSSL	15:117db924cf7c	1731	Td[0][GETBYTE(s1, 3)] ^
wolfSSL	15:117db924cf7c	1732	Td[1][GETBYTE(s0, 2)] ^
wolfSSL	15:117db924cf7c	1733	Td[2][GETBYTE(s3, 1)] ^
wolfSSL	15:117db924cf7c	1734	Td[3][GETBYTE(s2, 0)] ^
wolfSSL	15:117db924cf7c	1735	rk[5];
wolfSSL	15:117db924cf7c	1736	t2 =
wolfSSL	15:117db924cf7c	1737	Td[0][GETBYTE(s2, 3)] ^
wolfSSL	15:117db924cf7c	1738	Td[1][GETBYTE(s1, 2)] ^
wolfSSL	15:117db924cf7c	1739	Td[2][GETBYTE(s0, 1)] ^
wolfSSL	15:117db924cf7c	1740	Td[3][GETBYTE(s3, 0)] ^
wolfSSL	15:117db924cf7c	1741	rk[6];
wolfSSL	15:117db924cf7c	1742	t3 =
wolfSSL	15:117db924cf7c	1743	Td[0][GETBYTE(s3, 3)] ^
wolfSSL	15:117db924cf7c	1744	Td[1][GETBYTE(s2, 2)] ^
wolfSSL	15:117db924cf7c	1745	Td[2][GETBYTE(s1, 1)] ^
wolfSSL	15:117db924cf7c	1746	Td[3][GETBYTE(s0, 0)] ^
wolfSSL	15:117db924cf7c	1747	rk[7];
wolfSSL	15:117db924cf7c	1748
wolfSSL	15:117db924cf7c	1749	rk += 8;
wolfSSL	15:117db924cf7c	1750	if (--r == 0) {
wolfSSL	15:117db924cf7c	1751	break;
wolfSSL	15:117db924cf7c	1752	}
wolfSSL	15:117db924cf7c	1753
wolfSSL	15:117db924cf7c	1754	s0 =
wolfSSL	15:117db924cf7c	1755	Td[0][GETBYTE(t0, 3)] ^
wolfSSL	15:117db924cf7c	1756	Td[1][GETBYTE(t3, 2)] ^
wolfSSL	15:117db924cf7c	1757	Td[2][GETBYTE(t2, 1)] ^
wolfSSL	15:117db924cf7c	1758	Td[3][GETBYTE(t1, 0)] ^
wolfSSL	15:117db924cf7c	1759	rk[0];
wolfSSL	15:117db924cf7c	1760	s1 =
wolfSSL	15:117db924cf7c	1761	Td[0][GETBYTE(t1, 3)] ^
wolfSSL	15:117db924cf7c	1762	Td[1][GETBYTE(t0, 2)] ^
wolfSSL	15:117db924cf7c	1763	Td[2][GETBYTE(t3, 1)] ^
wolfSSL	15:117db924cf7c	1764	Td[3][GETBYTE(t2, 0)] ^
wolfSSL	15:117db924cf7c	1765	rk[1];
wolfSSL	15:117db924cf7c	1766	s2 =
wolfSSL	15:117db924cf7c	1767	Td[0][GETBYTE(t2, 3)] ^
wolfSSL	15:117db924cf7c	1768	Td[1][GETBYTE(t1, 2)] ^
wolfSSL	15:117db924cf7c	1769	Td[2][GETBYTE(t0, 1)] ^
wolfSSL	15:117db924cf7c	1770	Td[3][GETBYTE(t3, 0)] ^
wolfSSL	15:117db924cf7c	1771	rk[2];
wolfSSL	15:117db924cf7c	1772	s3 =
wolfSSL	15:117db924cf7c	1773	Td[0][GETBYTE(t3, 3)] ^
wolfSSL	15:117db924cf7c	1774	Td[1][GETBYTE(t2, 2)] ^
wolfSSL	15:117db924cf7c	1775	Td[2][GETBYTE(t1, 1)] ^
wolfSSL	15:117db924cf7c	1776	Td[3][GETBYTE(t0, 0)] ^
wolfSSL	15:117db924cf7c	1777	rk[3];
wolfSSL	15:117db924cf7c	1778	}
wolfSSL	15:117db924cf7c	1779	/*
wolfSSL	15:117db924cf7c	1780	* apply last round and
wolfSSL	15:117db924cf7c	1781	* map cipher state to byte array block:
wolfSSL	15:117db924cf7c	1782	*/
wolfSSL	15:117db924cf7c	1783
wolfSSL	15:117db924cf7c	1784	t0 \|= PreFetchTd4();
wolfSSL	15:117db924cf7c	1785
wolfSSL	15:117db924cf7c	1786	s0 =
wolfSSL	15:117db924cf7c	1787	((word32)Td4[GETBYTE(t0, 3)] << 24) ^
wolfSSL	15:117db924cf7c	1788	((word32)Td4[GETBYTE(t3, 2)] << 16) ^
wolfSSL	15:117db924cf7c	1789	((word32)Td4[GETBYTE(t2, 1)] << 8) ^
wolfSSL	15:117db924cf7c	1790	((word32)Td4[GETBYTE(t1, 0)]) ^
wolfSSL	15:117db924cf7c	1791	rk[0];
wolfSSL	15:117db924cf7c	1792	s1 =
wolfSSL	15:117db924cf7c	1793	((word32)Td4[GETBYTE(t1, 3)] << 24) ^
wolfSSL	15:117db924cf7c	1794	((word32)Td4[GETBYTE(t0, 2)] << 16) ^
wolfSSL	15:117db924cf7c	1795	((word32)Td4[GETBYTE(t3, 1)] << 8) ^
wolfSSL	15:117db924cf7c	1796	((word32)Td4[GETBYTE(t2, 0)]) ^
wolfSSL	15:117db924cf7c	1797	rk[1];
wolfSSL	15:117db924cf7c	1798	s2 =
wolfSSL	15:117db924cf7c	1799	((word32)Td4[GETBYTE(t2, 3)] << 24) ^
wolfSSL	15:117db924cf7c	1800	((word32)Td4[GETBYTE(t1, 2)] << 16) ^
wolfSSL	15:117db924cf7c	1801	((word32)Td4[GETBYTE(t0, 1)] << 8) ^
wolfSSL	15:117db924cf7c	1802	((word32)Td4[GETBYTE(t3, 0)]) ^
wolfSSL	15:117db924cf7c	1803	rk[2];
wolfSSL	15:117db924cf7c	1804	s3 =
wolfSSL	15:117db924cf7c	1805	((word32)Td4[GETBYTE(t3, 3)] << 24) ^
wolfSSL	15:117db924cf7c	1806	((word32)Td4[GETBYTE(t2, 2)] << 16) ^
wolfSSL	15:117db924cf7c	1807	((word32)Td4[GETBYTE(t1, 1)] << 8) ^
wolfSSL	15:117db924cf7c	1808	((word32)Td4[GETBYTE(t0, 0)]) ^
wolfSSL	15:117db924cf7c	1809	rk[3];
wolfSSL	15:117db924cf7c	1810
wolfSSL	15:117db924cf7c	1811	/* write out */
wolfSSL	15:117db924cf7c	1812	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	1813	s0 = ByteReverseWord32(s0);
wolfSSL	15:117db924cf7c	1814	s1 = ByteReverseWord32(s1);
wolfSSL	15:117db924cf7c	1815	s2 = ByteReverseWord32(s2);
wolfSSL	15:117db924cf7c	1816	s3 = ByteReverseWord32(s3);
wolfSSL	15:117db924cf7c	1817	#endif
wolfSSL	15:117db924cf7c	1818
wolfSSL	15:117db924cf7c	1819	XMEMCPY(outBlock, &s0, sizeof(s0));
wolfSSL	15:117db924cf7c	1820	XMEMCPY(outBlock + sizeof(s0), &s1, sizeof(s1));
wolfSSL	15:117db924cf7c	1821	XMEMCPY(outBlock + 2 * sizeof(s0), &s2, sizeof(s2));
wolfSSL	15:117db924cf7c	1822	XMEMCPY(outBlock + 3 * sizeof(s0), &s3, sizeof(s3));
wolfSSL	15:117db924cf7c	1823	}
wolfSSL	15:117db924cf7c	1824	#endif /* HAVE_AES_CBC \|\| WOLFSSL_AES_DIRECT */
wolfSSL	15:117db924cf7c	1825	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	1826
wolfSSL	15:117db924cf7c	1827	#endif /* NEED_AES_TABLES */
wolfSSL	15:117db924cf7c	1828
wolfSSL	15:117db924cf7c	1829
wolfSSL	15:117db924cf7c	1830
wolfSSL	15:117db924cf7c	1831	/* wc_AesSetKey */
wolfSSL	15:117db924cf7c	1832	#if defined(STM32_CRYPTO)
wolfSSL	15:117db924cf7c	1833
wolfSSL	15:117db924cf7c	1834	int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	1835	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	1836	{
wolfSSL	15:117db924cf7c	1837	word32 *rk = aes->key;
wolfSSL	15:117db924cf7c	1838
wolfSSL	15:117db924cf7c	1839	(void)dir;
wolfSSL	15:117db924cf7c	1840
wolfSSL	15:117db924cf7c	1841	if (!((keylen == 16) \|\| (keylen == 24) \|\| (keylen == 32)))
wolfSSL	15:117db924cf7c	1842	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	1843
wolfSSL	15:117db924cf7c	1844	aes->keylen = keylen;
wolfSSL	15:117db924cf7c	1845	aes->rounds = keylen/4 + 6;
wolfSSL	15:117db924cf7c	1846	XMEMCPY(rk, userKey, keylen);
wolfSSL	15:117db924cf7c	1847	#ifndef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	1848	ByteReverseWords(rk, rk, keylen);
wolfSSL	15:117db924cf7c	1849	#endif
wolfSSL	15:117db924cf7c	1850	#if defined(WOLFSSL_AES_CFB) \|\| defined(WOLFSSL_AES_COUNTER)
wolfSSL	15:117db924cf7c	1851	aes->left = 0;
wolfSSL	15:117db924cf7c	1852	#endif
wolfSSL	15:117db924cf7c	1853
wolfSSL	15:117db924cf7c	1854	return wc_AesSetIV(aes, iv);
wolfSSL	15:117db924cf7c	1855	}
wolfSSL	15:117db924cf7c	1856	#if defined(WOLFSSL_AES_DIRECT)
wolfSSL	15:117db924cf7c	1857	int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	1858	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	1859	{
wolfSSL	15:117db924cf7c	1860	return wc_AesSetKey(aes, userKey, keylen, iv, dir);
wolfSSL	15:117db924cf7c	1861	}
wolfSSL	15:117db924cf7c	1862	#endif
wolfSSL	15:117db924cf7c	1863
wolfSSL	15:117db924cf7c	1864	#elif defined(HAVE_COLDFIRE_SEC)
wolfSSL	15:117db924cf7c	1865	#if defined (HAVE_THREADX)
wolfSSL	15:117db924cf7c	1866	#include "memory_pools.h"
wolfSSL	15:117db924cf7c	1867	extern TX_BYTE_POOL mp_ncached; /* Non Cached memory pool */
wolfSSL	15:117db924cf7c	1868	#endif
wolfSSL	15:117db924cf7c	1869
wolfSSL	15:117db924cf7c	1870	#define AES_BUFFER_SIZE (AES_BLOCK_SIZE * 64)
wolfSSL	15:117db924cf7c	1871	static unsigned char *AESBuffIn = NULL;
wolfSSL	15:117db924cf7c	1872	static unsigned char *AESBuffOut = NULL;
wolfSSL	15:117db924cf7c	1873	static byte *secReg;
wolfSSL	15:117db924cf7c	1874	static byte *secKey;
wolfSSL	15:117db924cf7c	1875	static volatile SECdescriptorType *secDesc;
wolfSSL	15:117db924cf7c	1876
wolfSSL	15:117db924cf7c	1877	static wolfSSL_Mutex Mutex_AesSEC;
wolfSSL	15:117db924cf7c	1878
wolfSSL	15:117db924cf7c	1879	#define SEC_DESC_AES_CBC_ENCRYPT 0x60300010
wolfSSL	15:117db924cf7c	1880	#define SEC_DESC_AES_CBC_DECRYPT 0x60200010
wolfSSL	15:117db924cf7c	1881
wolfSSL	15:117db924cf7c	1882	extern volatile unsigned char __MBAR[];
wolfSSL	15:117db924cf7c	1883
wolfSSL	15:117db924cf7c	1884	int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	1885	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	1886	{
wolfSSL	15:117db924cf7c	1887	if (AESBuffIn == NULL) {
wolfSSL	15:117db924cf7c	1888	#if defined (HAVE_THREADX)
wolfSSL	15:117db924cf7c	1889	int s1, s2, s3, s4, s5;
wolfSSL	15:117db924cf7c	1890	s5 = tx_byte_allocate(&mp_ncached,(void *)&secDesc,
wolfSSL	15:117db924cf7c	1891	sizeof(SECdescriptorType), TX_NO_WAIT);
wolfSSL	15:117db924cf7c	1892	s1 = tx_byte_allocate(&mp_ncached, (void *)&AESBuffIn,
wolfSSL	15:117db924cf7c	1893	AES_BUFFER_SIZE, TX_NO_WAIT);
wolfSSL	15:117db924cf7c	1894	s2 = tx_byte_allocate(&mp_ncached, (void *)&AESBuffOut,
wolfSSL	15:117db924cf7c	1895	AES_BUFFER_SIZE, TX_NO_WAIT);
wolfSSL	15:117db924cf7c	1896	s3 = tx_byte_allocate(&mp_ncached, (void *)&secKey,
wolfSSL	15:117db924cf7c	1897	AES_BLOCK_SIZE*2, TX_NO_WAIT);
wolfSSL	15:117db924cf7c	1898	s4 = tx_byte_allocate(&mp_ncached, (void *)&secReg,
wolfSSL	15:117db924cf7c	1899	AES_BLOCK_SIZE, TX_NO_WAIT);
wolfSSL	15:117db924cf7c	1900
wolfSSL	15:117db924cf7c	1901	if (s1 \|\| s2 \|\| s3 \|\| s4 \|\| s5)
wolfSSL	15:117db924cf7c	1902	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	1903	#else
wolfSSL	15:117db924cf7c	1904	#warning "Allocate non-Cache buffers"
wolfSSL	15:117db924cf7c	1905	#endif
wolfSSL	15:117db924cf7c	1906
wolfSSL	15:117db924cf7c	1907	wc_InitMutex(&Mutex_AesSEC);
wolfSSL	15:117db924cf7c	1908	}
wolfSSL	15:117db924cf7c	1909
wolfSSL	15:117db924cf7c	1910	if (!((keylen == 16) \|\| (keylen == 24) \|\| (keylen == 32)))
wolfSSL	15:117db924cf7c	1911	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	1912
wolfSSL	15:117db924cf7c	1913	if (aes == NULL)
wolfSSL	15:117db924cf7c	1914	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	1915
wolfSSL	15:117db924cf7c	1916	aes->keylen = keylen;
wolfSSL	15:117db924cf7c	1917	aes->rounds = keylen/4 + 6;
wolfSSL	15:117db924cf7c	1918	XMEMCPY(aes->key, userKey, keylen);
wolfSSL	15:117db924cf7c	1919
wolfSSL	15:117db924cf7c	1920	if (iv)
wolfSSL	15:117db924cf7c	1921	XMEMCPY(aes->reg, iv, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	1922
wolfSSL	15:117db924cf7c	1923	#if defined(WOLFSSL_AES_CFB) \|\| defined(WOLFSSL_AES_COUNTER)
wolfSSL	15:117db924cf7c	1924	aes->left = 0;
wolfSSL	15:117db924cf7c	1925	#endif
wolfSSL	15:117db924cf7c	1926
wolfSSL	15:117db924cf7c	1927	return 0;
wolfSSL	15:117db924cf7c	1928	}
wolfSSL	15:117db924cf7c	1929	#elif defined(FREESCALE_LTC)
wolfSSL	15:117db924cf7c	1930	int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen, const byte* iv,
wolfSSL	15:117db924cf7c	1931	int dir)
wolfSSL	15:117db924cf7c	1932	{
wolfSSL	15:117db924cf7c	1933	if (!((keylen == 16) \|\| (keylen == 24) \|\| (keylen == 32)))
wolfSSL	15:117db924cf7c	1934	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	1935
wolfSSL	15:117db924cf7c	1936	aes->rounds = keylen/4 + 6;
wolfSSL	15:117db924cf7c	1937	XMEMCPY(aes->key, userKey, keylen);
wolfSSL	15:117db924cf7c	1938
wolfSSL	15:117db924cf7c	1939	#if defined(WOLFSSL_AES_CFB) \|\| defined(WOLFSSL_AES_COUNTER)
wolfSSL	15:117db924cf7c	1940	aes->left = 0;
wolfSSL	15:117db924cf7c	1941	#endif
wolfSSL	15:117db924cf7c	1942
wolfSSL	15:117db924cf7c	1943	return wc_AesSetIV(aes, iv);
wolfSSL	15:117db924cf7c	1944	}
wolfSSL	15:117db924cf7c	1945
wolfSSL	15:117db924cf7c	1946	int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	1947	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	1948	{
wolfSSL	15:117db924cf7c	1949	return wc_AesSetKey(aes, userKey, keylen, iv, dir);
wolfSSL	15:117db924cf7c	1950	}
wolfSSL	15:117db924cf7c	1951	#elif defined(FREESCALE_MMCAU)
wolfSSL	15:117db924cf7c	1952	int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	1953	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	1954	{
wolfSSL	15:117db924cf7c	1955	int ret;
wolfSSL	15:117db924cf7c	1956	byte rk = (byte)aes->key;
wolfSSL	15:117db924cf7c	1957
wolfSSL	15:117db924cf7c	1958	(void)dir;
wolfSSL	15:117db924cf7c	1959
wolfSSL	15:117db924cf7c	1960	if (!((keylen == 16) \|\| (keylen == 24) \|\| (keylen == 32)))
wolfSSL	15:117db924cf7c	1961	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	1962
wolfSSL	15:117db924cf7c	1963	if (rk == NULL)
wolfSSL	15:117db924cf7c	1964	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	1965
wolfSSL	15:117db924cf7c	1966	#if defined(WOLFSSL_AES_CFB) \|\| defined(WOLFSSL_AES_COUNTER)
wolfSSL	15:117db924cf7c	1967	aes->left = 0;
wolfSSL	15:117db924cf7c	1968	#endif
wolfSSL	15:117db924cf7c	1969
wolfSSL	15:117db924cf7c	1970	aes->rounds = keylen/4 + 6;
wolfSSL	15:117db924cf7c	1971
wolfSSL	15:117db924cf7c	1972	ret = wolfSSL_CryptHwMutexLock();
wolfSSL	15:117db924cf7c	1973	if(ret == 0) {
wolfSSL	15:117db924cf7c	1974	#ifdef FREESCALE_MMCAU_CLASSIC
wolfSSL	15:117db924cf7c	1975	cau_aes_set_key(userKey, keylen*8, rk);
wolfSSL	15:117db924cf7c	1976	#else
wolfSSL	15:117db924cf7c	1977	MMCAU_AES_SetKey(userKey, keylen, rk);
wolfSSL	15:117db924cf7c	1978	#endif
wolfSSL	15:117db924cf7c	1979	wolfSSL_CryptHwMutexUnLock();
wolfSSL	15:117db924cf7c	1980
wolfSSL	15:117db924cf7c	1981	ret = wc_AesSetIV(aes, iv);
wolfSSL	15:117db924cf7c	1982	}
wolfSSL	15:117db924cf7c	1983
wolfSSL	15:117db924cf7c	1984	return ret;
wolfSSL	15:117db924cf7c	1985	}
wolfSSL	15:117db924cf7c	1986
wolfSSL	15:117db924cf7c	1987	int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	1988	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	1989	{
wolfSSL	15:117db924cf7c	1990	return wc_AesSetKey(aes, userKey, keylen, iv, dir);
wolfSSL	15:117db924cf7c	1991	}
wolfSSL	15:117db924cf7c	1992
wolfSSL	15:117db924cf7c	1993	#elif defined(WOLFSSL_NRF51_AES)
wolfSSL	15:117db924cf7c	1994	int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	1995	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	1996	{
wolfSSL	15:117db924cf7c	1997	int ret;
wolfSSL	15:117db924cf7c	1998
wolfSSL	15:117db924cf7c	1999	(void)dir;
wolfSSL	15:117db924cf7c	2000	(void)iv;
wolfSSL	15:117db924cf7c	2001
wolfSSL	15:117db924cf7c	2002	if (keylen != 16)
wolfSSL	15:117db924cf7c	2003	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2004
wolfSSL	15:117db924cf7c	2005	aes->keylen = keylen;
wolfSSL	15:117db924cf7c	2006	aes->rounds = keylen/4 + 6;
wolfSSL	15:117db924cf7c	2007	ret = nrf51_aes_set_key(userKey);
wolfSSL	15:117db924cf7c	2008
wolfSSL	15:117db924cf7c	2009	#if defined(WOLFSSL_AES_CFB) \|\| defined(WOLFSSL_AES_COUNTER)
wolfSSL	15:117db924cf7c	2010	aes->left = 0;
wolfSSL	15:117db924cf7c	2011	#endif
wolfSSL	15:117db924cf7c	2012
wolfSSL	15:117db924cf7c	2013	return ret;
wolfSSL	15:117db924cf7c	2014	}
wolfSSL	15:117db924cf7c	2015
wolfSSL	15:117db924cf7c	2016	int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	2017	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	2018	{
wolfSSL	15:117db924cf7c	2019	return wc_AesSetKey(aes, userKey, keylen, iv, dir);
wolfSSL	15:117db924cf7c	2020	}
wolfSSL	15:117db924cf7c	2021
wolfSSL	15:117db924cf7c	2022	#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES)
wolfSSL	15:117db924cf7c	2023	/* implemented in wolfcrypt/src/port/caam/caam_aes.c */
wolfSSL	15:117db924cf7c	2024
wolfSSL	15:117db924cf7c	2025	#else
wolfSSL	15:117db924cf7c	2026	static int wc_AesSetKeyLocal(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	2027	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	2028	{
wolfSSL	15:117db924cf7c	2029	word32 *rk = aes->key;
wolfSSL	15:117db924cf7c	2030	#ifdef NEED_AES_TABLES
wolfSSL	15:117db924cf7c	2031	word32 temp;
wolfSSL	15:117db924cf7c	2032	unsigned int i = 0;
wolfSSL	15:117db924cf7c	2033	#endif
wolfSSL	15:117db924cf7c	2034
wolfSSL	15:117db924cf7c	2035	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	2036	aes->use_aesni = 0;
wolfSSL	15:117db924cf7c	2037	#endif /* WOLFSSL_AESNI */
wolfSSL	15:117db924cf7c	2038	#if defined(WOLFSSL_AES_CFB) \|\| defined(WOLFSSL_AES_COUNTER)
wolfSSL	15:117db924cf7c	2039	aes->left = 0;
wolfSSL	15:117db924cf7c	2040	#endif
wolfSSL	15:117db924cf7c	2041
wolfSSL	15:117db924cf7c	2042	aes->keylen = keylen;
wolfSSL	15:117db924cf7c	2043	aes->rounds = (keylen/4) + 6;
wolfSSL	15:117db924cf7c	2044
wolfSSL	15:117db924cf7c	2045	XMEMCPY(rk, userKey, keylen);
wolfSSL	15:117db924cf7c	2046	#if defined(LITTLE_ENDIAN_ORDER) && !defined(WOLFSSL_PIC32MZ_CRYPT)
wolfSSL	15:117db924cf7c	2047	ByteReverseWords(rk, rk, keylen);
wolfSSL	15:117db924cf7c	2048	#endif
wolfSSL	15:117db924cf7c	2049
wolfSSL	15:117db924cf7c	2050	#ifdef NEED_AES_TABLES
wolfSSL	15:117db924cf7c	2051
wolfSSL	15:117db924cf7c	2052	switch (keylen) {
wolfSSL	15:117db924cf7c	2053	#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 128 && \
wolfSSL	15:117db924cf7c	2054	defined(WOLFSSL_AES_128)
wolfSSL	15:117db924cf7c	2055	case 16:
wolfSSL	15:117db924cf7c	2056	while (1)
wolfSSL	15:117db924cf7c	2057	{
wolfSSL	15:117db924cf7c	2058	temp = rk[3];
wolfSSL	15:117db924cf7c	2059	rk[4] = rk[0] ^
wolfSSL	15:117db924cf7c	2060	(Te[2][GETBYTE(temp, 2)] & 0xff000000) ^
wolfSSL	15:117db924cf7c	2061	(Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^
wolfSSL	15:117db924cf7c	2062	(Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^
wolfSSL	15:117db924cf7c	2063	(Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^
wolfSSL	15:117db924cf7c	2064	rcon[i];
wolfSSL	15:117db924cf7c	2065	rk[5] = rk[1] ^ rk[4];
wolfSSL	15:117db924cf7c	2066	rk[6] = rk[2] ^ rk[5];
wolfSSL	15:117db924cf7c	2067	rk[7] = rk[3] ^ rk[6];
wolfSSL	15:117db924cf7c	2068	if (++i == 10)
wolfSSL	15:117db924cf7c	2069	break;
wolfSSL	15:117db924cf7c	2070	rk += 4;
wolfSSL	15:117db924cf7c	2071	}
wolfSSL	15:117db924cf7c	2072	break;
wolfSSL	15:117db924cf7c	2073	#endif /* 128 */
wolfSSL	15:117db924cf7c	2074
wolfSSL	15:117db924cf7c	2075	#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 192 && \
wolfSSL	15:117db924cf7c	2076	defined(WOLFSSL_AES_192)
wolfSSL	15:117db924cf7c	2077	case 24:
wolfSSL	15:117db924cf7c	2078	/* for (;;) here triggers a bug in VC60 SP4 w/ Pro Pack */
wolfSSL	15:117db924cf7c	2079	while (1)
wolfSSL	15:117db924cf7c	2080	{
wolfSSL	15:117db924cf7c	2081	temp = rk[ 5];
wolfSSL	15:117db924cf7c	2082	rk[ 6] = rk[ 0] ^
wolfSSL	15:117db924cf7c	2083	(Te[2][GETBYTE(temp, 2)] & 0xff000000) ^
wolfSSL	15:117db924cf7c	2084	(Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^
wolfSSL	15:117db924cf7c	2085	(Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^
wolfSSL	15:117db924cf7c	2086	(Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^
wolfSSL	15:117db924cf7c	2087	rcon[i];
wolfSSL	15:117db924cf7c	2088	rk[ 7] = rk[ 1] ^ rk[ 6];
wolfSSL	15:117db924cf7c	2089	rk[ 8] = rk[ 2] ^ rk[ 7];
wolfSSL	15:117db924cf7c	2090	rk[ 9] = rk[ 3] ^ rk[ 8];
wolfSSL	15:117db924cf7c	2091	if (++i == 8)
wolfSSL	15:117db924cf7c	2092	break;
wolfSSL	15:117db924cf7c	2093	rk[10] = rk[ 4] ^ rk[ 9];
wolfSSL	15:117db924cf7c	2094	rk[11] = rk[ 5] ^ rk[10];
wolfSSL	15:117db924cf7c	2095	rk += 6;
wolfSSL	15:117db924cf7c	2096	}
wolfSSL	15:117db924cf7c	2097	break;
wolfSSL	15:117db924cf7c	2098	#endif /* 192 */
wolfSSL	15:117db924cf7c	2099
wolfSSL	15:117db924cf7c	2100	#if defined(AES_MAX_KEY_SIZE) && AES_MAX_KEY_SIZE >= 256 && \
wolfSSL	15:117db924cf7c	2101	defined(WOLFSSL_AES_256)
wolfSSL	15:117db924cf7c	2102	case 32:
wolfSSL	15:117db924cf7c	2103	while (1)
wolfSSL	15:117db924cf7c	2104	{
wolfSSL	15:117db924cf7c	2105	temp = rk[ 7];
wolfSSL	15:117db924cf7c	2106	rk[ 8] = rk[ 0] ^
wolfSSL	15:117db924cf7c	2107	(Te[2][GETBYTE(temp, 2)] & 0xff000000) ^
wolfSSL	15:117db924cf7c	2108	(Te[3][GETBYTE(temp, 1)] & 0x00ff0000) ^
wolfSSL	15:117db924cf7c	2109	(Te[0][GETBYTE(temp, 0)] & 0x0000ff00) ^
wolfSSL	15:117db924cf7c	2110	(Te[1][GETBYTE(temp, 3)] & 0x000000ff) ^
wolfSSL	15:117db924cf7c	2111	rcon[i];
wolfSSL	15:117db924cf7c	2112	rk[ 9] = rk[ 1] ^ rk[ 8];
wolfSSL	15:117db924cf7c	2113	rk[10] = rk[ 2] ^ rk[ 9];
wolfSSL	15:117db924cf7c	2114	rk[11] = rk[ 3] ^ rk[10];
wolfSSL	15:117db924cf7c	2115	if (++i == 7)
wolfSSL	15:117db924cf7c	2116	break;
wolfSSL	15:117db924cf7c	2117	temp = rk[11];
wolfSSL	15:117db924cf7c	2118	rk[12] = rk[ 4] ^
wolfSSL	15:117db924cf7c	2119	(Te[2][GETBYTE(temp, 3)] & 0xff000000) ^
wolfSSL	15:117db924cf7c	2120	(Te[3][GETBYTE(temp, 2)] & 0x00ff0000) ^
wolfSSL	15:117db924cf7c	2121	(Te[0][GETBYTE(temp, 1)] & 0x0000ff00) ^
wolfSSL	15:117db924cf7c	2122	(Te[1][GETBYTE(temp, 0)] & 0x000000ff);
wolfSSL	15:117db924cf7c	2123	rk[13] = rk[ 5] ^ rk[12];
wolfSSL	15:117db924cf7c	2124	rk[14] = rk[ 6] ^ rk[13];
wolfSSL	15:117db924cf7c	2125	rk[15] = rk[ 7] ^ rk[14];
wolfSSL	15:117db924cf7c	2126
wolfSSL	15:117db924cf7c	2127	rk += 8;
wolfSSL	15:117db924cf7c	2128	}
wolfSSL	15:117db924cf7c	2129	break;
wolfSSL	15:117db924cf7c	2130	#endif /* 256 */
wolfSSL	15:117db924cf7c	2131
wolfSSL	15:117db924cf7c	2132	default:
wolfSSL	15:117db924cf7c	2133	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2134	} /* switch */
wolfSSL	15:117db924cf7c	2135
wolfSSL	15:117db924cf7c	2136	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2137	if (dir == AES_DECRYPTION) {
wolfSSL	15:117db924cf7c	2138	unsigned int j;
wolfSSL	15:117db924cf7c	2139	rk = aes->key;
wolfSSL	15:117db924cf7c	2140
wolfSSL	15:117db924cf7c	2141	/* invert the order of the round keys: */
wolfSSL	15:117db924cf7c	2142	for (i = 0, j = 4* aes->rounds; i < j; i += 4, j -= 4) {
wolfSSL	15:117db924cf7c	2143	temp = rk[i ]; rk[i ] = rk[j ]; rk[j ] = temp;
wolfSSL	15:117db924cf7c	2144	temp = rk[i + 1]; rk[i + 1] = rk[j + 1]; rk[j + 1] = temp;
wolfSSL	15:117db924cf7c	2145	temp = rk[i + 2]; rk[i + 2] = rk[j + 2]; rk[j + 2] = temp;
wolfSSL	15:117db924cf7c	2146	temp = rk[i + 3]; rk[i + 3] = rk[j + 3]; rk[j + 3] = temp;
wolfSSL	15:117db924cf7c	2147	}
wolfSSL	15:117db924cf7c	2148	/* apply the inverse MixColumn transform to all round keys but the
wolfSSL	15:117db924cf7c	2149	first and the last: */
wolfSSL	15:117db924cf7c	2150	for (i = 1; i < aes->rounds; i++) {
wolfSSL	15:117db924cf7c	2151	rk += 4;
wolfSSL	15:117db924cf7c	2152	rk[0] =
wolfSSL	15:117db924cf7c	2153	Td[0][Te[1][GETBYTE(rk[0], 3)] & 0xff] ^
wolfSSL	15:117db924cf7c	2154	Td[1][Te[1][GETBYTE(rk[0], 2)] & 0xff] ^
wolfSSL	15:117db924cf7c	2155	Td[2][Te[1][GETBYTE(rk[0], 1)] & 0xff] ^
wolfSSL	15:117db924cf7c	2156	Td[3][Te[1][GETBYTE(rk[0], 0)] & 0xff];
wolfSSL	15:117db924cf7c	2157	rk[1] =
wolfSSL	15:117db924cf7c	2158	Td[0][Te[1][GETBYTE(rk[1], 3)] & 0xff] ^
wolfSSL	15:117db924cf7c	2159	Td[1][Te[1][GETBYTE(rk[1], 2)] & 0xff] ^
wolfSSL	15:117db924cf7c	2160	Td[2][Te[1][GETBYTE(rk[1], 1)] & 0xff] ^
wolfSSL	15:117db924cf7c	2161	Td[3][Te[1][GETBYTE(rk[1], 0)] & 0xff];
wolfSSL	15:117db924cf7c	2162	rk[2] =
wolfSSL	15:117db924cf7c	2163	Td[0][Te[1][GETBYTE(rk[2], 3)] & 0xff] ^
wolfSSL	15:117db924cf7c	2164	Td[1][Te[1][GETBYTE(rk[2], 2)] & 0xff] ^
wolfSSL	15:117db924cf7c	2165	Td[2][Te[1][GETBYTE(rk[2], 1)] & 0xff] ^
wolfSSL	15:117db924cf7c	2166	Td[3][Te[1][GETBYTE(rk[2], 0)] & 0xff];
wolfSSL	15:117db924cf7c	2167	rk[3] =
wolfSSL	15:117db924cf7c	2168	Td[0][Te[1][GETBYTE(rk[3], 3)] & 0xff] ^
wolfSSL	15:117db924cf7c	2169	Td[1][Te[1][GETBYTE(rk[3], 2)] & 0xff] ^
wolfSSL	15:117db924cf7c	2170	Td[2][Te[1][GETBYTE(rk[3], 1)] & 0xff] ^
wolfSSL	15:117db924cf7c	2171	Td[3][Te[1][GETBYTE(rk[3], 0)] & 0xff];
wolfSSL	15:117db924cf7c	2172	}
wolfSSL	15:117db924cf7c	2173	}
wolfSSL	15:117db924cf7c	2174	#else
wolfSSL	15:117db924cf7c	2175	(void)dir;
wolfSSL	15:117db924cf7c	2176	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	2177	#endif /* NEED_AES_TABLES */
wolfSSL	15:117db924cf7c	2178
wolfSSL	15:117db924cf7c	2179	return wc_AesSetIV(aes, iv);
wolfSSL	15:117db924cf7c	2180	}
wolfSSL	15:117db924cf7c	2181
wolfSSL	15:117db924cf7c	2182	int wc_AesSetKey(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	2183	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	2184	{
wolfSSL	15:117db924cf7c	2185	int ret;
wolfSSL	15:117db924cf7c	2186	#if defined(AES_MAX_KEY_SIZE)
wolfSSL	15:117db924cf7c	2187	const word32 max_key_len = (AES_MAX_KEY_SIZE / 8);
wolfSSL	15:117db924cf7c	2188	#endif
wolfSSL	15:117db924cf7c	2189
wolfSSL	15:117db924cf7c	2190	#ifdef WOLFSSL_IMX6_CAAM_BLOB
wolfSSL	15:117db924cf7c	2191	byte local[32];
wolfSSL	15:117db924cf7c	2192	word32 localSz = 32;
wolfSSL	15:117db924cf7c	2193
wolfSSL	15:117db924cf7c	2194	if (keylen == (16 + WC_CAAM_BLOB_SZ) \|\|
wolfSSL	15:117db924cf7c	2195	keylen == (24 + WC_CAAM_BLOB_SZ) \|\|
wolfSSL	15:117db924cf7c	2196	keylen == (32 + WC_CAAM_BLOB_SZ)) {
wolfSSL	15:117db924cf7c	2197	if (wc_caamOpenBlob((byte*)userKey, keylen, local, &localSz) != 0) {
wolfSSL	15:117db924cf7c	2198	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2199	}
wolfSSL	15:117db924cf7c	2200
wolfSSL	15:117db924cf7c	2201	/* set local values */
wolfSSL	15:117db924cf7c	2202	userKey = local;
wolfSSL	15:117db924cf7c	2203	keylen = localSz;
wolfSSL	15:117db924cf7c	2204	}
wolfSSL	15:117db924cf7c	2205	#endif
wolfSSL	15:117db924cf7c	2206	if (aes == NULL \|\|
wolfSSL	15:117db924cf7c	2207	!((keylen == 16) \|\| (keylen == 24) \|\| (keylen == 32))) {
wolfSSL	15:117db924cf7c	2208	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2209	}
wolfSSL	15:117db924cf7c	2210
wolfSSL	15:117db924cf7c	2211	#if defined(AES_MAX_KEY_SIZE)
wolfSSL	15:117db924cf7c	2212	/* Check key length */
wolfSSL	15:117db924cf7c	2213	if (keylen > max_key_len) {
wolfSSL	15:117db924cf7c	2214	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2215	}
wolfSSL	15:117db924cf7c	2216	#endif
wolfSSL	15:117db924cf7c	2217	aes->keylen = keylen;
wolfSSL	15:117db924cf7c	2218	aes->rounds = keylen/4 + 6;
wolfSSL	15:117db924cf7c	2219
wolfSSL	15:117db924cf7c	2220	#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
wolfSSL	15:117db924cf7c	2221	if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES) {
wolfSSL	15:117db924cf7c	2222	XMEMCPY(aes->asyncKey, userKey, keylen);
wolfSSL	15:117db924cf7c	2223	if (iv)
wolfSSL	15:117db924cf7c	2224	XMEMCPY(aes->asyncIv, iv, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2225	}
wolfSSL	15:117db924cf7c	2226	#endif /* WOLFSSL_ASYNC_CRYPT */
wolfSSL	15:117db924cf7c	2227
wolfSSL	15:117db924cf7c	2228	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	2229	if (checkAESNI == 0) {
wolfSSL	15:117db924cf7c	2230	haveAESNI = Check_CPU_support_AES();
wolfSSL	15:117db924cf7c	2231	checkAESNI = 1;
wolfSSL	15:117db924cf7c	2232	}
wolfSSL	15:117db924cf7c	2233	if (haveAESNI) {
wolfSSL	15:117db924cf7c	2234	#if defined(WOLFSSL_AES_COUNTER) \|\| defined(WOLFSSL_AES_CFB)
wolfSSL	15:117db924cf7c	2235	aes->left = 0;
wolfSSL	15:117db924cf7c	2236	#endif /* WOLFSSL_AES_COUNTER */
wolfSSL	15:117db924cf7c	2237	aes->use_aesni = 1;
wolfSSL	15:117db924cf7c	2238	if (iv)
wolfSSL	15:117db924cf7c	2239	XMEMCPY(aes->reg, iv, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2240	if (dir == AES_ENCRYPTION)
wolfSSL	15:117db924cf7c	2241	return AES_set_encrypt_key(userKey, keylen * 8, aes);
wolfSSL	15:117db924cf7c	2242	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2243	else
wolfSSL	15:117db924cf7c	2244	return AES_set_decrypt_key(userKey, keylen * 8, aes);
wolfSSL	15:117db924cf7c	2245	#endif
wolfSSL	15:117db924cf7c	2246	}
wolfSSL	15:117db924cf7c	2247	#endif /* WOLFSSL_AESNI */
wolfSSL	15:117db924cf7c	2248
wolfSSL	15:117db924cf7c	2249	ret = wc_AesSetKeyLocal(aes, userKey, keylen, iv, dir);
wolfSSL	15:117db924cf7c	2250
wolfSSL	15:117db924cf7c	2251	#ifdef WOLFSSL_IMX6_CAAM_BLOB
wolfSSL	15:117db924cf7c	2252	ForceZero(local, sizeof(local));
wolfSSL	15:117db924cf7c	2253	#endif
wolfSSL	15:117db924cf7c	2254	return ret;
wolfSSL	15:117db924cf7c	2255	}
wolfSSL	15:117db924cf7c	2256
wolfSSL	15:117db924cf7c	2257	#if defined(WOLFSSL_AES_DIRECT) \|\| defined(WOLFSSL_AES_COUNTER)
wolfSSL	15:117db924cf7c	2258	/* AES-CTR and AES-DIRECT need to use this for key setup, no aesni yet */
wolfSSL	15:117db924cf7c	2259	int wc_AesSetKeyDirect(Aes* aes, const byte* userKey, word32 keylen,
wolfSSL	15:117db924cf7c	2260	const byte* iv, int dir)
wolfSSL	15:117db924cf7c	2261	{
wolfSSL	15:117db924cf7c	2262	int ret;
wolfSSL	15:117db924cf7c	2263
wolfSSL	15:117db924cf7c	2264	#ifdef WOLFSSL_IMX6_CAAM_BLOB
wolfSSL	15:117db924cf7c	2265	byte local[32];
wolfSSL	15:117db924cf7c	2266	word32 localSz = 32;
wolfSSL	15:117db924cf7c	2267
wolfSSL	15:117db924cf7c	2268	if (keylen == (16 + WC_CAAM_BLOB_SZ) \|\|
wolfSSL	15:117db924cf7c	2269	keylen == (24 + WC_CAAM_BLOB_SZ) \|\|
wolfSSL	15:117db924cf7c	2270	keylen == (32 + WC_CAAM_BLOB_SZ)) {
wolfSSL	15:117db924cf7c	2271	if (wc_caamOpenBlob((byte*)userKey, keylen, local, &localSz)
wolfSSL	15:117db924cf7c	2272	!= 0) {
wolfSSL	15:117db924cf7c	2273	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2274	}
wolfSSL	15:117db924cf7c	2275
wolfSSL	15:117db924cf7c	2276	/* set local values */
wolfSSL	15:117db924cf7c	2277	userKey = local;
wolfSSL	15:117db924cf7c	2278	keylen = localSz;
wolfSSL	15:117db924cf7c	2279	}
wolfSSL	15:117db924cf7c	2280	#endif
wolfSSL	15:117db924cf7c	2281	ret = wc_AesSetKeyLocal(aes, userKey, keylen, iv, dir);
wolfSSL	15:117db924cf7c	2282
wolfSSL	15:117db924cf7c	2283	#ifdef WOLFSSL_IMX6_CAAM_BLOB
wolfSSL	15:117db924cf7c	2284	ForceZero(local, sizeof(local));
wolfSSL	15:117db924cf7c	2285	#endif
wolfSSL	15:117db924cf7c	2286
wolfSSL	15:117db924cf7c	2287	return ret;
wolfSSL	15:117db924cf7c	2288	}
wolfSSL	15:117db924cf7c	2289	#endif /* WOLFSSL_AES_DIRECT \|\| WOLFSSL_AES_COUNTER */
wolfSSL	15:117db924cf7c	2290	#endif /* wc_AesSetKey block */
wolfSSL	15:117db924cf7c	2291
wolfSSL	15:117db924cf7c	2292
wolfSSL	15:117db924cf7c	2293	/* wc_AesSetIV is shared between software and hardware */
wolfSSL	15:117db924cf7c	2294	int wc_AesSetIV(Aes* aes, const byte* iv)
wolfSSL	15:117db924cf7c	2295	{
wolfSSL	15:117db924cf7c	2296	if (aes == NULL)
wolfSSL	15:117db924cf7c	2297	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2298
wolfSSL	15:117db924cf7c	2299	if (iv)
wolfSSL	15:117db924cf7c	2300	XMEMCPY(aes->reg, iv, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2301	else
wolfSSL	15:117db924cf7c	2302	XMEMSET(aes->reg, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2303
wolfSSL	15:117db924cf7c	2304	return 0;
wolfSSL	15:117db924cf7c	2305	}
wolfSSL	15:117db924cf7c	2306
wolfSSL	15:117db924cf7c	2307	/* AES-DIRECT */
wolfSSL	15:117db924cf7c	2308	#if defined(WOLFSSL_AES_DIRECT)
wolfSSL	15:117db924cf7c	2309	#if defined(HAVE_COLDFIRE_SEC)
wolfSSL	15:117db924cf7c	2310	#error "Coldfire SEC doesn't yet support AES direct"
wolfSSL	15:117db924cf7c	2311
wolfSSL	15:117db924cf7c	2312	#elif defined(FREESCALE_LTC)
wolfSSL	15:117db924cf7c	2313	/* Allow direct access to one block encrypt */
wolfSSL	15:117db924cf7c	2314	void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
wolfSSL	15:117db924cf7c	2315	{
wolfSSL	15:117db924cf7c	2316	byte *key;
wolfSSL	15:117db924cf7c	2317	uint32_t keySize;
wolfSSL	15:117db924cf7c	2318
wolfSSL	15:117db924cf7c	2319	key = (byte*)aes->key;
wolfSSL	15:117db924cf7c	2320	wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	2321
wolfSSL	15:117db924cf7c	2322	LTC_AES_EncryptEcb(LTC_BASE, in, out, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	2323	key, keySize);
wolfSSL	15:117db924cf7c	2324	}
wolfSSL	15:117db924cf7c	2325
wolfSSL	15:117db924cf7c	2326	/* Allow direct access to one block decrypt */
wolfSSL	15:117db924cf7c	2327	void wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
wolfSSL	15:117db924cf7c	2328	{
wolfSSL	15:117db924cf7c	2329	byte *key;
wolfSSL	15:117db924cf7c	2330	uint32_t keySize;
wolfSSL	15:117db924cf7c	2331
wolfSSL	15:117db924cf7c	2332	key = (byte*)aes->key;
wolfSSL	15:117db924cf7c	2333	wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	2334
wolfSSL	15:117db924cf7c	2335	LTC_AES_DecryptEcb(LTC_BASE, in, out, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	2336	key, keySize, kLTC_EncryptKey);
wolfSSL	15:117db924cf7c	2337	}
wolfSSL	15:117db924cf7c	2338
wolfSSL	15:117db924cf7c	2339	#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES)
wolfSSL	15:117db924cf7c	2340	/* implemented in wolfcrypt/src/port/caam/caam_aes.c */
wolfSSL	15:117db924cf7c	2341
wolfSSL	15:117db924cf7c	2342	#else
wolfSSL	15:117db924cf7c	2343	/* Allow direct access to one block encrypt */
wolfSSL	15:117db924cf7c	2344	void wc_AesEncryptDirect(Aes* aes, byte* out, const byte* in)
wolfSSL	15:117db924cf7c	2345	{
wolfSSL	15:117db924cf7c	2346	wc_AesEncrypt(aes, in, out);
wolfSSL	15:117db924cf7c	2347	}
wolfSSL	15:117db924cf7c	2348	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2349	/* Allow direct access to one block decrypt */
wolfSSL	15:117db924cf7c	2350	void wc_AesDecryptDirect(Aes* aes, byte* out, const byte* in)
wolfSSL	15:117db924cf7c	2351	{
wolfSSL	15:117db924cf7c	2352	wc_AesDecrypt(aes, in, out);
wolfSSL	15:117db924cf7c	2353	}
wolfSSL	15:117db924cf7c	2354	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	2355	#endif /* AES direct block */
wolfSSL	15:117db924cf7c	2356	#endif /* WOLFSSL_AES_DIRECT */
wolfSSL	15:117db924cf7c	2357
wolfSSL	15:117db924cf7c	2358
wolfSSL	15:117db924cf7c	2359	/* AES-CBC */
wolfSSL	15:117db924cf7c	2360	#ifdef HAVE_AES_CBC
wolfSSL	15:117db924cf7c	2361	#if defined(STM32_CRYPTO)
wolfSSL	15:117db924cf7c	2362
wolfSSL	15:117db924cf7c	2363	#ifdef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	2364	int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2365	{
wolfSSL	15:117db924cf7c	2366	int ret = 0;
wolfSSL	15:117db924cf7c	2367	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2368	CRYP_HandleTypeDef hcryp;
wolfSSL	15:117db924cf7c	2369
wolfSSL	15:117db924cf7c	2370	XMEMSET(&hcryp, 0, sizeof(CRYP_HandleTypeDef));
wolfSSL	15:117db924cf7c	2371	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	2372	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	2373	hcryp.Init.KeySize = CRYP_KEYSIZE_128B;
wolfSSL	15:117db924cf7c	2374	break;
wolfSSL	15:117db924cf7c	2375	#ifdef CRYP_KEYSIZE_192B
wolfSSL	15:117db924cf7c	2376	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	2377	hcryp.Init.KeySize = CRYP_KEYSIZE_192B;
wolfSSL	15:117db924cf7c	2378	break;
wolfSSL	15:117db924cf7c	2379	#endif
wolfSSL	15:117db924cf7c	2380	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	2381	hcryp.Init.KeySize = CRYP_KEYSIZE_256B;
wolfSSL	15:117db924cf7c	2382	break;
wolfSSL	15:117db924cf7c	2383	default:
wolfSSL	15:117db924cf7c	2384	break;
wolfSSL	15:117db924cf7c	2385	}
wolfSSL	15:117db924cf7c	2386	hcryp.Instance = CRYP;
wolfSSL	15:117db924cf7c	2387	hcryp.Init.DataType = CRYP_DATATYPE_8B;
wolfSSL	15:117db924cf7c	2388	hcryp.Init.pKey = (uint8_t*)aes->key;
wolfSSL	15:117db924cf7c	2389	hcryp.Init.pInitVect = (uint8_t*)aes->reg;
wolfSSL	15:117db924cf7c	2390
wolfSSL	15:117db924cf7c	2391	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	2392
wolfSSL	15:117db924cf7c	2393	while (blocks--) {
wolfSSL	15:117db924cf7c	2394	if (HAL_CRYP_AESCBC_Encrypt(&hcryp, (uint8_t*)in, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	2395	out, STM32_HAL_TIMEOUT) != HAL_OK) {
wolfSSL	15:117db924cf7c	2396	ret = WC_TIMEOUT_E;
wolfSSL	15:117db924cf7c	2397	break;
wolfSSL	15:117db924cf7c	2398	}
wolfSSL	15:117db924cf7c	2399
wolfSSL	15:117db924cf7c	2400	/* store iv for next call */
wolfSSL	15:117db924cf7c	2401	XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2402
wolfSSL	15:117db924cf7c	2403	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2404	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2405	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2406	}
wolfSSL	15:117db924cf7c	2407
wolfSSL	15:117db924cf7c	2408	HAL_CRYP_DeInit(&hcryp);
wolfSSL	15:117db924cf7c	2409
wolfSSL	15:117db924cf7c	2410	return ret;
wolfSSL	15:117db924cf7c	2411	}
wolfSSL	15:117db924cf7c	2412	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2413	int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2414	{
wolfSSL	15:117db924cf7c	2415	int ret = 0;
wolfSSL	15:117db924cf7c	2416	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2417	CRYP_HandleTypeDef hcryp;
wolfSSL	15:117db924cf7c	2418
wolfSSL	15:117db924cf7c	2419	XMEMSET(&hcryp, 0, sizeof(CRYP_HandleTypeDef));
wolfSSL	15:117db924cf7c	2420	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	2421	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	2422	hcryp.Init.KeySize = CRYP_KEYSIZE_128B;
wolfSSL	15:117db924cf7c	2423	break;
wolfSSL	15:117db924cf7c	2424	#ifdef CRYP_KEYSIZE_192B
wolfSSL	15:117db924cf7c	2425	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	2426	hcryp.Init.KeySize = CRYP_KEYSIZE_192B;
wolfSSL	15:117db924cf7c	2427	break;
wolfSSL	15:117db924cf7c	2428	#endif
wolfSSL	15:117db924cf7c	2429	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	2430	hcryp.Init.KeySize = CRYP_KEYSIZE_256B;
wolfSSL	15:117db924cf7c	2431	break;
wolfSSL	15:117db924cf7c	2432	default:
wolfSSL	15:117db924cf7c	2433	break;
wolfSSL	15:117db924cf7c	2434	}
wolfSSL	15:117db924cf7c	2435	hcryp.Instance = CRYP;
wolfSSL	15:117db924cf7c	2436	hcryp.Init.DataType = CRYP_DATATYPE_8B;
wolfSSL	15:117db924cf7c	2437	hcryp.Init.pKey = (uint8_t*)aes->key;
wolfSSL	15:117db924cf7c	2438	hcryp.Init.pInitVect = (uint8_t*)aes->reg;
wolfSSL	15:117db924cf7c	2439
wolfSSL	15:117db924cf7c	2440	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	2441
wolfSSL	15:117db924cf7c	2442	while (blocks--) {
wolfSSL	15:117db924cf7c	2443	if (HAL_CRYP_AESCBC_Decrypt(&hcryp, (uint8_t*)in, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	2444	out, STM32_HAL_TIMEOUT) != HAL_OK) {
wolfSSL	15:117db924cf7c	2445	ret = WC_TIMEOUT_E;
wolfSSL	15:117db924cf7c	2446	}
wolfSSL	15:117db924cf7c	2447
wolfSSL	15:117db924cf7c	2448	/* store iv for next call */
wolfSSL	15:117db924cf7c	2449	XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2450
wolfSSL	15:117db924cf7c	2451	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2452	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2453	}
wolfSSL	15:117db924cf7c	2454
wolfSSL	15:117db924cf7c	2455	HAL_CRYP_DeInit(&hcryp);
wolfSSL	15:117db924cf7c	2456
wolfSSL	15:117db924cf7c	2457	return ret;
wolfSSL	15:117db924cf7c	2458	}
wolfSSL	15:117db924cf7c	2459	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	2460	#else
wolfSSL	15:117db924cf7c	2461	int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2462	{
wolfSSL	15:117db924cf7c	2463	word32 enc_key, iv;
wolfSSL	15:117db924cf7c	2464	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2465	CRYP_InitTypeDef AES_CRYP_InitStructure;
wolfSSL	15:117db924cf7c	2466	CRYP_KeyInitTypeDef AES_CRYP_KeyInitStructure;
wolfSSL	15:117db924cf7c	2467	CRYP_IVInitTypeDef AES_CRYP_IVInitStructure;
wolfSSL	15:117db924cf7c	2468
wolfSSL	15:117db924cf7c	2469	enc_key = aes->key;
wolfSSL	15:117db924cf7c	2470	iv = aes->reg;
wolfSSL	15:117db924cf7c	2471
wolfSSL	15:117db924cf7c	2472	/* crypto structure initialization */
wolfSSL	15:117db924cf7c	2473	CRYP_KeyStructInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	2474	CRYP_StructInit(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	2475	CRYP_IVStructInit(&AES_CRYP_IVInitStructure);
wolfSSL	15:117db924cf7c	2476
wolfSSL	15:117db924cf7c	2477	/* reset registers to their default values */
wolfSSL	15:117db924cf7c	2478	CRYP_DeInit();
wolfSSL	15:117db924cf7c	2479
wolfSSL	15:117db924cf7c	2480	/* load key into correct registers */
wolfSSL	15:117db924cf7c	2481	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	2482	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	2483	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_128b;
wolfSSL	15:117db924cf7c	2484	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[0];
wolfSSL	15:117db924cf7c	2485	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[1];
wolfSSL	15:117db924cf7c	2486	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[2];
wolfSSL	15:117db924cf7c	2487	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[3];
wolfSSL	15:117db924cf7c	2488	break;
wolfSSL	15:117db924cf7c	2489
wolfSSL	15:117db924cf7c	2490	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	2491	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_192b;
wolfSSL	15:117db924cf7c	2492	AES_CRYP_KeyInitStructure.CRYP_Key1Left = enc_key[0];
wolfSSL	15:117db924cf7c	2493	AES_CRYP_KeyInitStructure.CRYP_Key1Right = enc_key[1];
wolfSSL	15:117db924cf7c	2494	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[2];
wolfSSL	15:117db924cf7c	2495	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[3];
wolfSSL	15:117db924cf7c	2496	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[4];
wolfSSL	15:117db924cf7c	2497	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[5];
wolfSSL	15:117db924cf7c	2498	break;
wolfSSL	15:117db924cf7c	2499
wolfSSL	15:117db924cf7c	2500	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	2501	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_256b;
wolfSSL	15:117db924cf7c	2502	AES_CRYP_KeyInitStructure.CRYP_Key0Left = enc_key[0];
wolfSSL	15:117db924cf7c	2503	AES_CRYP_KeyInitStructure.CRYP_Key0Right = enc_key[1];
wolfSSL	15:117db924cf7c	2504	AES_CRYP_KeyInitStructure.CRYP_Key1Left = enc_key[2];
wolfSSL	15:117db924cf7c	2505	AES_CRYP_KeyInitStructure.CRYP_Key1Right = enc_key[3];
wolfSSL	15:117db924cf7c	2506	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[4];
wolfSSL	15:117db924cf7c	2507	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[5];
wolfSSL	15:117db924cf7c	2508	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[6];
wolfSSL	15:117db924cf7c	2509	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[7];
wolfSSL	15:117db924cf7c	2510	break;
wolfSSL	15:117db924cf7c	2511
wolfSSL	15:117db924cf7c	2512	default:
wolfSSL	15:117db924cf7c	2513	break;
wolfSSL	15:117db924cf7c	2514	}
wolfSSL	15:117db924cf7c	2515	CRYP_KeyInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	2516
wolfSSL	15:117db924cf7c	2517	/* set iv */
wolfSSL	15:117db924cf7c	2518	ByteReverseWords(iv, iv, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2519	AES_CRYP_IVInitStructure.CRYP_IV0Left = iv[0];
wolfSSL	15:117db924cf7c	2520	AES_CRYP_IVInitStructure.CRYP_IV0Right = iv[1];
wolfSSL	15:117db924cf7c	2521	AES_CRYP_IVInitStructure.CRYP_IV1Left = iv[2];
wolfSSL	15:117db924cf7c	2522	AES_CRYP_IVInitStructure.CRYP_IV1Right = iv[3];
wolfSSL	15:117db924cf7c	2523	CRYP_IVInit(&AES_CRYP_IVInitStructure);
wolfSSL	15:117db924cf7c	2524
wolfSSL	15:117db924cf7c	2525	/* set direction, mode, and datatype */
wolfSSL	15:117db924cf7c	2526	AES_CRYP_InitStructure.CRYP_AlgoDir = CRYP_AlgoDir_Encrypt;
wolfSSL	15:117db924cf7c	2527	AES_CRYP_InitStructure.CRYP_AlgoMode = CRYP_AlgoMode_AES_CBC;
wolfSSL	15:117db924cf7c	2528	AES_CRYP_InitStructure.CRYP_DataType = CRYP_DataType_8b;
wolfSSL	15:117db924cf7c	2529	CRYP_Init(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	2530
wolfSSL	15:117db924cf7c	2531	/* enable crypto processor */
wolfSSL	15:117db924cf7c	2532	CRYP_Cmd(ENABLE);
wolfSSL	15:117db924cf7c	2533
wolfSSL	15:117db924cf7c	2534	while (blocks--) {
wolfSSL	15:117db924cf7c	2535	/* flush IN/OUT FIFOs */
wolfSSL	15:117db924cf7c	2536	CRYP_FIFOFlush();
wolfSSL	15:117db924cf7c	2537
wolfSSL	15:117db924cf7c	2538	CRYP_DataIn((uint32_t)&in[0]);
wolfSSL	15:117db924cf7c	2539	CRYP_DataIn((uint32_t)&in[4]);
wolfSSL	15:117db924cf7c	2540	CRYP_DataIn((uint32_t)&in[8]);
wolfSSL	15:117db924cf7c	2541	CRYP_DataIn((uint32_t)&in[12]);
wolfSSL	15:117db924cf7c	2542
wolfSSL	15:117db924cf7c	2543	/* wait until the complete message has been processed */
wolfSSL	15:117db924cf7c	2544	while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
wolfSSL	15:117db924cf7c	2545
wolfSSL	15:117db924cf7c	2546	(uint32_t)&out[0] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	2547	(uint32_t)&out[4] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	2548	(uint32_t)&out[8] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	2549	(uint32_t)&out[12] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	2550
wolfSSL	15:117db924cf7c	2551	/* store iv for next call */
wolfSSL	15:117db924cf7c	2552	XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2553
wolfSSL	15:117db924cf7c	2554	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2555	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2556	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2557	}
wolfSSL	15:117db924cf7c	2558
wolfSSL	15:117db924cf7c	2559	/* disable crypto processor */
wolfSSL	15:117db924cf7c	2560	CRYP_Cmd(DISABLE);
wolfSSL	15:117db924cf7c	2561
wolfSSL	15:117db924cf7c	2562	return 0;
wolfSSL	15:117db924cf7c	2563	}
wolfSSL	15:117db924cf7c	2564
wolfSSL	15:117db924cf7c	2565	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2566	int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2567	{
wolfSSL	15:117db924cf7c	2568	word32 dec_key, iv;
wolfSSL	15:117db924cf7c	2569	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2570	CRYP_InitTypeDef AES_CRYP_InitStructure;
wolfSSL	15:117db924cf7c	2571	CRYP_KeyInitTypeDef AES_CRYP_KeyInitStructure;
wolfSSL	15:117db924cf7c	2572	CRYP_IVInitTypeDef AES_CRYP_IVInitStructure;
wolfSSL	15:117db924cf7c	2573
wolfSSL	15:117db924cf7c	2574	dec_key = aes->key;
wolfSSL	15:117db924cf7c	2575	iv = aes->reg;
wolfSSL	15:117db924cf7c	2576
wolfSSL	15:117db924cf7c	2577	/* crypto structure initialization */
wolfSSL	15:117db924cf7c	2578	CRYP_KeyStructInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	2579	CRYP_StructInit(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	2580	CRYP_IVStructInit(&AES_CRYP_IVInitStructure);
wolfSSL	15:117db924cf7c	2581
wolfSSL	15:117db924cf7c	2582	/* if input and output same will overwrite input iv */
wolfSSL	15:117db924cf7c	2583	XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2584
wolfSSL	15:117db924cf7c	2585	/* reset registers to their default values */
wolfSSL	15:117db924cf7c	2586	CRYP_DeInit();
wolfSSL	15:117db924cf7c	2587
wolfSSL	15:117db924cf7c	2588	/* load key into correct registers */
wolfSSL	15:117db924cf7c	2589	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	2590	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	2591	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_128b;
wolfSSL	15:117db924cf7c	2592	AES_CRYP_KeyInitStructure.CRYP_Key2Left = dec_key[0];
wolfSSL	15:117db924cf7c	2593	AES_CRYP_KeyInitStructure.CRYP_Key2Right = dec_key[1];
wolfSSL	15:117db924cf7c	2594	AES_CRYP_KeyInitStructure.CRYP_Key3Left = dec_key[2];
wolfSSL	15:117db924cf7c	2595	AES_CRYP_KeyInitStructure.CRYP_Key3Right = dec_key[3];
wolfSSL	15:117db924cf7c	2596	break;
wolfSSL	15:117db924cf7c	2597
wolfSSL	15:117db924cf7c	2598	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	2599	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_192b;
wolfSSL	15:117db924cf7c	2600	AES_CRYP_KeyInitStructure.CRYP_Key1Left = dec_key[0];
wolfSSL	15:117db924cf7c	2601	AES_CRYP_KeyInitStructure.CRYP_Key1Right = dec_key[1];
wolfSSL	15:117db924cf7c	2602	AES_CRYP_KeyInitStructure.CRYP_Key2Left = dec_key[2];
wolfSSL	15:117db924cf7c	2603	AES_CRYP_KeyInitStructure.CRYP_Key2Right = dec_key[3];
wolfSSL	15:117db924cf7c	2604	AES_CRYP_KeyInitStructure.CRYP_Key3Left = dec_key[4];
wolfSSL	15:117db924cf7c	2605	AES_CRYP_KeyInitStructure.CRYP_Key3Right = dec_key[5];
wolfSSL	15:117db924cf7c	2606	break;
wolfSSL	15:117db924cf7c	2607
wolfSSL	15:117db924cf7c	2608	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	2609	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_256b;
wolfSSL	15:117db924cf7c	2610	AES_CRYP_KeyInitStructure.CRYP_Key0Left = dec_key[0];
wolfSSL	15:117db924cf7c	2611	AES_CRYP_KeyInitStructure.CRYP_Key0Right = dec_key[1];
wolfSSL	15:117db924cf7c	2612	AES_CRYP_KeyInitStructure.CRYP_Key1Left = dec_key[2];
wolfSSL	15:117db924cf7c	2613	AES_CRYP_KeyInitStructure.CRYP_Key1Right = dec_key[3];
wolfSSL	15:117db924cf7c	2614	AES_CRYP_KeyInitStructure.CRYP_Key2Left = dec_key[4];
wolfSSL	15:117db924cf7c	2615	AES_CRYP_KeyInitStructure.CRYP_Key2Right = dec_key[5];
wolfSSL	15:117db924cf7c	2616	AES_CRYP_KeyInitStructure.CRYP_Key3Left = dec_key[6];
wolfSSL	15:117db924cf7c	2617	AES_CRYP_KeyInitStructure.CRYP_Key3Right = dec_key[7];
wolfSSL	15:117db924cf7c	2618	break;
wolfSSL	15:117db924cf7c	2619
wolfSSL	15:117db924cf7c	2620	default:
wolfSSL	15:117db924cf7c	2621	break;
wolfSSL	15:117db924cf7c	2622	}
wolfSSL	15:117db924cf7c	2623
wolfSSL	15:117db924cf7c	2624	/* set direction, mode, and datatype for key preparation */
wolfSSL	15:117db924cf7c	2625	AES_CRYP_InitStructure.CRYP_AlgoDir = CRYP_AlgoDir_Decrypt;
wolfSSL	15:117db924cf7c	2626	AES_CRYP_InitStructure.CRYP_AlgoMode = CRYP_AlgoMode_AES_Key;
wolfSSL	15:117db924cf7c	2627	AES_CRYP_InitStructure.CRYP_DataType = CRYP_DataType_32b;
wolfSSL	15:117db924cf7c	2628	CRYP_Init(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	2629	CRYP_KeyInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	2630
wolfSSL	15:117db924cf7c	2631	/* enable crypto processor */
wolfSSL	15:117db924cf7c	2632	CRYP_Cmd(ENABLE);
wolfSSL	15:117db924cf7c	2633
wolfSSL	15:117db924cf7c	2634	/* wait until key has been prepared */
wolfSSL	15:117db924cf7c	2635	while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
wolfSSL	15:117db924cf7c	2636
wolfSSL	15:117db924cf7c	2637	/* set direction, mode, and datatype for decryption */
wolfSSL	15:117db924cf7c	2638	AES_CRYP_InitStructure.CRYP_AlgoDir = CRYP_AlgoDir_Decrypt;
wolfSSL	15:117db924cf7c	2639	AES_CRYP_InitStructure.CRYP_AlgoMode = CRYP_AlgoMode_AES_CBC;
wolfSSL	15:117db924cf7c	2640	AES_CRYP_InitStructure.CRYP_DataType = CRYP_DataType_8b;
wolfSSL	15:117db924cf7c	2641	CRYP_Init(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	2642
wolfSSL	15:117db924cf7c	2643	/* set iv */
wolfSSL	15:117db924cf7c	2644	ByteReverseWords(iv, iv, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2645
wolfSSL	15:117db924cf7c	2646	AES_CRYP_IVInitStructure.CRYP_IV0Left = iv[0];
wolfSSL	15:117db924cf7c	2647	AES_CRYP_IVInitStructure.CRYP_IV0Right = iv[1];
wolfSSL	15:117db924cf7c	2648	AES_CRYP_IVInitStructure.CRYP_IV1Left = iv[2];
wolfSSL	15:117db924cf7c	2649	AES_CRYP_IVInitStructure.CRYP_IV1Right = iv[3];
wolfSSL	15:117db924cf7c	2650	CRYP_IVInit(&AES_CRYP_IVInitStructure);
wolfSSL	15:117db924cf7c	2651
wolfSSL	15:117db924cf7c	2652	/* enable crypto processor */
wolfSSL	15:117db924cf7c	2653	CRYP_Cmd(ENABLE);
wolfSSL	15:117db924cf7c	2654
wolfSSL	15:117db924cf7c	2655	while (blocks--) {
wolfSSL	15:117db924cf7c	2656	/* flush IN/OUT FIFOs */
wolfSSL	15:117db924cf7c	2657	CRYP_FIFOFlush();
wolfSSL	15:117db924cf7c	2658
wolfSSL	15:117db924cf7c	2659	CRYP_DataIn((uint32_t)&in[0]);
wolfSSL	15:117db924cf7c	2660	CRYP_DataIn((uint32_t)&in[4]);
wolfSSL	15:117db924cf7c	2661	CRYP_DataIn((uint32_t)&in[8]);
wolfSSL	15:117db924cf7c	2662	CRYP_DataIn((uint32_t)&in[12]);
wolfSSL	15:117db924cf7c	2663
wolfSSL	15:117db924cf7c	2664	/* wait until the complete message has been processed */
wolfSSL	15:117db924cf7c	2665	while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
wolfSSL	15:117db924cf7c	2666
wolfSSL	15:117db924cf7c	2667	(uint32_t)&out[0] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	2668	(uint32_t)&out[4] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	2669	(uint32_t)&out[8] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	2670	(uint32_t)&out[12] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	2671
wolfSSL	15:117db924cf7c	2672	/* store iv for next call */
wolfSSL	15:117db924cf7c	2673	XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2674
wolfSSL	15:117db924cf7c	2675	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2676	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2677	}
wolfSSL	15:117db924cf7c	2678
wolfSSL	15:117db924cf7c	2679	/* disable crypto processor */
wolfSSL	15:117db924cf7c	2680	CRYP_Cmd(DISABLE);
wolfSSL	15:117db924cf7c	2681
wolfSSL	15:117db924cf7c	2682	return 0;
wolfSSL	15:117db924cf7c	2683	}
wolfSSL	15:117db924cf7c	2684	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	2685	#endif /* WOLFSSL_STM32_CUBEMX */
wolfSSL	15:117db924cf7c	2686
wolfSSL	15:117db924cf7c	2687	#elif defined(HAVE_COLDFIRE_SEC)
wolfSSL	15:117db924cf7c	2688	static int wc_AesCbcCrypt(Aes* aes, byte* po, const byte* pi, word32 sz,
wolfSSL	15:117db924cf7c	2689	word32 descHeader)
wolfSSL	15:117db924cf7c	2690	{
wolfSSL	15:117db924cf7c	2691	#ifdef DEBUG_WOLFSSL
wolfSSL	15:117db924cf7c	2692	int i; int stat1, stat2; int ret;
wolfSSL	15:117db924cf7c	2693	#endif
wolfSSL	15:117db924cf7c	2694
wolfSSL	15:117db924cf7c	2695	int size;
wolfSSL	15:117db924cf7c	2696	volatile int v;
wolfSSL	15:117db924cf7c	2697
wolfSSL	15:117db924cf7c	2698	if ((pi == NULL) \|\| (po == NULL))
wolfSSL	15:117db924cf7c	2699	return BAD_FUNC_ARG; /wrong pointer/
wolfSSL	15:117db924cf7c	2700
wolfSSL	15:117db924cf7c	2701	wc_LockMutex(&Mutex_AesSEC);
wolfSSL	15:117db924cf7c	2702
wolfSSL	15:117db924cf7c	2703	/* Set descriptor for SEC */
wolfSSL	15:117db924cf7c	2704	secDesc->length1 = 0x0;
wolfSSL	15:117db924cf7c	2705	secDesc->pointer1 = NULL;
wolfSSL	15:117db924cf7c	2706
wolfSSL	15:117db924cf7c	2707	secDesc->length2 = AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2708	secDesc->pointer2 = (byte )secReg; / Initial Vector */
wolfSSL	15:117db924cf7c	2709
wolfSSL	15:117db924cf7c	2710	switch(aes->rounds) {
wolfSSL	15:117db924cf7c	2711	case 10: secDesc->length3 = 16; break;
wolfSSL	15:117db924cf7c	2712	case 12: secDesc->length3 = 24; break;
wolfSSL	15:117db924cf7c	2713	case 14: secDesc->length3 = 32; break;
wolfSSL	15:117db924cf7c	2714	}
wolfSSL	15:117db924cf7c	2715	XMEMCPY(secKey, aes->key, secDesc->length3);
wolfSSL	15:117db924cf7c	2716
wolfSSL	15:117db924cf7c	2717	secDesc->pointer3 = (byte *)secKey;
wolfSSL	15:117db924cf7c	2718	secDesc->pointer4 = AESBuffIn;
wolfSSL	15:117db924cf7c	2719	secDesc->pointer5 = AESBuffOut;
wolfSSL	15:117db924cf7c	2720	secDesc->length6 = 0x0;
wolfSSL	15:117db924cf7c	2721	secDesc->pointer6 = NULL;
wolfSSL	15:117db924cf7c	2722	secDesc->length7 = 0x0;
wolfSSL	15:117db924cf7c	2723	secDesc->pointer7 = NULL;
wolfSSL	15:117db924cf7c	2724	secDesc->nextDescriptorPtr = NULL;
wolfSSL	15:117db924cf7c	2725
wolfSSL	15:117db924cf7c	2726	while (sz) {
wolfSSL	15:117db924cf7c	2727	secDesc->header = descHeader;
wolfSSL	15:117db924cf7c	2728	XMEMCPY(secReg, aes->reg, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2729	if ((sz % AES_BUFFER_SIZE) == sz) {
wolfSSL	15:117db924cf7c	2730	size = sz;
wolfSSL	15:117db924cf7c	2731	sz = 0;
wolfSSL	15:117db924cf7c	2732	} else {
wolfSSL	15:117db924cf7c	2733	size = AES_BUFFER_SIZE;
wolfSSL	15:117db924cf7c	2734	sz -= AES_BUFFER_SIZE;
wolfSSL	15:117db924cf7c	2735	}
wolfSSL	15:117db924cf7c	2736	secDesc->length4 = size;
wolfSSL	15:117db924cf7c	2737	secDesc->length5 = size;
wolfSSL	15:117db924cf7c	2738
wolfSSL	15:117db924cf7c	2739	XMEMCPY(AESBuffIn, pi, size);
wolfSSL	15:117db924cf7c	2740	if(descHeader == SEC_DESC_AES_CBC_DECRYPT) {
wolfSSL	15:117db924cf7c	2741	XMEMCPY((void)aes->tmp, (void)&(pi[size-AES_BLOCK_SIZE]),
wolfSSL	15:117db924cf7c	2742	AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2743	}
wolfSSL	15:117db924cf7c	2744
wolfSSL	15:117db924cf7c	2745	/* Point SEC to the location of the descriptor */
wolfSSL	15:117db924cf7c	2746	MCF_SEC_FR0 = (uint32)secDesc;
wolfSSL	15:117db924cf7c	2747	/* Initialize SEC and wait for encryption to complete */
wolfSSL	15:117db924cf7c	2748	MCF_SEC_CCCR0 = 0x0000001a;
wolfSSL	15:117db924cf7c	2749	/* poll SISR to determine when channel is complete */
wolfSSL	15:117db924cf7c	2750	v=0;
wolfSSL	15:117db924cf7c	2751
wolfSSL	15:117db924cf7c	2752	while ((secDesc->header>> 24) != 0xff) v++;
wolfSSL	15:117db924cf7c	2753
wolfSSL	15:117db924cf7c	2754	#ifdef DEBUG_WOLFSSL
wolfSSL	15:117db924cf7c	2755	ret = MCF_SEC_SISRH;
wolfSSL	15:117db924cf7c	2756	stat1 = MCF_SEC_AESSR;
wolfSSL	15:117db924cf7c	2757	stat2 = MCF_SEC_AESISR;
wolfSSL	15:117db924cf7c	2758	if (ret & 0xe0000000) {
wolfSSL	15:117db924cf7c	2759	db_printf("Aes_Cbc(i=%d):ISRH=%08x, AESSR=%08x, "
wolfSSL	15:117db924cf7c	2760	"AESISR=%08x\n", i, ret, stat1, stat2);
wolfSSL	15:117db924cf7c	2761	}
wolfSSL	15:117db924cf7c	2762	#endif
wolfSSL	15:117db924cf7c	2763
wolfSSL	15:117db924cf7c	2764	XMEMCPY(po, AESBuffOut, size);
wolfSSL	15:117db924cf7c	2765
wolfSSL	15:117db924cf7c	2766	if (descHeader == SEC_DESC_AES_CBC_ENCRYPT) {
wolfSSL	15:117db924cf7c	2767	XMEMCPY((void)aes->reg, (void)&(po[size-AES_BLOCK_SIZE]),
wolfSSL	15:117db924cf7c	2768	AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2769	} else {
wolfSSL	15:117db924cf7c	2770	XMEMCPY((void)aes->reg, (void)aes->tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2771	}
wolfSSL	15:117db924cf7c	2772
wolfSSL	15:117db924cf7c	2773	pi += size;
wolfSSL	15:117db924cf7c	2774	po += size;
wolfSSL	15:117db924cf7c	2775	}
wolfSSL	15:117db924cf7c	2776
wolfSSL	15:117db924cf7c	2777	wc_UnLockMutex(&Mutex_AesSEC);
wolfSSL	15:117db924cf7c	2778	return 0;
wolfSSL	15:117db924cf7c	2779	}
wolfSSL	15:117db924cf7c	2780
wolfSSL	15:117db924cf7c	2781	int wc_AesCbcEncrypt(Aes* aes, byte* po, const byte* pi, word32 sz)
wolfSSL	15:117db924cf7c	2782	{
wolfSSL	15:117db924cf7c	2783	return (wc_AesCbcCrypt(aes, po, pi, sz, SEC_DESC_AES_CBC_ENCRYPT));
wolfSSL	15:117db924cf7c	2784	}
wolfSSL	15:117db924cf7c	2785
wolfSSL	15:117db924cf7c	2786	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2787	int wc_AesCbcDecrypt(Aes* aes, byte* po, const byte* pi, word32 sz)
wolfSSL	15:117db924cf7c	2788	{
wolfSSL	15:117db924cf7c	2789	return (wc_AesCbcCrypt(aes, po, pi, sz, SEC_DESC_AES_CBC_DECRYPT));
wolfSSL	15:117db924cf7c	2790	}
wolfSSL	15:117db924cf7c	2791	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	2792
wolfSSL	15:117db924cf7c	2793	#elif defined(FREESCALE_LTC)
wolfSSL	15:117db924cf7c	2794	int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2795	{
wolfSSL	15:117db924cf7c	2796	uint32_t keySize;
wolfSSL	15:117db924cf7c	2797	status_t status;
wolfSSL	15:117db924cf7c	2798	byte iv, enc_key;
wolfSSL	15:117db924cf7c	2799	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2800
wolfSSL	15:117db924cf7c	2801	iv = (byte*)aes->reg;
wolfSSL	15:117db924cf7c	2802	enc_key = (byte*)aes->key;
wolfSSL	15:117db924cf7c	2803
wolfSSL	15:117db924cf7c	2804	status = wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	2805	if (status != 0) {
wolfSSL	15:117db924cf7c	2806	return status;
wolfSSL	15:117db924cf7c	2807	}
wolfSSL	15:117db924cf7c	2808
wolfSSL	15:117db924cf7c	2809	status = LTC_AES_EncryptCbc(LTC_BASE, in, out, blocks * AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	2810	iv, enc_key, keySize);
wolfSSL	15:117db924cf7c	2811	return (status == kStatus_Success) ? 0 : -1;
wolfSSL	15:117db924cf7c	2812	}
wolfSSL	15:117db924cf7c	2813
wolfSSL	15:117db924cf7c	2814	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2815	int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2816	{
wolfSSL	15:117db924cf7c	2817	uint32_t keySize;
wolfSSL	15:117db924cf7c	2818	status_t status;
wolfSSL	15:117db924cf7c	2819	byte* iv, *dec_key;
wolfSSL	15:117db924cf7c	2820	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2821
wolfSSL	15:117db924cf7c	2822	iv = (byte*)aes->reg;
wolfSSL	15:117db924cf7c	2823	dec_key = (byte*)aes->key;
wolfSSL	15:117db924cf7c	2824
wolfSSL	15:117db924cf7c	2825	status = wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	2826	if (status != 0) {
wolfSSL	15:117db924cf7c	2827	return status;
wolfSSL	15:117db924cf7c	2828	}
wolfSSL	15:117db924cf7c	2829
wolfSSL	15:117db924cf7c	2830	status = LTC_AES_DecryptCbc(LTC_BASE, in, out, blocks * AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	2831	iv, dec_key, keySize, kLTC_EncryptKey);
wolfSSL	15:117db924cf7c	2832	return (status == kStatus_Success) ? 0 : -1;
wolfSSL	15:117db924cf7c	2833	}
wolfSSL	15:117db924cf7c	2834	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	2835
wolfSSL	15:117db924cf7c	2836	#elif defined(FREESCALE_MMCAU)
wolfSSL	15:117db924cf7c	2837	int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2838	{
wolfSSL	15:117db924cf7c	2839	int i;
wolfSSL	15:117db924cf7c	2840	int offset = 0;
wolfSSL	15:117db924cf7c	2841	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2842	byte *iv;
wolfSSL	15:117db924cf7c	2843	byte temp_block[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	2844
wolfSSL	15:117db924cf7c	2845	iv = (byte*)aes->reg;
wolfSSL	15:117db924cf7c	2846
wolfSSL	15:117db924cf7c	2847	while (blocks--) {
wolfSSL	15:117db924cf7c	2848	XMEMCPY(temp_block, in + offset, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2849
wolfSSL	15:117db924cf7c	2850	/* XOR block with IV for CBC */
wolfSSL	15:117db924cf7c	2851	for (i = 0; i < AES_BLOCK_SIZE; i++)
wolfSSL	15:117db924cf7c	2852	temp_block[i] ^= iv[i];
wolfSSL	15:117db924cf7c	2853
wolfSSL	15:117db924cf7c	2854	wc_AesEncrypt(aes, temp_block, out + offset);
wolfSSL	15:117db924cf7c	2855
wolfSSL	15:117db924cf7c	2856	offset += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2857
wolfSSL	15:117db924cf7c	2858	/* store IV for next block */
wolfSSL	15:117db924cf7c	2859	XMEMCPY(iv, out + offset - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2860	}
wolfSSL	15:117db924cf7c	2861
wolfSSL	15:117db924cf7c	2862	return 0;
wolfSSL	15:117db924cf7c	2863	}
wolfSSL	15:117db924cf7c	2864	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2865	int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2866	{
wolfSSL	15:117db924cf7c	2867	int i;
wolfSSL	15:117db924cf7c	2868	int offset = 0;
wolfSSL	15:117db924cf7c	2869	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2870	byte* iv;
wolfSSL	15:117db924cf7c	2871	byte temp_block[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	2872
wolfSSL	15:117db924cf7c	2873	iv = (byte*)aes->reg;
wolfSSL	15:117db924cf7c	2874
wolfSSL	15:117db924cf7c	2875	while (blocks--) {
wolfSSL	15:117db924cf7c	2876	XMEMCPY(temp_block, in + offset, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2877
wolfSSL	15:117db924cf7c	2878	wc_AesDecrypt(aes, in + offset, out + offset);
wolfSSL	15:117db924cf7c	2879
wolfSSL	15:117db924cf7c	2880	/* XOR block with IV for CBC */
wolfSSL	15:117db924cf7c	2881	for (i = 0; i < AES_BLOCK_SIZE; i++)
wolfSSL	15:117db924cf7c	2882	(out + offset)[i] ^= iv[i];
wolfSSL	15:117db924cf7c	2883
wolfSSL	15:117db924cf7c	2884	/* store IV for next block */
wolfSSL	15:117db924cf7c	2885	XMEMCPY(iv, temp_block, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2886
wolfSSL	15:117db924cf7c	2887	offset += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	2888	}
wolfSSL	15:117db924cf7c	2889
wolfSSL	15:117db924cf7c	2890	return 0;
wolfSSL	15:117db924cf7c	2891	}
wolfSSL	15:117db924cf7c	2892	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	2893
wolfSSL	15:117db924cf7c	2894	#elif defined(WOLFSSL_PIC32MZ_CRYPT)
wolfSSL	15:117db924cf7c	2895
wolfSSL	15:117db924cf7c	2896	int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2897	{
wolfSSL	15:117db924cf7c	2898	int ret;
wolfSSL	15:117db924cf7c	2899
wolfSSL	15:117db924cf7c	2900	/* hardware fails on input that is not a multiple of AES block size */
wolfSSL	15:117db924cf7c	2901	if (sz % AES_BLOCK_SIZE != 0) {
wolfSSL	15:117db924cf7c	2902	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2903	}
wolfSSL	15:117db924cf7c	2904
wolfSSL	15:117db924cf7c	2905	ret = wc_Pic32AesCrypt(
wolfSSL	15:117db924cf7c	2906	aes->key, aes->keylen, aes->reg, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	2907	out, in, sz, PIC32_ENCRYPTION,
wolfSSL	15:117db924cf7c	2908	PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCBC);
wolfSSL	15:117db924cf7c	2909
wolfSSL	15:117db924cf7c	2910	/* store iv for next call */
wolfSSL	15:117db924cf7c	2911	if (ret == 0) {
wolfSSL	15:117db924cf7c	2912	XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2913	}
wolfSSL	15:117db924cf7c	2914
wolfSSL	15:117db924cf7c	2915	return ret;
wolfSSL	15:117db924cf7c	2916	}
wolfSSL	15:117db924cf7c	2917	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	2918	int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2919	{
wolfSSL	15:117db924cf7c	2920	int ret;
wolfSSL	15:117db924cf7c	2921	byte scratch[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	2922
wolfSSL	15:117db924cf7c	2923	/* hardware fails on input that is not a multiple of AES block size */
wolfSSL	15:117db924cf7c	2924	if (sz % AES_BLOCK_SIZE != 0) {
wolfSSL	15:117db924cf7c	2925	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2926	}
wolfSSL	15:117db924cf7c	2927	XMEMCPY(scratch, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2928
wolfSSL	15:117db924cf7c	2929	ret = wc_Pic32AesCrypt(
wolfSSL	15:117db924cf7c	2930	aes->key, aes->keylen, aes->reg, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	2931	out, in, sz, PIC32_DECRYPTION,
wolfSSL	15:117db924cf7c	2932	PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCBC);
wolfSSL	15:117db924cf7c	2933
wolfSSL	15:117db924cf7c	2934	/* store iv for next call */
wolfSSL	15:117db924cf7c	2935	if (ret == 0) {
wolfSSL	15:117db924cf7c	2936	XMEMCPY((byte*)aes->reg, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2937	}
wolfSSL	15:117db924cf7c	2938
wolfSSL	15:117db924cf7c	2939	return ret;
wolfSSL	15:117db924cf7c	2940	}
wolfSSL	15:117db924cf7c	2941	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	2942
wolfSSL	15:117db924cf7c	2943	#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES)
wolfSSL	15:117db924cf7c	2944	/* implemented in wolfcrypt/src/port/caam/caam_aes.c */
wolfSSL	15:117db924cf7c	2945
wolfSSL	15:117db924cf7c	2946	#else
wolfSSL	15:117db924cf7c	2947
wolfSSL	15:117db924cf7c	2948	int wc_AesCbcEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	2949	{
wolfSSL	15:117db924cf7c	2950	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2951
wolfSSL	15:117db924cf7c	2952	if (aes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	2953	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	2954	}
wolfSSL	15:117db924cf7c	2955
wolfSSL	15:117db924cf7c	2956	#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
wolfSSL	15:117db924cf7c	2957	/* if async and byte count above threshold */
wolfSSL	15:117db924cf7c	2958	if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
wolfSSL	15:117db924cf7c	2959	sz >= WC_ASYNC_THRESH_AES_CBC) {
wolfSSL	15:117db924cf7c	2960	#if defined(HAVE_CAVIUM)
wolfSSL	15:117db924cf7c	2961	return NitroxAesCbcEncrypt(aes, out, in, sz);
wolfSSL	15:117db924cf7c	2962	#elif defined(HAVE_INTEL_QA)
wolfSSL	15:117db924cf7c	2963	return IntelQaSymAesCbcEncrypt(&aes->asyncDev, out, in, sz,
wolfSSL	15:117db924cf7c	2964	(const byte*)aes->asyncKey, aes->keylen,
wolfSSL	15:117db924cf7c	2965	(const byte*)aes->asyncIv, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	2966	#else /* WOLFSSL_ASYNC_CRYPT_TEST */
wolfSSL	15:117db924cf7c	2967	if (wc_AsyncTestInit(&aes->asyncDev, ASYNC_TEST_AES_CBC_ENCRYPT)) {
wolfSSL	15:117db924cf7c	2968	WC_ASYNC_TEST* testDev = &aes->asyncDev.test;
wolfSSL	15:117db924cf7c	2969	testDev->aes.aes = aes;
wolfSSL	15:117db924cf7c	2970	testDev->aes.out = out;
wolfSSL	15:117db924cf7c	2971	testDev->aes.in = in;
wolfSSL	15:117db924cf7c	2972	testDev->aes.sz = sz;
wolfSSL	15:117db924cf7c	2973	return WC_PENDING_E;
wolfSSL	15:117db924cf7c	2974	}
wolfSSL	15:117db924cf7c	2975	#endif
wolfSSL	15:117db924cf7c	2976	}
wolfSSL	15:117db924cf7c	2977	#endif /* WOLFSSL_ASYNC_CRYPT */
wolfSSL	15:117db924cf7c	2978
wolfSSL	15:117db924cf7c	2979	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	2980	if (haveAESNI) {
wolfSSL	15:117db924cf7c	2981	#ifdef DEBUG_AESNI
wolfSSL	15:117db924cf7c	2982	printf("about to aes cbc encrypt\n");
wolfSSL	15:117db924cf7c	2983	printf("in = %p\n", in);
wolfSSL	15:117db924cf7c	2984	printf("out = %p\n", out);
wolfSSL	15:117db924cf7c	2985	printf("aes->key = %p\n", aes->key);
wolfSSL	15:117db924cf7c	2986	printf("aes->reg = %p\n", aes->reg);
wolfSSL	15:117db924cf7c	2987	printf("aes->rounds = %d\n", aes->rounds);
wolfSSL	15:117db924cf7c	2988	printf("sz = %d\n", sz);
wolfSSL	15:117db924cf7c	2989	#endif
wolfSSL	15:117db924cf7c	2990
wolfSSL	15:117db924cf7c	2991	/* check alignment, decrypt doesn't need alignment */
wolfSSL	15:117db924cf7c	2992	if ((wolfssl_word)in % AESNI_ALIGN) {
wolfSSL	15:117db924cf7c	2993	#ifndef NO_WOLFSSL_ALLOC_ALIGN
wolfSSL	15:117db924cf7c	2994	byte* tmp = (byte*)XMALLOC(sz + AES_BLOCK_SIZE + AESNI_ALIGN,
wolfSSL	15:117db924cf7c	2995	aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	2996	byte* tmp_align;
wolfSSL	15:117db924cf7c	2997	if (tmp == NULL) return MEMORY_E;
wolfSSL	15:117db924cf7c	2998
wolfSSL	15:117db924cf7c	2999	tmp_align = tmp + (AESNI_ALIGN - ((size_t)tmp % AESNI_ALIGN));
wolfSSL	15:117db924cf7c	3000	XMEMCPY(tmp_align, in, sz);
wolfSSL	15:117db924cf7c	3001	AES_CBC_encrypt(tmp_align, tmp_align, (byte*)aes->reg, sz,
wolfSSL	15:117db924cf7c	3002	(byte*)aes->key, aes->rounds);
wolfSSL	15:117db924cf7c	3003	/* store iv for next call */
wolfSSL	15:117db924cf7c	3004	XMEMCPY(aes->reg, tmp_align + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3005
wolfSSL	15:117db924cf7c	3006	XMEMCPY(out, tmp_align, sz);
wolfSSL	15:117db924cf7c	3007	XFREE(tmp, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	3008	return 0;
wolfSSL	15:117db924cf7c	3009	#else
wolfSSL	15:117db924cf7c	3010	WOLFSSL_MSG("AES-CBC encrypt with bad alignment");
wolfSSL	15:117db924cf7c	3011	return BAD_ALIGN_E;
wolfSSL	15:117db924cf7c	3012	#endif
wolfSSL	15:117db924cf7c	3013	}
wolfSSL	15:117db924cf7c	3014
wolfSSL	15:117db924cf7c	3015	AES_CBC_encrypt(in, out, (byte)aes->reg, sz, (byte)aes->key,
wolfSSL	15:117db924cf7c	3016	aes->rounds);
wolfSSL	15:117db924cf7c	3017	/* store iv for next call */
wolfSSL	15:117db924cf7c	3018	XMEMCPY(aes->reg, out + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3019
wolfSSL	15:117db924cf7c	3020	return 0;
wolfSSL	15:117db924cf7c	3021	}
wolfSSL	15:117db924cf7c	3022	#endif
wolfSSL	15:117db924cf7c	3023
wolfSSL	15:117db924cf7c	3024	while (blocks--) {
wolfSSL	15:117db924cf7c	3025	xorbuf((byte*)aes->reg, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3026	wc_AesEncrypt(aes, (byte)aes->reg, (byte)aes->reg);
wolfSSL	15:117db924cf7c	3027	XMEMCPY(out, aes->reg, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3028
wolfSSL	15:117db924cf7c	3029	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3030	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3031	}
wolfSSL	15:117db924cf7c	3032
wolfSSL	15:117db924cf7c	3033	return 0;
wolfSSL	15:117db924cf7c	3034	}
wolfSSL	15:117db924cf7c	3035
wolfSSL	15:117db924cf7c	3036	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	3037	int wc_AesCbcDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	3038	{
wolfSSL	15:117db924cf7c	3039	word32 blocks;
wolfSSL	15:117db924cf7c	3040
wolfSSL	15:117db924cf7c	3041	if (aes == NULL \|\| out == NULL \|\| in == NULL
wolfSSL	15:117db924cf7c	3042	\|\| sz % AES_BLOCK_SIZE != 0) {
wolfSSL	15:117db924cf7c	3043	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	3044	}
wolfSSL	15:117db924cf7c	3045
wolfSSL	15:117db924cf7c	3046	#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
wolfSSL	15:117db924cf7c	3047	/* if async and byte count above threshold */
wolfSSL	15:117db924cf7c	3048	if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
wolfSSL	15:117db924cf7c	3049	sz >= WC_ASYNC_THRESH_AES_CBC) {
wolfSSL	15:117db924cf7c	3050	#if defined(HAVE_CAVIUM)
wolfSSL	15:117db924cf7c	3051	return NitroxAesCbcDecrypt(aes, out, in, sz);
wolfSSL	15:117db924cf7c	3052	#elif defined(HAVE_INTEL_QA)
wolfSSL	15:117db924cf7c	3053	return IntelQaSymAesCbcDecrypt(&aes->asyncDev, out, in, sz,
wolfSSL	15:117db924cf7c	3054	(const byte*)aes->asyncKey, aes->keylen,
wolfSSL	15:117db924cf7c	3055	(const byte*)aes->asyncIv, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3056	#else /* WOLFSSL_ASYNC_CRYPT_TEST */
wolfSSL	15:117db924cf7c	3057	if (wc_AsyncTestInit(&aes->asyncDev, ASYNC_TEST_AES_CBC_DECRYPT)) {
wolfSSL	15:117db924cf7c	3058	WC_ASYNC_TEST* testDev = &aes->asyncDev.test;
wolfSSL	15:117db924cf7c	3059	testDev->aes.aes = aes;
wolfSSL	15:117db924cf7c	3060	testDev->aes.out = out;
wolfSSL	15:117db924cf7c	3061	testDev->aes.in = in;
wolfSSL	15:117db924cf7c	3062	testDev->aes.sz = sz;
wolfSSL	15:117db924cf7c	3063	return WC_PENDING_E;
wolfSSL	15:117db924cf7c	3064	}
wolfSSL	15:117db924cf7c	3065	#endif
wolfSSL	15:117db924cf7c	3066	}
wolfSSL	15:117db924cf7c	3067	#endif
wolfSSL	15:117db924cf7c	3068
wolfSSL	15:117db924cf7c	3069	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	3070	if (haveAESNI) {
wolfSSL	15:117db924cf7c	3071	#ifdef DEBUG_AESNI
wolfSSL	15:117db924cf7c	3072	printf("about to aes cbc decrypt\n");
wolfSSL	15:117db924cf7c	3073	printf("in = %p\n", in);
wolfSSL	15:117db924cf7c	3074	printf("out = %p\n", out);
wolfSSL	15:117db924cf7c	3075	printf("aes->key = %p\n", aes->key);
wolfSSL	15:117db924cf7c	3076	printf("aes->reg = %p\n", aes->reg);
wolfSSL	15:117db924cf7c	3077	printf("aes->rounds = %d\n", aes->rounds);
wolfSSL	15:117db924cf7c	3078	printf("sz = %d\n", sz);
wolfSSL	15:117db924cf7c	3079	#endif
wolfSSL	15:117db924cf7c	3080
wolfSSL	15:117db924cf7c	3081	/* if input and output same will overwrite input iv */
wolfSSL	15:117db924cf7c	3082	XMEMCPY(aes->tmp, in + sz - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3083	#if defined(WOLFSSL_AESNI_BY4)
wolfSSL	15:117db924cf7c	3084	AES_CBC_decrypt_by4(in, out, (byte)aes->reg, sz, (byte)aes->key,
wolfSSL	15:117db924cf7c	3085	aes->rounds);
wolfSSL	15:117db924cf7c	3086	#elif defined(WOLFSSL_AESNI_BY6)
wolfSSL	15:117db924cf7c	3087	AES_CBC_decrypt_by6(in, out, (byte)aes->reg, sz, (byte)aes->key,
wolfSSL	15:117db924cf7c	3088	aes->rounds);
wolfSSL	15:117db924cf7c	3089	#else /* WOLFSSL_AESNI_BYx */
wolfSSL	15:117db924cf7c	3090	AES_CBC_decrypt_by8(in, out, (byte)aes->reg, sz, (byte)aes->key,
wolfSSL	15:117db924cf7c	3091	aes->rounds);
wolfSSL	15:117db924cf7c	3092	#endif /* WOLFSSL_AESNI_BYx */
wolfSSL	15:117db924cf7c	3093	/* store iv for next call */
wolfSSL	15:117db924cf7c	3094	XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3095	return 0;
wolfSSL	15:117db924cf7c	3096	}
wolfSSL	15:117db924cf7c	3097	#endif
wolfSSL	15:117db924cf7c	3098
wolfSSL	15:117db924cf7c	3099	blocks = sz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3100	while (blocks--) {
wolfSSL	15:117db924cf7c	3101	XMEMCPY(aes->tmp, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3102	wc_AesDecrypt(aes, (byte*)aes->tmp, out);
wolfSSL	15:117db924cf7c	3103	xorbuf(out, (byte*)aes->reg, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3104	XMEMCPY(aes->reg, aes->tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3105
wolfSSL	15:117db924cf7c	3106	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3107	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3108	}
wolfSSL	15:117db924cf7c	3109
wolfSSL	15:117db924cf7c	3110	return 0;
wolfSSL	15:117db924cf7c	3111	}
wolfSSL	15:117db924cf7c	3112	#endif
wolfSSL	15:117db924cf7c	3113
wolfSSL	15:117db924cf7c	3114	#endif /* AES-CBC block */
wolfSSL	15:117db924cf7c	3115	#endif /* HAVE_AES_CBC */
wolfSSL	15:117db924cf7c	3116
wolfSSL	15:117db924cf7c	3117	/* AES-CTR */
wolfSSL	15:117db924cf7c	3118	#if defined(WOLFSSL_AES_COUNTER)
wolfSSL	15:117db924cf7c	3119
wolfSSL	15:117db924cf7c	3120	#ifdef STM32_CRYPTO
wolfSSL	15:117db924cf7c	3121	#define NEED_AES_CTR_SOFT
wolfSSL	15:117db924cf7c	3122	#define XTRANSFORM_AESCTRBLOCK wc_AesCtrEncryptBlock
wolfSSL	15:117db924cf7c	3123
wolfSSL	15:117db924cf7c	3124	int wc_AesCtrEncryptBlock(Aes* aes, byte* out, const byte* in)
wolfSSL	15:117db924cf7c	3125	{
wolfSSL	15:117db924cf7c	3126	int ret = 0;
wolfSSL	15:117db924cf7c	3127	#ifdef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	3128	CRYP_HandleTypeDef hcryp;
wolfSSL	15:117db924cf7c	3129
wolfSSL	15:117db924cf7c	3130	XMEMSET(&hcryp, 0, sizeof(CRYP_HandleTypeDef));
wolfSSL	15:117db924cf7c	3131	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	3132	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	3133	hcryp.Init.KeySize = CRYP_KEYSIZE_128B;
wolfSSL	15:117db924cf7c	3134	break;
wolfSSL	15:117db924cf7c	3135	#ifdef CRYP_KEYSIZE_192B
wolfSSL	15:117db924cf7c	3136	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	3137	hcryp.Init.KeySize = CRYP_KEYSIZE_192B;
wolfSSL	15:117db924cf7c	3138	break;
wolfSSL	15:117db924cf7c	3139	#endif
wolfSSL	15:117db924cf7c	3140	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	3141	hcryp.Init.KeySize = CRYP_KEYSIZE_256B;
wolfSSL	15:117db924cf7c	3142	break;
wolfSSL	15:117db924cf7c	3143	default:
wolfSSL	15:117db924cf7c	3144	break;
wolfSSL	15:117db924cf7c	3145	}
wolfSSL	15:117db924cf7c	3146	hcryp.Instance = CRYP;
wolfSSL	15:117db924cf7c	3147	hcryp.Init.DataType = CRYP_DATATYPE_8B;
wolfSSL	15:117db924cf7c	3148	hcryp.Init.pKey = (byte*)aes->key;
wolfSSL	15:117db924cf7c	3149	hcryp.Init.pInitVect = (byte*)aes->reg;
wolfSSL	15:117db924cf7c	3150
wolfSSL	15:117db924cf7c	3151	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	3152
wolfSSL	15:117db924cf7c	3153	if (HAL_CRYP_AESCTR_Encrypt(&hcryp, (byte*)in, AES_BLOCK_SIZE, out,
wolfSSL	15:117db924cf7c	3154	STM32_HAL_TIMEOUT) != HAL_OK) {
wolfSSL	15:117db924cf7c	3155	/* failed */
wolfSSL	15:117db924cf7c	3156	ret = WC_TIMEOUT_E;
wolfSSL	15:117db924cf7c	3157	}
wolfSSL	15:117db924cf7c	3158
wolfSSL	15:117db924cf7c	3159	HAL_CRYP_DeInit(&hcryp);
wolfSSL	15:117db924cf7c	3160
wolfSSL	15:117db924cf7c	3161	#else /* STD_PERI_LIB */
wolfSSL	15:117db924cf7c	3162	word32 enc_key, iv;
wolfSSL	15:117db924cf7c	3163	CRYP_InitTypeDef AES_CRYP_InitStructure;
wolfSSL	15:117db924cf7c	3164	CRYP_KeyInitTypeDef AES_CRYP_KeyInitStructure;
wolfSSL	15:117db924cf7c	3165	CRYP_IVInitTypeDef AES_CRYP_IVInitStructure;
wolfSSL	15:117db924cf7c	3166
wolfSSL	15:117db924cf7c	3167	enc_key = aes->key;
wolfSSL	15:117db924cf7c	3168	iv = aes->reg;
wolfSSL	15:117db924cf7c	3169
wolfSSL	15:117db924cf7c	3170	/* crypto structure initialization */
wolfSSL	15:117db924cf7c	3171	CRYP_KeyStructInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	3172	CRYP_StructInit(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	3173	CRYP_IVStructInit(&AES_CRYP_IVInitStructure);
wolfSSL	15:117db924cf7c	3174
wolfSSL	15:117db924cf7c	3175	/* reset registers to their default values */
wolfSSL	15:117db924cf7c	3176	CRYP_DeInit();
wolfSSL	15:117db924cf7c	3177
wolfSSL	15:117db924cf7c	3178	/* load key into correct registers */
wolfSSL	15:117db924cf7c	3179	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	3180	case 10: /* 128-bit key */
wolfSSL	15:117db924cf7c	3181	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_128b;
wolfSSL	15:117db924cf7c	3182	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[0];
wolfSSL	15:117db924cf7c	3183	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[1];
wolfSSL	15:117db924cf7c	3184	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[2];
wolfSSL	15:117db924cf7c	3185	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[3];
wolfSSL	15:117db924cf7c	3186	break;
wolfSSL	15:117db924cf7c	3187	case 12: /* 192-bit key */
wolfSSL	15:117db924cf7c	3188	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_192b;
wolfSSL	15:117db924cf7c	3189	AES_CRYP_KeyInitStructure.CRYP_Key1Left = enc_key[0];
wolfSSL	15:117db924cf7c	3190	AES_CRYP_KeyInitStructure.CRYP_Key1Right = enc_key[1];
wolfSSL	15:117db924cf7c	3191	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[2];
wolfSSL	15:117db924cf7c	3192	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[3];
wolfSSL	15:117db924cf7c	3193	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[4];
wolfSSL	15:117db924cf7c	3194	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[5];
wolfSSL	15:117db924cf7c	3195	break;
wolfSSL	15:117db924cf7c	3196	case 14: /* 256-bit key */
wolfSSL	15:117db924cf7c	3197	AES_CRYP_InitStructure.CRYP_KeySize = CRYP_KeySize_256b;
wolfSSL	15:117db924cf7c	3198	AES_CRYP_KeyInitStructure.CRYP_Key0Left = enc_key[0];
wolfSSL	15:117db924cf7c	3199	AES_CRYP_KeyInitStructure.CRYP_Key0Right = enc_key[1];
wolfSSL	15:117db924cf7c	3200	AES_CRYP_KeyInitStructure.CRYP_Key1Left = enc_key[2];
wolfSSL	15:117db924cf7c	3201	AES_CRYP_KeyInitStructure.CRYP_Key1Right = enc_key[3];
wolfSSL	15:117db924cf7c	3202	AES_CRYP_KeyInitStructure.CRYP_Key2Left = enc_key[4];
wolfSSL	15:117db924cf7c	3203	AES_CRYP_KeyInitStructure.CRYP_Key2Right = enc_key[5];
wolfSSL	15:117db924cf7c	3204	AES_CRYP_KeyInitStructure.CRYP_Key3Left = enc_key[6];
wolfSSL	15:117db924cf7c	3205	AES_CRYP_KeyInitStructure.CRYP_Key3Right = enc_key[7];
wolfSSL	15:117db924cf7c	3206	break;
wolfSSL	15:117db924cf7c	3207	default:
wolfSSL	15:117db924cf7c	3208	break;
wolfSSL	15:117db924cf7c	3209	}
wolfSSL	15:117db924cf7c	3210	CRYP_KeyInit(&AES_CRYP_KeyInitStructure);
wolfSSL	15:117db924cf7c	3211
wolfSSL	15:117db924cf7c	3212	/* set iv */
wolfSSL	15:117db924cf7c	3213	AES_CRYP_IVInitStructure.CRYP_IV0Left = ByteReverseWord32(iv[0]);
wolfSSL	15:117db924cf7c	3214	AES_CRYP_IVInitStructure.CRYP_IV0Right = ByteReverseWord32(iv[1]);
wolfSSL	15:117db924cf7c	3215	AES_CRYP_IVInitStructure.CRYP_IV1Left = ByteReverseWord32(iv[2]);
wolfSSL	15:117db924cf7c	3216	AES_CRYP_IVInitStructure.CRYP_IV1Right = ByteReverseWord32(iv[3]);
wolfSSL	15:117db924cf7c	3217	CRYP_IVInit(&AES_CRYP_IVInitStructure);
wolfSSL	15:117db924cf7c	3218
wolfSSL	15:117db924cf7c	3219	/* set direction, mode, and datatype */
wolfSSL	15:117db924cf7c	3220	AES_CRYP_InitStructure.CRYP_AlgoDir = CRYP_AlgoDir_Encrypt;
wolfSSL	15:117db924cf7c	3221	AES_CRYP_InitStructure.CRYP_AlgoMode = CRYP_AlgoMode_AES_CTR;
wolfSSL	15:117db924cf7c	3222	AES_CRYP_InitStructure.CRYP_DataType = CRYP_DataType_8b;
wolfSSL	15:117db924cf7c	3223	CRYP_Init(&AES_CRYP_InitStructure);
wolfSSL	15:117db924cf7c	3224
wolfSSL	15:117db924cf7c	3225	/* enable crypto processor */
wolfSSL	15:117db924cf7c	3226	CRYP_Cmd(ENABLE);
wolfSSL	15:117db924cf7c	3227
wolfSSL	15:117db924cf7c	3228	/* flush IN/OUT FIFOs */
wolfSSL	15:117db924cf7c	3229	CRYP_FIFOFlush();
wolfSSL	15:117db924cf7c	3230
wolfSSL	15:117db924cf7c	3231	CRYP_DataIn((uint32_t)&in[0]);
wolfSSL	15:117db924cf7c	3232	CRYP_DataIn((uint32_t)&in[4]);
wolfSSL	15:117db924cf7c	3233	CRYP_DataIn((uint32_t)&in[8]);
wolfSSL	15:117db924cf7c	3234	CRYP_DataIn((uint32_t)&in[12]);
wolfSSL	15:117db924cf7c	3235
wolfSSL	15:117db924cf7c	3236	/* wait until the complete message has been processed */
wolfSSL	15:117db924cf7c	3237	while (CRYP_GetFlagStatus(CRYP_FLAG_BUSY) != RESET) {}
wolfSSL	15:117db924cf7c	3238
wolfSSL	15:117db924cf7c	3239	(uint32_t)&out[0] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	3240	(uint32_t)&out[4] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	3241	(uint32_t)&out[8] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	3242	(uint32_t)&out[12] = CRYP_DataOut();
wolfSSL	15:117db924cf7c	3243
wolfSSL	15:117db924cf7c	3244	/* disable crypto processor */
wolfSSL	15:117db924cf7c	3245	CRYP_Cmd(DISABLE);
wolfSSL	15:117db924cf7c	3246
wolfSSL	15:117db924cf7c	3247	#endif /* WOLFSSL_STM32_CUBEMX */
wolfSSL	15:117db924cf7c	3248	return ret;
wolfSSL	15:117db924cf7c	3249	}
wolfSSL	15:117db924cf7c	3250
wolfSSL	15:117db924cf7c	3251
wolfSSL	15:117db924cf7c	3252	#elif defined(WOLFSSL_PIC32MZ_CRYPT)
wolfSSL	15:117db924cf7c	3253
wolfSSL	15:117db924cf7c	3254	#define NEED_AES_CTR_SOFT
wolfSSL	15:117db924cf7c	3255	#define XTRANSFORM_AESCTRBLOCK wc_AesCtrEncryptBlock
wolfSSL	15:117db924cf7c	3256
wolfSSL	15:117db924cf7c	3257	int wc_AesCtrEncryptBlock(Aes* aes, byte* out, const byte* in)
wolfSSL	15:117db924cf7c	3258	{
wolfSSL	15:117db924cf7c	3259	word32 tmpIv[AES_BLOCK_SIZE / sizeof(word32)];
wolfSSL	15:117db924cf7c	3260	XMEMCPY(tmpIv, aes->reg, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3261	return wc_Pic32AesCrypt(
wolfSSL	15:117db924cf7c	3262	aes->key, aes->keylen, tmpIv, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	3263	out, in, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	3264	PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_RCTR);
wolfSSL	15:117db924cf7c	3265	}
wolfSSL	15:117db924cf7c	3266
wolfSSL	15:117db924cf7c	3267	#elif defined(HAVE_COLDFIRE_SEC)
wolfSSL	15:117db924cf7c	3268	#error "Coldfire SEC doesn't currently support AES-CTR mode"
wolfSSL	15:117db924cf7c	3269
wolfSSL	15:117db924cf7c	3270	#elif defined(FREESCALE_LTC)
wolfSSL	15:117db924cf7c	3271	int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	3272	{
wolfSSL	15:117db924cf7c	3273	uint32_t keySize;
wolfSSL	15:117db924cf7c	3274	byte iv, enc_key;
wolfSSL	15:117db924cf7c	3275	byte* tmp;
wolfSSL	15:117db924cf7c	3276
wolfSSL	15:117db924cf7c	3277	if (aes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	3278	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	3279	}
wolfSSL	15:117db924cf7c	3280
wolfSSL	15:117db924cf7c	3281	/* consume any unused bytes left in aes->tmp */
wolfSSL	15:117db924cf7c	3282	tmp = (byte*)aes->tmp + AES_BLOCK_SIZE - aes->left;
wolfSSL	15:117db924cf7c	3283	while (aes->left && sz) {
wolfSSL	15:117db924cf7c	3284	(out++) = (in++) ^ *(tmp++);
wolfSSL	15:117db924cf7c	3285	aes->left--;
wolfSSL	15:117db924cf7c	3286	sz--;
wolfSSL	15:117db924cf7c	3287	}
wolfSSL	15:117db924cf7c	3288
wolfSSL	15:117db924cf7c	3289	if (sz) {
wolfSSL	15:117db924cf7c	3290	iv = (byte*)aes->reg;
wolfSSL	15:117db924cf7c	3291	enc_key = (byte*)aes->key;
wolfSSL	15:117db924cf7c	3292
wolfSSL	15:117db924cf7c	3293	wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	3294
wolfSSL	15:117db924cf7c	3295	LTC_AES_CryptCtr(LTC_BASE, in, out, sz,
wolfSSL	15:117db924cf7c	3296	iv, enc_key, keySize, (byte*)aes->tmp,
wolfSSL	15:117db924cf7c	3297	(uint32_t*)&aes->left);
wolfSSL	15:117db924cf7c	3298	}
wolfSSL	15:117db924cf7c	3299
wolfSSL	15:117db924cf7c	3300	return 0;
wolfSSL	15:117db924cf7c	3301	}
wolfSSL	15:117db924cf7c	3302
wolfSSL	15:117db924cf7c	3303	#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES)
wolfSSL	15:117db924cf7c	3304	/* implemented in wolfcrypt/src/port/caam/caam_aes.c */
wolfSSL	15:117db924cf7c	3305
wolfSSL	15:117db924cf7c	3306	#else
wolfSSL	15:117db924cf7c	3307
wolfSSL	15:117db924cf7c	3308	/* Use software based AES counter */
wolfSSL	15:117db924cf7c	3309	#define NEED_AES_CTR_SOFT
wolfSSL	15:117db924cf7c	3310	#endif
wolfSSL	15:117db924cf7c	3311
wolfSSL	15:117db924cf7c	3312	#ifdef NEED_AES_CTR_SOFT
wolfSSL	15:117db924cf7c	3313	/* Increment AES counter */
wolfSSL	15:117db924cf7c	3314	static WC_INLINE void IncrementAesCounter(byte* inOutCtr)
wolfSSL	15:117db924cf7c	3315	{
wolfSSL	15:117db924cf7c	3316	/* in network byte order so start at end and work back */
wolfSSL	15:117db924cf7c	3317	int i;
wolfSSL	15:117db924cf7c	3318	for (i = AES_BLOCK_SIZE - 1; i >= 0; i--) {
wolfSSL	15:117db924cf7c	3319	if (++inOutCtr[i]) /* we're done unless we overflow */
wolfSSL	15:117db924cf7c	3320	return;
wolfSSL	15:117db924cf7c	3321	}
wolfSSL	15:117db924cf7c	3322	}
wolfSSL	15:117db924cf7c	3323
wolfSSL	15:117db924cf7c	3324	int wc_AesCtrEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	3325	{
wolfSSL	15:117db924cf7c	3326	byte* tmp;
wolfSSL	15:117db924cf7c	3327
wolfSSL	15:117db924cf7c	3328	if (aes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	3329	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	3330	}
wolfSSL	15:117db924cf7c	3331
wolfSSL	15:117db924cf7c	3332	/* consume any unused bytes left in aes->tmp */
wolfSSL	15:117db924cf7c	3333	tmp = (byte*)aes->tmp + AES_BLOCK_SIZE - aes->left;
wolfSSL	15:117db924cf7c	3334	while (aes->left && sz) {
wolfSSL	15:117db924cf7c	3335	(out++) = (in++) ^ *(tmp++);
wolfSSL	15:117db924cf7c	3336	aes->left--;
wolfSSL	15:117db924cf7c	3337	sz--;
wolfSSL	15:117db924cf7c	3338	}
wolfSSL	15:117db924cf7c	3339
wolfSSL	15:117db924cf7c	3340	/* do as many block size ops as possible */
wolfSSL	15:117db924cf7c	3341	while (sz >= AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	3342	#ifdef XTRANSFORM_AESCTRBLOCK
wolfSSL	15:117db924cf7c	3343	XTRANSFORM_AESCTRBLOCK(aes, out, in);
wolfSSL	15:117db924cf7c	3344	#else
wolfSSL	15:117db924cf7c	3345	wc_AesEncrypt(aes, (byte*)aes->reg, out);
wolfSSL	15:117db924cf7c	3346	xorbuf(out, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3347	#endif
wolfSSL	15:117db924cf7c	3348	IncrementAesCounter((byte*)aes->reg);
wolfSSL	15:117db924cf7c	3349
wolfSSL	15:117db924cf7c	3350	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3351	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3352	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3353	aes->left = 0;
wolfSSL	15:117db924cf7c	3354	}
wolfSSL	15:117db924cf7c	3355
wolfSSL	15:117db924cf7c	3356	/* handle non block size remaining and store unused byte count in left */
wolfSSL	15:117db924cf7c	3357	if (sz) {
wolfSSL	15:117db924cf7c	3358	wc_AesEncrypt(aes, (byte)aes->reg, (byte)aes->tmp);
wolfSSL	15:117db924cf7c	3359	IncrementAesCounter((byte*)aes->reg);
wolfSSL	15:117db924cf7c	3360
wolfSSL	15:117db924cf7c	3361	aes->left = AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	3362	tmp = (byte*)aes->tmp;
wolfSSL	15:117db924cf7c	3363
wolfSSL	15:117db924cf7c	3364	while (sz--) {
wolfSSL	15:117db924cf7c	3365	(out++) = (in++) ^ *(tmp++);
wolfSSL	15:117db924cf7c	3366	aes->left--;
wolfSSL	15:117db924cf7c	3367	}
wolfSSL	15:117db924cf7c	3368	}
wolfSSL	15:117db924cf7c	3369
wolfSSL	15:117db924cf7c	3370	return 0;
wolfSSL	15:117db924cf7c	3371	}
wolfSSL	15:117db924cf7c	3372
wolfSSL	15:117db924cf7c	3373	#endif /* NEED_AES_CTR_SOFT */
wolfSSL	15:117db924cf7c	3374
wolfSSL	15:117db924cf7c	3375	#endif /* WOLFSSL_AES_COUNTER */
wolfSSL	15:117db924cf7c	3376	#endif /* !WOLFSSL_ARMASM */
wolfSSL	15:117db924cf7c	3377
wolfSSL	15:117db924cf7c	3378
wolfSSL	15:117db924cf7c	3379	/*
wolfSSL	15:117db924cf7c	3380	* The IV for AES GCM and CCM, stored in struct Aes's member reg, is comprised
wolfSSL	15:117db924cf7c	3381	* of two parts in order:
wolfSSL	15:117db924cf7c	3382	* 1. The fixed field which may be 0 or 4 bytes long. In TLS, this is set
wolfSSL	15:117db924cf7c	3383	* to the implicit IV.
wolfSSL	15:117db924cf7c	3384	* 2. The explicit IV is generated by wolfCrypt. It needs to be managed
wolfSSL	15:117db924cf7c	3385	* by wolfCrypt to ensure the IV is unique for each call to encrypt.
wolfSSL	15:117db924cf7c	3386	* The IV may be a 96-bit random value, or the 32-bit fixed value and a
wolfSSL	15:117db924cf7c	3387	* 64-bit set of 0 or random data. The final 32-bits of reg is used as a
wolfSSL	15:117db924cf7c	3388	* block counter during the encryption.
wolfSSL	15:117db924cf7c	3389	*/
wolfSSL	15:117db924cf7c	3390
wolfSSL	15:117db924cf7c	3391	#if (defined(HAVE_AESGCM) && !defined(WC_NO_RNG)) \|\| defined(HAVE_AESCCM)
wolfSSL	15:117db924cf7c	3392	static WC_INLINE void IncCtr(byte* ctr, word32 ctrSz)
wolfSSL	15:117db924cf7c	3393	{
wolfSSL	15:117db924cf7c	3394	int i;
wolfSSL	15:117db924cf7c	3395	for (i = ctrSz-1; i >= 0; i--) {
wolfSSL	15:117db924cf7c	3396	if (++ctr[i])
wolfSSL	15:117db924cf7c	3397	break;
wolfSSL	15:117db924cf7c	3398	}
wolfSSL	15:117db924cf7c	3399	}
wolfSSL	15:117db924cf7c	3400	#endif /* HAVE_AESGCM \|\| HAVE_AESCCM */
wolfSSL	15:117db924cf7c	3401
wolfSSL	15:117db924cf7c	3402
wolfSSL	15:117db924cf7c	3403	#ifdef HAVE_AESGCM
wolfSSL	15:117db924cf7c	3404
wolfSSL	15:117db924cf7c	3405	#if defined(HAVE_COLDFIRE_SEC)
wolfSSL	15:117db924cf7c	3406	#error "Coldfire SEC doesn't currently support AES-GCM mode"
wolfSSL	15:117db924cf7c	3407
wolfSSL	15:117db924cf7c	3408	#elif defined(WOLFSSL_NRF51_AES)
wolfSSL	15:117db924cf7c	3409	#error "nRF51 doesn't currently support AES-GCM mode"
wolfSSL	15:117db924cf7c	3410
wolfSSL	15:117db924cf7c	3411	#endif
wolfSSL	15:117db924cf7c	3412
wolfSSL	15:117db924cf7c	3413	#ifdef WOLFSSL_ARMASM
wolfSSL	15:117db924cf7c	3414	/* implementation is located in wolfcrypt/src/port/arm/armv8-aes.c */
wolfSSL	15:117db924cf7c	3415	#else /* software + AESNI implementation */
wolfSSL	15:117db924cf7c	3416
wolfSSL	15:117db924cf7c	3417	#if !defined(FREESCALE_LTC_AES_GCM)
wolfSSL	15:117db924cf7c	3418	static WC_INLINE void IncrementGcmCounter(byte* inOutCtr)
wolfSSL	15:117db924cf7c	3419	{
wolfSSL	15:117db924cf7c	3420	int i;
wolfSSL	15:117db924cf7c	3421
wolfSSL	15:117db924cf7c	3422	/* in network byte order so start at end and work back */
wolfSSL	15:117db924cf7c	3423	for (i = AES_BLOCK_SIZE - 1; i >= AES_BLOCK_SIZE - CTR_SZ; i--) {
wolfSSL	15:117db924cf7c	3424	if (++inOutCtr[i]) /* we're done unless we overflow */
wolfSSL	15:117db924cf7c	3425	return;
wolfSSL	15:117db924cf7c	3426	}
wolfSSL	15:117db924cf7c	3427	}
wolfSSL	15:117db924cf7c	3428	#endif /* !FREESCALE_LTC_AES_GCM */
wolfSSL	15:117db924cf7c	3429
wolfSSL	15:117db924cf7c	3430	#if defined(GCM_SMALL) \|\| defined(GCM_TABLE)
wolfSSL	15:117db924cf7c	3431
wolfSSL	15:117db924cf7c	3432	static WC_INLINE void FlattenSzInBits(byte* buf, word32 sz)
wolfSSL	15:117db924cf7c	3433	{
wolfSSL	15:117db924cf7c	3434	/* Multiply the sz by 8 */
wolfSSL	15:117db924cf7c	3435	word32 szHi = (sz >> (8*sizeof(sz) - 3));
wolfSSL	15:117db924cf7c	3436	sz <<= 3;
wolfSSL	15:117db924cf7c	3437
wolfSSL	15:117db924cf7c	3438	/* copy over the words of the sz into the destination buffer */
wolfSSL	15:117db924cf7c	3439	buf[0] = (szHi >> 24) & 0xff;
wolfSSL	15:117db924cf7c	3440	buf[1] = (szHi >> 16) & 0xff;
wolfSSL	15:117db924cf7c	3441	buf[2] = (szHi >> 8) & 0xff;
wolfSSL	15:117db924cf7c	3442	buf[3] = szHi & 0xff;
wolfSSL	15:117db924cf7c	3443	buf[4] = (sz >> 24) & 0xff;
wolfSSL	15:117db924cf7c	3444	buf[5] = (sz >> 16) & 0xff;
wolfSSL	15:117db924cf7c	3445	buf[6] = (sz >> 8) & 0xff;
wolfSSL	15:117db924cf7c	3446	buf[7] = sz & 0xff;
wolfSSL	15:117db924cf7c	3447	}
wolfSSL	15:117db924cf7c	3448
wolfSSL	15:117db924cf7c	3449
wolfSSL	15:117db924cf7c	3450	static WC_INLINE void RIGHTSHIFTX(byte* x)
wolfSSL	15:117db924cf7c	3451	{
wolfSSL	15:117db924cf7c	3452	int i;
wolfSSL	15:117db924cf7c	3453	int carryOut = 0;
wolfSSL	15:117db924cf7c	3454	int carryIn = 0;
wolfSSL	15:117db924cf7c	3455	int borrow = x[15] & 0x01;
wolfSSL	15:117db924cf7c	3456
wolfSSL	15:117db924cf7c	3457	for (i = 0; i < AES_BLOCK_SIZE; i++) {
wolfSSL	15:117db924cf7c	3458	carryOut = x[i] & 0x01;
wolfSSL	15:117db924cf7c	3459	x[i] = (x[i] >> 1) \| (carryIn ? 0x80 : 0);
wolfSSL	15:117db924cf7c	3460	carryIn = carryOut;
wolfSSL	15:117db924cf7c	3461	}
wolfSSL	15:117db924cf7c	3462	if (borrow) x[0] ^= 0xE1;
wolfSSL	15:117db924cf7c	3463	}
wolfSSL	15:117db924cf7c	3464
wolfSSL	15:117db924cf7c	3465	#endif /* defined(GCM_SMALL) \|\| defined(GCM_TABLE) */
wolfSSL	15:117db924cf7c	3466
wolfSSL	15:117db924cf7c	3467
wolfSSL	15:117db924cf7c	3468	#ifdef GCM_TABLE
wolfSSL	15:117db924cf7c	3469
wolfSSL	15:117db924cf7c	3470	static void GenerateM0(Aes* aes)
wolfSSL	15:117db924cf7c	3471	{
wolfSSL	15:117db924cf7c	3472	int i, j;
wolfSSL	15:117db924cf7c	3473	byte (*m)[AES_BLOCK_SIZE] = aes->M0;
wolfSSL	15:117db924cf7c	3474
wolfSSL	15:117db924cf7c	3475	XMEMCPY(m[128], aes->H, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3476
wolfSSL	15:117db924cf7c	3477	for (i = 64; i > 0; i /= 2) {
wolfSSL	15:117db924cf7c	3478	XMEMCPY(m[i], m[i*2], AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3479	RIGHTSHIFTX(m[i]);
wolfSSL	15:117db924cf7c	3480	}
wolfSSL	15:117db924cf7c	3481
wolfSSL	15:117db924cf7c	3482	for (i = 2; i < 256; i *= 2) {
wolfSSL	15:117db924cf7c	3483	for (j = 1; j < i; j++) {
wolfSSL	15:117db924cf7c	3484	XMEMCPY(m[i+j], m[i], AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3485	xorbuf(m[i+j], m[j], AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3486	}
wolfSSL	15:117db924cf7c	3487	}
wolfSSL	15:117db924cf7c	3488
wolfSSL	15:117db924cf7c	3489	XMEMSET(m[0], 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3490	}
wolfSSL	15:117db924cf7c	3491
wolfSSL	15:117db924cf7c	3492	#endif /* GCM_TABLE */
wolfSSL	15:117db924cf7c	3493
wolfSSL	15:117db924cf7c	3494
wolfSSL	15:117db924cf7c	3495	int wc_AesGcmSetKey(Aes* aes, const byte* key, word32 len)
wolfSSL	15:117db924cf7c	3496	{
wolfSSL	15:117db924cf7c	3497	int ret;
wolfSSL	15:117db924cf7c	3498	byte iv[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	3499
wolfSSL	15:117db924cf7c	3500	#ifdef WOLFSSL_IMX6_CAAM_BLOB
wolfSSL	15:117db924cf7c	3501	byte local[32];
wolfSSL	15:117db924cf7c	3502	word32 localSz = 32;
wolfSSL	15:117db924cf7c	3503
wolfSSL	15:117db924cf7c	3504	if (len == (16 + WC_CAAM_BLOB_SZ) \|\|
wolfSSL	15:117db924cf7c	3505	len == (24 + WC_CAAM_BLOB_SZ) \|\|
wolfSSL	15:117db924cf7c	3506	len == (32 + WC_CAAM_BLOB_SZ)) {
wolfSSL	15:117db924cf7c	3507	if (wc_caamOpenBlob((byte*)key, len, local, &localSz) != 0) {
wolfSSL	15:117db924cf7c	3508	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	3509	}
wolfSSL	15:117db924cf7c	3510
wolfSSL	15:117db924cf7c	3511	/* set local values */
wolfSSL	15:117db924cf7c	3512	key = local;
wolfSSL	15:117db924cf7c	3513	len = localSz;
wolfSSL	15:117db924cf7c	3514	}
wolfSSL	15:117db924cf7c	3515	#endif
wolfSSL	15:117db924cf7c	3516
wolfSSL	15:117db924cf7c	3517	if (!((len == 16) \|\| (len == 24) \|\| (len == 32)))
wolfSSL	15:117db924cf7c	3518	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	3519
wolfSSL	15:117db924cf7c	3520	XMEMSET(iv, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	3521	ret = wc_AesSetKey(aes, key, len, iv, AES_ENCRYPTION);
wolfSSL	15:117db924cf7c	3522
wolfSSL	15:117db924cf7c	3523	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	3524	/* AES-NI code generates its own H value. */
wolfSSL	15:117db924cf7c	3525	if (haveAESNI)
wolfSSL	15:117db924cf7c	3526	return ret;
wolfSSL	15:117db924cf7c	3527	#endif /* WOLFSSL_AESNI */
wolfSSL	15:117db924cf7c	3528
wolfSSL	15:117db924cf7c	3529	#if !defined(FREESCALE_LTC_AES_GCM)
wolfSSL	15:117db924cf7c	3530	if (ret == 0) {
wolfSSL	15:117db924cf7c	3531	wc_AesEncrypt(aes, iv, aes->H);
wolfSSL	15:117db924cf7c	3532	#ifdef GCM_TABLE
wolfSSL	15:117db924cf7c	3533	GenerateM0(aes);
wolfSSL	15:117db924cf7c	3534	#endif /* GCM_TABLE */
wolfSSL	15:117db924cf7c	3535	}
wolfSSL	15:117db924cf7c	3536	#endif /* FREESCALE_LTC_AES_GCM */
wolfSSL	15:117db924cf7c	3537
wolfSSL	15:117db924cf7c	3538	#if defined(WOLFSSL_XILINX_CRYPT)
wolfSSL	15:117db924cf7c	3539	wc_AesGcmSetKey_ex(aes, key, len, XSECURE_CSU_AES_KEY_SRC_KUP);
wolfSSL	15:117db924cf7c	3540	#endif
wolfSSL	15:117db924cf7c	3541
wolfSSL	15:117db924cf7c	3542	#ifdef WOLFSSL_IMX6_CAAM_BLOB
wolfSSL	15:117db924cf7c	3543	ForceZero(local, sizeof(local));
wolfSSL	15:117db924cf7c	3544	#endif
wolfSSL	15:117db924cf7c	3545
wolfSSL	15:117db924cf7c	3546	return ret;
wolfSSL	15:117db924cf7c	3547	}
wolfSSL	15:117db924cf7c	3548
wolfSSL	15:117db924cf7c	3549
wolfSSL	15:117db924cf7c	3550	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	3551
wolfSSL	15:117db924cf7c	3552	#if defined(USE_INTEL_SPEEDUP)
wolfSSL	15:117db924cf7c	3553	#define HAVE_INTEL_AVX1
wolfSSL	15:117db924cf7c	3554	#define HAVE_INTEL_AVX2
wolfSSL	15:117db924cf7c	3555	#endif /* USE_INTEL_SPEEDUP */
wolfSSL	15:117db924cf7c	3556
wolfSSL	15:117db924cf7c	3557	#ifdef _MSC_VER
wolfSSL	15:117db924cf7c	3558	#define S(w,z) ((char)((unsigned long long)(w) >> (8*(7-(z))) & 0xFF))
wolfSSL	15:117db924cf7c	3559	#define M128_INIT(x,y) { S((x),7), S((x),6), S((x),5), S((x),4), \
wolfSSL	15:117db924cf7c	3560	S((x),3), S((x),2), S((x),1), S((x),0), \
wolfSSL	15:117db924cf7c	3561	S((y),7), S((y),6), S((y),5), S((y),4), \
wolfSSL	15:117db924cf7c	3562	S((y),3), S((y),2), S((y),1), S((y),0) }
wolfSSL	15:117db924cf7c	3563	#else
wolfSSL	15:117db924cf7c	3564	#define M128_INIT(x,y) { (x), (y) }
wolfSSL	15:117db924cf7c	3565	#endif
wolfSSL	15:117db924cf7c	3566
wolfSSL	15:117db924cf7c	3567	static const __m128i MOD2_128 = M128_INIT(0x1,
wolfSSL	15:117db924cf7c	3568	(long long int)0xc200000000000000UL);
wolfSSL	15:117db924cf7c	3569
wolfSSL	15:117db924cf7c	3570
wolfSSL	15:117db924cf7c	3571	/* See Intel® Carry-Less Multiplication Instruction
wolfSSL	15:117db924cf7c	3572	* and its Usage for Computing the GCM Mode White Paper
wolfSSL	15:117db924cf7c	3573	* by Shay Gueron, Intel Mobility Group, Israel Development Center;
wolfSSL	15:117db924cf7c	3574	* and Michael E. Kounavis, Intel Labs, Circuits and Systems Research */
wolfSSL	15:117db924cf7c	3575
wolfSSL	15:117db924cf7c	3576
wolfSSL	15:117db924cf7c	3577	/* Figure 9. AES-GCM – Encrypt With Single Block Ghash at a Time */
wolfSSL	15:117db924cf7c	3578
wolfSSL	15:117db924cf7c	3579	static const __m128i ONE = M128_INIT(0x0, 0x1);
wolfSSL	15:117db924cf7c	3580	#ifndef AES_GCM_AESNI_NO_UNROLL
wolfSSL	15:117db924cf7c	3581	static const __m128i TWO = M128_INIT(0x0, 0x2);
wolfSSL	15:117db924cf7c	3582	static const __m128i THREE = M128_INIT(0x0, 0x3);
wolfSSL	15:117db924cf7c	3583	static const __m128i FOUR = M128_INIT(0x0, 0x4);
wolfSSL	15:117db924cf7c	3584	static const __m128i FIVE = M128_INIT(0x0, 0x5);
wolfSSL	15:117db924cf7c	3585	static const __m128i SIX = M128_INIT(0x0, 0x6);
wolfSSL	15:117db924cf7c	3586	static const __m128i SEVEN = M128_INIT(0x0, 0x7);
wolfSSL	15:117db924cf7c	3587	static const __m128i EIGHT = M128_INIT(0x0, 0x8);
wolfSSL	15:117db924cf7c	3588	#endif
wolfSSL	15:117db924cf7c	3589	static const __m128i BSWAP_EPI64 = M128_INIT(0x0001020304050607, 0x08090a0b0c0d0e0f);
wolfSSL	15:117db924cf7c	3590	static const __m128i BSWAP_MASK = M128_INIT(0x08090a0b0c0d0e0f, 0x0001020304050607);
wolfSSL	15:117db924cf7c	3591
wolfSSL	15:117db924cf7c	3592
wolfSSL	15:117db924cf7c	3593	#ifndef _MSC_VER
wolfSSL	15:117db924cf7c	3594
wolfSSL	15:117db924cf7c	3595	#define _VAR(a) "" #a ""
wolfSSL	15:117db924cf7c	3596	#define VAR(a) _VAR(a)
wolfSSL	15:117db924cf7c	3597
wolfSSL	15:117db924cf7c	3598	#define HR %%xmm14
wolfSSL	15:117db924cf7c	3599	#define XR %%xmm15
wolfSSL	15:117db924cf7c	3600	#define KR %%ebx
wolfSSL	15:117db924cf7c	3601	#define KR64 %%rbx
wolfSSL	15:117db924cf7c	3602	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	3603	#define CTR1 128(%%rsp)
wolfSSL	15:117db924cf7c	3604	#define TR 144(%%rsp)
wolfSSL	15:117db924cf7c	3605	#define HTR %%rsp
wolfSSL	15:117db924cf7c	3606	#define STACK_OFFSET 160
wolfSSL	15:117db924cf7c	3607	#else
wolfSSL	15:117db924cf7c	3608	#define CTR1 (%%rsp)
wolfSSL	15:117db924cf7c	3609	#define TR 16(%%rsp)
wolfSSL	15:117db924cf7c	3610	#define STACK_OFFSET 32
wolfSSL	15:117db924cf7c	3611	#endif
wolfSSL	15:117db924cf7c	3612
wolfSSL	15:117db924cf7c	3613	#define AESENC() \
wolfSSL	15:117db924cf7c	3614	"aesenc %%xmm12, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3615	"aesenc %%xmm12, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3616	"aesenc %%xmm12, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3617	"aesenc %%xmm12, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3618	"aesenc %%xmm12, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3619	"aesenc %%xmm12, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3620	"aesenc %%xmm12, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3621	"aesenc %%xmm12, %%xmm11\n\t"
wolfSSL	15:117db924cf7c	3622
wolfSSL	15:117db924cf7c	3623	#define AESENC_SET(o) \
wolfSSL	15:117db924cf7c	3624	"movdqa " #o "(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3625	AESENC()
wolfSSL	15:117db924cf7c	3626
wolfSSL	15:117db924cf7c	3627	#define AESENC_CTR() \
wolfSSL	15:117db924cf7c	3628	"movdqu " VAR(CTR1) ", %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3629	"movdqa %[BSWAP_EPI64], %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3630	"movdqu %%xmm4, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3631	"pshufb %%xmm1, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3632	"movdqa %%xmm0, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3633	"paddd %[ONE], %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3634	"pshufb %%xmm1, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3635	"movdqa %%xmm0, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3636	"paddd %[TWO], %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3637	"pshufb %%xmm1, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3638	"movdqa %%xmm0, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3639	"paddd %[THREE], %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3640	"pshufb %%xmm1, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3641	"movdqa %%xmm0, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3642	"paddd %[FOUR], %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3643	"pshufb %%xmm1, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3644	"movdqa %%xmm0, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3645	"paddd %[FIVE], %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3646	"pshufb %%xmm1, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3647	"movdqa %%xmm0, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3648	"paddd %[SIX], %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3649	"pshufb %%xmm1, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3650	"movdqa %%xmm0, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	3651	"paddd %[SEVEN], %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	3652	"pshufb %%xmm1, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	3653	"paddd %[EIGHT], %%xmm0\n\t"
wolfSSL	15:117db924cf7c	3654
wolfSSL	15:117db924cf7c	3655	#define AESENC_XOR() \
wolfSSL	15:117db924cf7c	3656	"movdqa (%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3657	"movdqu %%xmm0, " VAR(CTR1) "\n\t" \
wolfSSL	15:117db924cf7c	3658	"pxor %%xmm12, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3659	"pxor %%xmm12, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3660	"pxor %%xmm12, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3661	"pxor %%xmm12, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3662	"pxor %%xmm12, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3663	"pxor %%xmm12, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3664	"pxor %%xmm12, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3665	"pxor %%xmm12, %%xmm11\n\t"
wolfSSL	15:117db924cf7c	3666
wolfSSL	15:117db924cf7c	3667	/* Encrypt and carry-less multiply for AVX1. */
wolfSSL	15:117db924cf7c	3668	#define AESENC_PCLMUL_1(src, o1, o2, o3) \
wolfSSL	15:117db924cf7c	3669	"movdqu " #o3 "(" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3670	"movdqu " #o2 "(" #src "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3671	"aesenc " #o1 "(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3672	"pshufb %[BSWAP_MASK], %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3673	"pxor %%xmm2, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3674	"pshufd $0x4e, %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3675	"pshufd $0x4e, %%xmm0, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3676	"pxor %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3677	"pxor %%xmm0, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3678	"movdqa %%xmm0, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3679	"pclmulqdq $0x11, %%xmm12, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3680	"aesenc " #o1 "(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3681	"aesenc " #o1 "(%[KEY]), %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3682	"movdqa %%xmm0, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3683	"pclmulqdq $0x00, %%xmm12, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3684	"aesenc " #o1 "(%[KEY]), %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3685	"aesenc " #o1 "(%[KEY]), %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3686	"pclmulqdq $0x00, %%xmm14, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3687	"aesenc " #o1 "(%[KEY]), %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3688	"aesenc " #o1 "(%[KEY]), %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3689	"aesenc " #o1 "(%[KEY]), %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	3690	"pxor %%xmm2, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3691	"pxor %%xmm3, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3692
wolfSSL	15:117db924cf7c	3693	#define AESENC_PCLMUL_N(src, o1, o2, o3) \
wolfSSL	15:117db924cf7c	3694	"movdqu " #o3 "(" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3695	"movdqu " #o2 "(" #src" ), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3696	"pshufd $0x4e, %%xmm12, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	3697	"pshufb %[BSWAP_MASK], %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3698	"aesenc " #o1 "(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3699	"pxor %%xmm12, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	3700	"pshufd $0x4e, %%xmm0, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3701	"pxor %%xmm0, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3702	"movdqa %%xmm0, %%xmm15\n\t" \
wolfSSL	15:117db924cf7c	3703	"pclmulqdq $0x11, %%xmm12, %%xmm15\n\t" \
wolfSSL	15:117db924cf7c	3704	"aesenc " #o1 "(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3705	"aesenc " #o1 "(%[KEY]), %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3706	"pclmulqdq $0x00, %%xmm0, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3707	"aesenc " #o1 "(%[KEY]), %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3708	"aesenc " #o1 "(%[KEY]), %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3709	"pclmulqdq $0x00, %%xmm14, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	3710	"aesenc " #o1 "(%[KEY]), %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3711	"aesenc " #o1 "(%[KEY]), %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3712	"aesenc " #o1 "(%[KEY]), %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	3713	"pxor %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3714	"pxor %%xmm12, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3715	"pxor %%xmm15, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3716	"pxor %%xmm15, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3717	"pxor %%xmm13, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3718
wolfSSL	15:117db924cf7c	3719	#define AESENC_PCLMUL_L(o) \
wolfSSL	15:117db924cf7c	3720	"movdqa %%xmm1, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3721	"psrldq $8, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3722	"pslldq $8, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3723	"aesenc " #o "(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3724	"pxor %%xmm14, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3725	"pxor %%xmm1, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3726	"movdqa %%xmm2, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3727	"movdqa %%xmm2, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	3728	"movdqa %%xmm2, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3729	"aesenc " #o "(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3730	"pslld $31, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3731	"pslld $30, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	3732	"pslld $25, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3733	"aesenc " #o "(%[KEY]), %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3734	"pxor %%xmm13, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3735	"pxor %%xmm14, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3736	"aesenc " #o "(%[KEY]), %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3737	"movdqa %%xmm12, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	3738	"pslldq $12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	3739	"psrldq $4, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	3740	"aesenc " #o "(%[KEY]), %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3741	"pxor %%xmm12, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3742	"movdqa %%xmm2, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3743	"movdqa %%xmm2, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3744	"movdqa %%xmm2, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3745	"aesenc " #o "(%[KEY]), %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3746	"psrld $1, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3747	"psrld $2, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3748	"psrld $7, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3749	"aesenc " #o "(%[KEY]), %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3750	"pxor %%xmm1, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3751	"pxor %%xmm0, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3752	"aesenc " #o "(%[KEY]), %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	3753	"pxor %%xmm13, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	3754	"pxor %%xmm14, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3755	"pxor %%xmm3, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3756
wolfSSL	15:117db924cf7c	3757	/* Encrypt and carry-less multiply with last key. */
wolfSSL	15:117db924cf7c	3758	#define AESENC_LAST(in, out) \
wolfSSL	15:117db924cf7c	3759	"aesenclast %%xmm12, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3760	"aesenclast %%xmm12, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3761	"movdqu (" #in "),%%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3762	"movdqu 16(" #in "),%%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3763	"pxor %%xmm0, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3764	"pxor %%xmm1, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3765	"movdqu %%xmm4, (" #out ")\n\t" \
wolfSSL	15:117db924cf7c	3766	"movdqu %%xmm5, 16(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	3767	"aesenclast %%xmm12, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3768	"aesenclast %%xmm12, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3769	"movdqu 32(" #in "),%%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3770	"movdqu 48(" #in "),%%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3771	"pxor %%xmm0, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3772	"pxor %%xmm1, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3773	"movdqu %%xmm6, 32(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	3774	"movdqu %%xmm7, 48(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	3775	"aesenclast %%xmm12, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3776	"aesenclast %%xmm12, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3777	"movdqu 64(" #in "),%%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3778	"movdqu 80(" #in "),%%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3779	"pxor %%xmm0, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3780	"pxor %%xmm1, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3781	"movdqu %%xmm8, 64(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	3782	"movdqu %%xmm9, 80(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	3783	"aesenclast %%xmm12, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3784	"aesenclast %%xmm12, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	3785	"movdqu 96(" #in "),%%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3786	"movdqu 112(" #in "),%%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3787	"pxor %%xmm0, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3788	"pxor %%xmm1, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	3789	"movdqu %%xmm10, 96(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	3790	"movdqu %%xmm11, 112(" #out ")\n\t"
wolfSSL	15:117db924cf7c	3791
wolfSSL	15:117db924cf7c	3792	#define _AESENC_AVX(r) \
wolfSSL	15:117db924cf7c	3793	"aesenc 16(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3794	"aesenc 32(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3795	"aesenc 48(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3796	"aesenc 64(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3797	"aesenc 80(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3798	"aesenc 96(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3799	"aesenc 112(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3800	"aesenc 128(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3801	"aesenc 144(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3802	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	3803	"movdqa 160(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3804	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	3805	"aesenc %%xmm5, " #r "\n\t" \
wolfSSL	15:117db924cf7c	3806	"aesenc 176(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3807	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	3808	"movdqa 192(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3809	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	3810	"aesenc %%xmm5, " #r "\n\t" \
wolfSSL	15:117db924cf7c	3811	"aesenc 208(%[KEY]), " #r "\n\t" \
wolfSSL	15:117db924cf7c	3812	"movdqa 224(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3813	"%=:\n\t" \
wolfSSL	15:117db924cf7c	3814	"aesenclast %%xmm5, " #r "\n\t"
wolfSSL	15:117db924cf7c	3815	#define AESENC_AVX(r) \
wolfSSL	15:117db924cf7c	3816	_AESENC_AVX(r)
wolfSSL	15:117db924cf7c	3817
wolfSSL	15:117db924cf7c	3818	#define AESENC_BLOCK(in, out) \
wolfSSL	15:117db924cf7c	3819	"movdqu " VAR(CTR1) ", %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3820	"movdqu %%xmm4, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3821	"pshufb %[BSWAP_EPI64], %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3822	"paddd %[ONE], %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3823	"pxor (%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3824	"movdqu %%xmm5, " VAR(CTR1) "\n\t" \
wolfSSL	15:117db924cf7c	3825	AESENC_AVX(%%xmm4) \
wolfSSL	15:117db924cf7c	3826	"movdqu (" #in "), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3827	"pxor %%xmm5, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3828	"movdqu %%xmm4, (" #out ")\n\t" \
wolfSSL	15:117db924cf7c	3829	"pshufb %[BSWAP_MASK], %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3830	"pxor %%xmm4, " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	3831
wolfSSL	15:117db924cf7c	3832	#define _AESENC_GFMUL(in, out, H, X) \
wolfSSL	15:117db924cf7c	3833	"movdqu " VAR(CTR1) ", %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3834	"movdqu %%xmm4, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3835	"pshufb %[BSWAP_EPI64], %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3836	"paddd %[ONE], %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3837	"pxor (%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3838	"movdqu %%xmm5, " VAR(CTR1) "\n\t" \
wolfSSL	15:117db924cf7c	3839	"movdqa " #X ", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3840	"pclmulqdq $0x10, " #H ", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3841	"aesenc 16(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3842	"aesenc 32(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3843	"movdqa " #X ", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3844	"pclmulqdq $0x01, " #H ", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3845	"aesenc 48(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3846	"aesenc 64(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3847	"movdqa " #X ", %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3848	"pclmulqdq $0x00, " #H ", %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3849	"aesenc 80(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3850	"movdqa " #X ", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3851	"pclmulqdq $0x11, " #H ", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3852	"aesenc 96(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3853	"pxor %%xmm7, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3854	"movdqa %%xmm6, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3855	"psrldq $8, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3856	"pslldq $8, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3857	"aesenc 112(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3858	"movdqa %%xmm1, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3859	"pxor %%xmm8, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3860	"pxor %%xmm6, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3861	"movdqa %[MOD2_128], %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3862	"movdqa %%xmm2, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3863	"pclmulqdq $0x10, %%xmm0, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3864	"aesenc 128(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3865	"pshufd $0x4e, %%xmm2, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3866	"pxor %%xmm7, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3867	"movdqa %%xmm6, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3868	"pclmulqdq $0x10, %%xmm0, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3869	"aesenc 144(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3870	"pshufd $0x4e, %%xmm6, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	3871	"pxor %%xmm7, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	3872	"pxor %%xmm3, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	3873	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	3874	"movdqu 160(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3875	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	3876	"aesenc %%xmm5, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3877	"aesenc 176(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3878	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	3879	"movdqu 192(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3880	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	3881	"aesenc %%xmm5, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3882	"aesenc 208(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3883	"movdqa 224(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3884	"%=:\n\t" \
wolfSSL	15:117db924cf7c	3885	"aesenclast %%xmm5, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3886	"movdqu (" #in "), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3887	"pxor %%xmm5, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3888	"movdqu %%xmm4, (" #out ")\n\t"
wolfSSL	15:117db924cf7c	3889	#define AESENC_GFMUL(in, out, H, X) \
wolfSSL	15:117db924cf7c	3890	_AESENC_GFMUL(in, out, H, X)
wolfSSL	15:117db924cf7c	3891
wolfSSL	15:117db924cf7c	3892	#define _GHASH_GFMUL_AVX(r, r2, a, b) \
wolfSSL	15:117db924cf7c	3893	"pshufd $0x4e, "#a", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3894	"pshufd $0x4e, "#b", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3895	"movdqa "#b", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3896	"movdqa "#b", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3897	"pclmulqdq $0x11, "#a", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3898	"pclmulqdq $0x00, "#a", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3899	"pxor "#a", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3900	"pxor "#b", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3901	"pclmulqdq $0x00, %%xmm2, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3902	"pxor %%xmm0, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3903	"pxor %%xmm3, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3904	"movdqa %%xmm1, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3905	"movdqa %%xmm0, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	3906	"movdqa %%xmm3, " #r "\n\t" \
wolfSSL	15:117db924cf7c	3907	"pslldq $8, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3908	"psrldq $8, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3909	"pxor %%xmm2, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	3910	"pxor %%xmm1, " #r "\n\t"
wolfSSL	15:117db924cf7c	3911	#define GHASH_GFMUL_AVX(r, r2, a, b) \
wolfSSL	15:117db924cf7c	3912	_GHASH_GFMUL_AVX(r, r2, a, b)
wolfSSL	15:117db924cf7c	3913
wolfSSL	15:117db924cf7c	3914	#define _GHASH_GFMUL_XOR_AVX(r, r2, a, b) \
wolfSSL	15:117db924cf7c	3915	"pshufd $0x4e, "#a", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3916	"pshufd $0x4e, "#b", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3917	"movdqa "#b", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3918	"movdqa "#b", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3919	"pclmulqdq $0x11, "#a", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	3920	"pclmulqdq $0x00, "#a", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3921	"pxor "#a", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3922	"pxor "#b", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3923	"pclmulqdq $0x00, %%xmm2, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3924	"pxor %%xmm0, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3925	"pxor %%xmm3, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3926	"movdqa %%xmm1, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3927	"pxor %%xmm0, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	3928	"pxor %%xmm3, " #r "\n\t" \
wolfSSL	15:117db924cf7c	3929	"pslldq $8, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3930	"psrldq $8, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3931	"pxor %%xmm2, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	3932	"pxor %%xmm1, " #r "\n\t"
wolfSSL	15:117db924cf7c	3933	#define GHASH_GFMUL_XOR_AVX(r, r2, a, b) \
wolfSSL	15:117db924cf7c	3934	_GHASH_GFMUL_XOR_AVX(r, r2, a, b)
wolfSSL	15:117db924cf7c	3935
wolfSSL	15:117db924cf7c	3936	#define GHASH_MID_AVX(r, r2) \
wolfSSL	15:117db924cf7c	3937	"movdqa "#r2", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3938	"movdqa " #r ", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3939	"psrld $31, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3940	"psrld $31, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3941	"pslld $1, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	3942	"pslld $1, " #r "\n\t" \
wolfSSL	15:117db924cf7c	3943	"movdqa %%xmm0, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3944	"pslldq $4, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3945	"psrldq $12, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3946	"pslldq $4, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3947	"por %%xmm2, " #r "\n\t" \
wolfSSL	15:117db924cf7c	3948	"por %%xmm0, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	3949	"por %%xmm1, " #r "\n\t"
wolfSSL	15:117db924cf7c	3950
wolfSSL	15:117db924cf7c	3951	#define _GHASH_GFMUL_RED_AVX(r, a, b) \
wolfSSL	15:117db924cf7c	3952	"pshufd $0x4e, "#a", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3953	"pshufd $0x4e, "#b", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3954	"movdqa "#b", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3955	"movdqa "#b", %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3956	"pclmulqdq $0x11, "#a", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	3957	"pclmulqdq $0x00, "#a", %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3958	"pxor "#a", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3959	"pxor "#b", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3960	"pclmulqdq $0x00, %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3961	"pxor %%xmm4, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3962	"pxor %%xmm7, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3963	"movdqa %%xmm5, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3964	"movdqa %%xmm7, " #r "\n\t" \
wolfSSL	15:117db924cf7c	3965	"pslldq $8, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3966	"psrldq $8, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3967	"pxor %%xmm6, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3968	"pxor %%xmm5, " #r "\n\t" \
wolfSSL	15:117db924cf7c	3969	"movdqa %%xmm4, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3970	"movdqa %%xmm4, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3971	"movdqa %%xmm4, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3972	"pslld $31, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3973	"pslld $30, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3974	"pslld $25, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3975	"pxor %%xmm9, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3976	"pxor %%xmm10, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3977	"movdqa %%xmm8, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3978	"psrldq $4, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	3979	"pslldq $12, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	3980	"pxor %%xmm8, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	3981	"movdqa %%xmm4, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3982	"movdqa %%xmm4, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3983	"movdqa %%xmm4, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3984	"psrld $1, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3985	"psrld $2, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	3986	"psrld $7, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	3987	"pxor %%xmm6, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3988	"pxor %%xmm5, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3989	"pxor %%xmm9, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3990	"pxor %%xmm4, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	3991	"pxor %%xmm10, " #r "\n\t"
wolfSSL	15:117db924cf7c	3992	#define GHASH_GFMUL_RED_AVX(r, a, b) \
wolfSSL	15:117db924cf7c	3993	_GHASH_GFMUL_RED_AVX(r, a, b)
wolfSSL	15:117db924cf7c	3994
wolfSSL	15:117db924cf7c	3995	#define GHASH_RED_AVX(r, r2) \
wolfSSL	15:117db924cf7c	3996	"movdqa "#r2", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	3997	"movdqa "#r2", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	3998	"movdqa "#r2", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	3999	"pslld $31, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4000	"pslld $30, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4001	"pslld $25, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4002	"pxor %%xmm1, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4003	"pxor %%xmm2, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4004	"movdqa %%xmm0, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4005	"psrldq $4, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4006	"pslldq $12, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4007	"pxor %%xmm0, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	4008	"movdqa "#r2", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4009	"movdqa "#r2", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4010	"movdqa "#r2", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4011	"psrld $1, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4012	"psrld $2, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4013	"psrld $7, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4014	"pxor %%xmm3, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4015	"pxor %%xmm0, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4016	"pxor %%xmm1, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4017	"pxor "#r2", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4018	"pxor %%xmm2, " #r "\n\t"
wolfSSL	15:117db924cf7c	4019
wolfSSL	15:117db924cf7c	4020	#define GHASH_GFMUL_RED_XOR_AVX(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4021	GHASH_GFMUL_XOR_AVX(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4022	GHASH_RED_AVX(r, r2)
wolfSSL	15:117db924cf7c	4023
wolfSSL	15:117db924cf7c	4024	#define GHASH_FULL_AVX(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4025	GHASH_GFMUL_AVX(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4026	GHASH_MID_AVX(r, r2) \
wolfSSL	15:117db924cf7c	4027	GHASH_RED_AVX(r, r2)
wolfSSL	15:117db924cf7c	4028
wolfSSL	15:117db924cf7c	4029	#define CALC_IV_12() \
wolfSSL	15:117db924cf7c	4030	"# Calculate values when IV is 12 bytes\n\t" \
wolfSSL	15:117db924cf7c	4031	"# Set counter based on IV\n\t" \
wolfSSL	15:117db924cf7c	4032	"movl $0x01000000, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4033	"pinsrq $0, 0(%%rax), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4034	"pinsrd $2, 8(%%rax), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4035	"pinsrd $3, %%ecx, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4036	"# H = Encrypt X(=0) and T = Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	4037	"movdqu %%xmm13, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4038	"movdqa 0(%[KEY]), " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4039	"pxor " VAR(HR) ", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4040	"movdqa 16(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4041	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4042	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4043	"movdqa 32(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4044	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4045	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4046	"movdqa 48(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4047	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4048	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4049	"movdqa 64(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4050	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4051	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4052	"movdqa 80(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4053	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4054	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4055	"movdqa 96(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4056	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4057	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4058	"movdqa 112(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4059	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4060	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4061	"movdqa 128(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4062	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4063	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4064	"movdqa 144(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4065	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4066	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4067	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4068	"movdqa 160(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4069	"jl 31f\n\t" \
wolfSSL	15:117db924cf7c	4070	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4071	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4072	"movdqa 176(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4073	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4074	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4075	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4076	"movdqa 192(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4077	"jl 31f\n\t" \
wolfSSL	15:117db924cf7c	4078	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4079	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4080	"movdqu 208(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4081	"aesenc %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4082	"aesenc %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4083	"movdqu 224(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4084	"31:\n\t" \
wolfSSL	15:117db924cf7c	4085	"aesenclast %%xmm12, " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4086	"aesenclast %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4087	"pshufb %[BSWAP_MASK], " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4088	"movdqu %%xmm1, " VAR(TR) "\n\t" \
wolfSSL	15:117db924cf7c	4089	"jmp 39f\n\t"
wolfSSL	15:117db924cf7c	4090
wolfSSL	15:117db924cf7c	4091	#define CALC_IV() \
wolfSSL	15:117db924cf7c	4092	"# Calculate values when IV is not 12 bytes\n\t" \
wolfSSL	15:117db924cf7c	4093	"# H = Encrypt X(=0)\n\t" \
wolfSSL	15:117db924cf7c	4094	"movdqa 0(%[KEY]), " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4095	AESENC_AVX(HR) \
wolfSSL	15:117db924cf7c	4096	"pshufb %[BSWAP_MASK], " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4097	"# Calc counter\n\t" \
wolfSSL	15:117db924cf7c	4098	"# Initialization vector\n\t" \
wolfSSL	15:117db924cf7c	4099	"cmpl $0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4100	"movq $0, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	4101	"je 45f\n\t" \
wolfSSL	15:117db924cf7c	4102	"cmpl $16, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4103	"jl 44f\n\t" \
wolfSSL	15:117db924cf7c	4104	"andl $0xfffffff0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4105	"\n" \
wolfSSL	15:117db924cf7c	4106	"43:\n\t" \
wolfSSL	15:117db924cf7c	4107	"movdqu (%%rax,%%rcx,1), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4108	"pshufb %[BSWAP_MASK], %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4109	"pxor %%xmm4, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4110	GHASH_FULL_AVX(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	4111	"addl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4112	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4113	"jl 43b\n\t" \
wolfSSL	15:117db924cf7c	4114	"movl %[ibytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	4115	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4116	"je 45f\n\t" \
wolfSSL	15:117db924cf7c	4117	"\n" \
wolfSSL	15:117db924cf7c	4118	"44:\n\t" \
wolfSSL	15:117db924cf7c	4119	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4120	"pxor %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4121	"xorl %%ebx, %%ebx\n\t" \
wolfSSL	15:117db924cf7c	4122	"movdqu %%xmm4, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	4123	"42:\n\t" \
wolfSSL	15:117db924cf7c	4124	"movzbl (%%rax,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	4125	"movb %%r13b, (%%rsp,%%rbx,1)\n\t" \
wolfSSL	15:117db924cf7c	4126	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4127	"incl %%ebx\n\t" \
wolfSSL	15:117db924cf7c	4128	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4129	"jl 42b\n\t" \
wolfSSL	15:117db924cf7c	4130	"movdqu (%%rsp), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4131	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4132	"pshufb %[BSWAP_MASK], %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4133	"pxor %%xmm4, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4134	GHASH_FULL_AVX(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	4135	"\n" \
wolfSSL	15:117db924cf7c	4136	"45:\n\t" \
wolfSSL	15:117db924cf7c	4137	"# T = Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	4138	"pxor %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4139	"shll $3, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4140	"pinsrq $0, %%rdx, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4141	"pxor %%xmm0, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4142	GHASH_FULL_AVX(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	4143	"pshufb %[BSWAP_MASK], %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4144	"# Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	4145	"movdqa 0(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4146	"pxor %%xmm13, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4147	AESENC_AVX(%%xmm4) \
wolfSSL	15:117db924cf7c	4148	"movdqu %%xmm4, " VAR(TR) "\n\t"
wolfSSL	15:117db924cf7c	4149
wolfSSL	15:117db924cf7c	4150	#define CALC_AAD() \
wolfSSL	15:117db924cf7c	4151	"# Additional authentication data\n\t" \
wolfSSL	15:117db924cf7c	4152	"movl %[abytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	4153	"cmpl $0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4154	"je 25f\n\t" \
wolfSSL	15:117db924cf7c	4155	"movq %[addt], %%rax\n\t" \
wolfSSL	15:117db924cf7c	4156	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4157	"cmpl $16, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4158	"jl 24f\n\t" \
wolfSSL	15:117db924cf7c	4159	"andl $0xfffffff0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4160	"\n" \
wolfSSL	15:117db924cf7c	4161	"23:\n\t" \
wolfSSL	15:117db924cf7c	4162	"movdqu (%%rax,%%rcx,1), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4163	"pshufb %[BSWAP_MASK], %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4164	"pxor %%xmm4, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	4165	GHASH_FULL_AVX(XR, %%xmm12, XR, HR) \
wolfSSL	15:117db924cf7c	4166	"addl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4167	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4168	"jl 23b\n\t" \
wolfSSL	15:117db924cf7c	4169	"movl %[abytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	4170	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4171	"je 25f\n\t" \
wolfSSL	15:117db924cf7c	4172	"\n" \
wolfSSL	15:117db924cf7c	4173	"24:\n\t" \
wolfSSL	15:117db924cf7c	4174	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4175	"pxor %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4176	"xorl %%ebx, %%ebx\n\t" \
wolfSSL	15:117db924cf7c	4177	"movdqu %%xmm4, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	4178	"22:\n\t" \
wolfSSL	15:117db924cf7c	4179	"movzbl (%%rax,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	4180	"movb %%r13b, (%%rsp,%%rbx,1)\n\t" \
wolfSSL	15:117db924cf7c	4181	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4182	"incl %%ebx\n\t" \
wolfSSL	15:117db924cf7c	4183	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4184	"jl 22b\n\t" \
wolfSSL	15:117db924cf7c	4185	"movdqu (%%rsp), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4186	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4187	"pshufb %[BSWAP_MASK], %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4188	"pxor %%xmm4, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	4189	GHASH_FULL_AVX(XR, %%xmm12, XR, HR) \
wolfSSL	15:117db924cf7c	4190	"\n" \
wolfSSL	15:117db924cf7c	4191	"25:\n\t"
wolfSSL	15:117db924cf7c	4192
wolfSSL	15:117db924cf7c	4193	#define CALC_HT_8_AVX() \
wolfSSL	15:117db924cf7c	4194	"movdqa " VAR(XR) ", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4195	"# H ^ 1\n\t" \
wolfSSL	15:117db924cf7c	4196	"movdqu " VAR(HR) ", 0(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	4197	"# H ^ 2\n\t" \
wolfSSL	15:117db924cf7c	4198	GHASH_GFMUL_RED_AVX(%%xmm0, HR, HR) \
wolfSSL	15:117db924cf7c	4199	"movdqu %%xmm0 , 16(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	4200	"# H ^ 3\n\t" \
wolfSSL	15:117db924cf7c	4201	GHASH_GFMUL_RED_AVX(%%xmm1, HR, %%xmm0) \
wolfSSL	15:117db924cf7c	4202	"movdqu %%xmm1 , 32(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	4203	"# H ^ 4\n\t" \
wolfSSL	15:117db924cf7c	4204	GHASH_GFMUL_RED_AVX(%%xmm3, %%xmm0, %%xmm0) \
wolfSSL	15:117db924cf7c	4205	"movdqu %%xmm3 , 48(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	4206	"# H ^ 5\n\t" \
wolfSSL	15:117db924cf7c	4207	GHASH_GFMUL_RED_AVX(%%xmm12, %%xmm0, %%xmm1) \
wolfSSL	15:117db924cf7c	4208	"movdqu %%xmm12, 64(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	4209	"# H ^ 6\n\t" \
wolfSSL	15:117db924cf7c	4210	GHASH_GFMUL_RED_AVX(%%xmm12, %%xmm1, %%xmm1) \
wolfSSL	15:117db924cf7c	4211	"movdqu %%xmm12, 80(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	4212	"# H ^ 7\n\t" \
wolfSSL	15:117db924cf7c	4213	GHASH_GFMUL_RED_AVX(%%xmm12, %%xmm1, %%xmm3) \
wolfSSL	15:117db924cf7c	4214	"movdqu %%xmm12, 96(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	4215	"# H ^ 8\n\t" \
wolfSSL	15:117db924cf7c	4216	GHASH_GFMUL_RED_AVX(%%xmm12, %%xmm3, %%xmm3) \
wolfSSL	15:117db924cf7c	4217	"movdqu %%xmm12, 112(" VAR(HTR) ")\n\t"
wolfSSL	15:117db924cf7c	4218
wolfSSL	15:117db924cf7c	4219	#define AESENC_128_GHASH_AVX(src, o) \
wolfSSL	15:117db924cf7c	4220	"leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" \
wolfSSL	15:117db924cf7c	4221	"leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" \
wolfSSL	15:117db924cf7c	4222	/* src is either %%rcx or %%rdx */ \
wolfSSL	15:117db924cf7c	4223	AESENC_CTR() \
wolfSSL	15:117db924cf7c	4224	AESENC_XOR() \
wolfSSL	15:117db924cf7c	4225	AESENC_PCLMUL_1(src, 16, o-128, 112) \
wolfSSL	15:117db924cf7c	4226	AESENC_PCLMUL_N(src, 32, o-112, 96) \
wolfSSL	15:117db924cf7c	4227	AESENC_PCLMUL_N(src, 48, o -96, 80) \
wolfSSL	15:117db924cf7c	4228	AESENC_PCLMUL_N(src, 64, o -80, 64) \
wolfSSL	15:117db924cf7c	4229	AESENC_PCLMUL_N(src, 80, o -64, 48) \
wolfSSL	15:117db924cf7c	4230	AESENC_PCLMUL_N(src, 96, o -48, 32) \
wolfSSL	15:117db924cf7c	4231	AESENC_PCLMUL_N(src, 112, o -32, 16) \
wolfSSL	15:117db924cf7c	4232	AESENC_PCLMUL_N(src, 128, o -16, 0) \
wolfSSL	15:117db924cf7c	4233	AESENC_PCLMUL_L(144) \
wolfSSL	15:117db924cf7c	4234	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4235	"movdqa 160(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4236	"jl 4f\n\t" \
wolfSSL	15:117db924cf7c	4237	AESENC() \
wolfSSL	15:117db924cf7c	4238	AESENC_SET(176) \
wolfSSL	15:117db924cf7c	4239	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4240	"movdqa 192(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4241	"jl 4f\n\t" \
wolfSSL	15:117db924cf7c	4242	AESENC() \
wolfSSL	15:117db924cf7c	4243	AESENC_SET(208) \
wolfSSL	15:117db924cf7c	4244	"movdqa 224(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4245	"\n" \
wolfSSL	15:117db924cf7c	4246	"4:\n\t" \
wolfSSL	15:117db924cf7c	4247	AESENC_LAST(%%rcx, %%rdx)
wolfSSL	15:117db924cf7c	4248
wolfSSL	15:117db924cf7c	4249	#define AESENC_LAST15_ENC_AVX() \
wolfSSL	15:117db924cf7c	4250	"movl %[nbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4251	"movl %%ecx, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4252	"andl $0x0f, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4253	"jz 55f\n\t" \
wolfSSL	15:117db924cf7c	4254	"movdqu " VAR(CTR1) ", %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4255	"pshufb %[BSWAP_EPI64], %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4256	"pxor 0(%[KEY]), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4257	AESENC_AVX(%%xmm13) \
wolfSSL	15:117db924cf7c	4258	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4259	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4260	"movdqu %%xmm13, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	4261	"\n" \
wolfSSL	15:117db924cf7c	4262	"51:\n\t" \
wolfSSL	15:117db924cf7c	4263	"movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	4264	"xorb (%%rsp,%%rcx,1), %%r13b\n\t" \
wolfSSL	15:117db924cf7c	4265	"movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \
wolfSSL	15:117db924cf7c	4266	"movb %%r13b, (%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	4267	"incl " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	4268	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4269	"cmpl %%edx, " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	4270	"jl 51b\n\t" \
wolfSSL	15:117db924cf7c	4271	"xorq %%r13, %%r13\n\t" \
wolfSSL	15:117db924cf7c	4272	"cmpl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4273	"je 53f\n\t" \
wolfSSL	15:117db924cf7c	4274	"\n" \
wolfSSL	15:117db924cf7c	4275	"52:\n\t" \
wolfSSL	15:117db924cf7c	4276	"movb %%r13b, (%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	4277	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4278	"cmpl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4279	"jl 52b\n\t" \
wolfSSL	15:117db924cf7c	4280	"53:\n\t" \
wolfSSL	15:117db924cf7c	4281	"movdqu (%%rsp), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4282	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4283	"pshufb %[BSWAP_MASK], %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4284	"pxor %%xmm13, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	4285	GHASH_GFMUL_RED_AVX(XR, HR, XR) \
wolfSSL	15:117db924cf7c	4286
wolfSSL	15:117db924cf7c	4287	#define AESENC_LAST15_DEC_AVX() \
wolfSSL	15:117db924cf7c	4288	"movl %[nbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4289	"movl %%ecx, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4290	"andl $0x0f, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4291	"jz 55f\n\t" \
wolfSSL	15:117db924cf7c	4292	"movdqu " VAR(CTR1) ", %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4293	"pshufb %[BSWAP_EPI64], %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4294	"pxor 0(%[KEY]), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4295	AESENC_AVX(%%xmm13) \
wolfSSL	15:117db924cf7c	4296	"subq $32, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4297	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4298	"movdqu %%xmm13, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	4299	"pxor %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4300	"movdqu %%xmm0, 16(%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	4301	"\n" \
wolfSSL	15:117db924cf7c	4302	"51:\n\t" \
wolfSSL	15:117db924cf7c	4303	"movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	4304	"movb %%r13b, 16(%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	4305	"xorb (%%rsp,%%rcx,1), %%r13b\n\t" \
wolfSSL	15:117db924cf7c	4306	"movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \
wolfSSL	15:117db924cf7c	4307	"incl " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	4308	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4309	"cmpl %%edx, " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	4310	"jl 51b\n\t" \
wolfSSL	15:117db924cf7c	4311	"53:\n\t" \
wolfSSL	15:117db924cf7c	4312	"movdqu 16(%%rsp), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4313	"addq $32, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4314	"pshufb %[BSWAP_MASK], %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4315	"pxor %%xmm13, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	4316	GHASH_GFMUL_RED_AVX(XR, HR, XR) \
wolfSSL	15:117db924cf7c	4317
wolfSSL	15:117db924cf7c	4318	#define CALC_TAG() \
wolfSSL	15:117db924cf7c	4319	"movl %[nbytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	4320	"movl %[abytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4321	"shlq $3, %%rdx\n\t" \
wolfSSL	15:117db924cf7c	4322	"shlq $3, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	4323	"pinsrq $0, %%rdx, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4324	"pinsrq $1, %%rcx, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4325	"pxor %%xmm0, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	4326	GHASH_GFMUL_RED_AVX(XR, HR, XR) \
wolfSSL	15:117db924cf7c	4327	"pshufb %[BSWAP_MASK], " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	4328	"movdqu " VAR(TR) ", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4329	"pxor " VAR(XR) ", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4330
wolfSSL	15:117db924cf7c	4331	#define STORE_TAG() \
wolfSSL	15:117db924cf7c	4332	"cmpl $16, %[tbytes]\n\t" \
wolfSSL	15:117db924cf7c	4333	"je 71f\n\t" \
wolfSSL	15:117db924cf7c	4334	"xorq %%rcx, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	4335	"movdqu %%xmm0, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	4336	"73:\n\t" \
wolfSSL	15:117db924cf7c	4337	"movzbl (%%rsp,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	4338	"movb %%r13b, (%[tag],%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	4339	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4340	"cmpl %[tbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4341	"jne 73b\n\t" \
wolfSSL	15:117db924cf7c	4342	"jmp 72f\n\t" \
wolfSSL	15:117db924cf7c	4343	"\n" \
wolfSSL	15:117db924cf7c	4344	"71:\n\t" \
wolfSSL	15:117db924cf7c	4345	"movdqu %%xmm0, (%[tag])\n\t" \
wolfSSL	15:117db924cf7c	4346	"\n" \
wolfSSL	15:117db924cf7c	4347	"72:\n\t"
wolfSSL	15:117db924cf7c	4348
wolfSSL	15:117db924cf7c	4349	#define CMP_TAG() \
wolfSSL	15:117db924cf7c	4350	"cmpl $16, %[tbytes]\n\t" \
wolfSSL	15:117db924cf7c	4351	"je 71f\n\t" \
wolfSSL	15:117db924cf7c	4352	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4353	"xorq %%rcx, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	4354	"xorq %%rax, %%rax\n\t" \
wolfSSL	15:117db924cf7c	4355	"movdqu %%xmm0, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	4356	"\n" \
wolfSSL	15:117db924cf7c	4357	"73:\n\t" \
wolfSSL	15:117db924cf7c	4358	"movzbl (%%rsp,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	4359	"xorb (%[tag],%%rcx,1), %%r13b\n\t" \
wolfSSL	15:117db924cf7c	4360	"orb %%r13b, %%al\n\t" \
wolfSSL	15:117db924cf7c	4361	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4362	"cmpl %[tbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4363	"jne 73b\n\t" \
wolfSSL	15:117db924cf7c	4364	"cmpb $0x00, %%al\n\t" \
wolfSSL	15:117db924cf7c	4365	"sete %%al\n\t" \
wolfSSL	15:117db924cf7c	4366	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	4367	"xorq %%rcx, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	4368	"jmp 72f\n\t" \
wolfSSL	15:117db924cf7c	4369	"\n" \
wolfSSL	15:117db924cf7c	4370	"71:\n\t" \
wolfSSL	15:117db924cf7c	4371	"movdqu (%[tag]), %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4372	"pcmpeqb %%xmm1, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4373	"pmovmskb %%xmm0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4374	"# %%edx == 0xFFFF then return 1 else => return 0\n\t" \
wolfSSL	15:117db924cf7c	4375	"xorl %%eax, %%eax\n\t" \
wolfSSL	15:117db924cf7c	4376	"cmpl $0xffff, %%edx\n\t" \
wolfSSL	15:117db924cf7c	4377	"sete %%al\n\t" \
wolfSSL	15:117db924cf7c	4378	"\n" \
wolfSSL	15:117db924cf7c	4379	"72:\n\t" \
wolfSSL	15:117db924cf7c	4380	"movl %%eax, (%[res])\n\t"
wolfSSL	15:117db924cf7c	4381
wolfSSL	15:117db924cf7c	4382	static void AES_GCM_encrypt(const unsigned char in, unsigned char out,
wolfSSL	15:117db924cf7c	4383	const unsigned char* addt,
wolfSSL	15:117db924cf7c	4384	const unsigned char* ivec, unsigned char *tag,
wolfSSL	15:117db924cf7c	4385	unsigned int nbytes, unsigned int abytes,
wolfSSL	15:117db924cf7c	4386	unsigned int ibytes, unsigned int tbytes,
wolfSSL	15:117db924cf7c	4387	const unsigned char* key, int nr)
wolfSSL	15:117db924cf7c	4388	{
wolfSSL	15:117db924cf7c	4389	register const unsigned char* iv asm("rax") = ivec;
wolfSSL	15:117db924cf7c	4390	register unsigned int ivLen asm("ebx") = ibytes;
wolfSSL	15:117db924cf7c	4391
wolfSSL	15:117db924cf7c	4392	__asm__ __volatile__ (
wolfSSL	15:117db924cf7c	4393	"subq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	4394	/* Counter is xmm13 */
wolfSSL	15:117db924cf7c	4395	"pxor %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	4396	"pxor " VAR(XR) ", " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	4397	"movl %[ibytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	4398	"cmpl $12, %%edx\n\t"
wolfSSL	15:117db924cf7c	4399	"jne 35f\n\t"
wolfSSL	15:117db924cf7c	4400	CALC_IV_12()
wolfSSL	15:117db924cf7c	4401	"\n"
wolfSSL	15:117db924cf7c	4402	"35:\n\t"
wolfSSL	15:117db924cf7c	4403	CALC_IV()
wolfSSL	15:117db924cf7c	4404	"\n"
wolfSSL	15:117db924cf7c	4405	"39:\n\t"
wolfSSL	15:117db924cf7c	4406
wolfSSL	15:117db924cf7c	4407	CALC_AAD()
wolfSSL	15:117db924cf7c	4408
wolfSSL	15:117db924cf7c	4409	"# Calculate counter and H\n\t"
wolfSSL	15:117db924cf7c	4410	"pshufb %[BSWAP_EPI64], %%xmm13\n\t"
wolfSSL	15:117db924cf7c	4411	"movdqa " VAR(HR) ", %%xmm5\n\t"
wolfSSL	15:117db924cf7c	4412	"paddd %[ONE], %%xmm13\n\t"
wolfSSL	15:117db924cf7c	4413	"movdqa " VAR(HR) ", %%xmm4\n\t"
wolfSSL	15:117db924cf7c	4414	"movdqu %%xmm13, " VAR(CTR1) "\n\t"
wolfSSL	15:117db924cf7c	4415	"psrlq $63, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	4416	"psllq $1, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	4417	"pslldq $8, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	4418	"por %%xmm5, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	4419	"pshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	4420	"psrad $31, " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	4421	"pand %[MOD2_128], " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	4422	"pxor %%xmm4, " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	4423
wolfSSL	15:117db924cf7c	4424	"xorl " VAR(KR) ", " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4425
wolfSSL	15:117db924cf7c	4426	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	4427	"cmpl $128, %[nbytes]\n\t"
wolfSSL	15:117db924cf7c	4428	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	4429	"jl 5f\n\t"
wolfSSL	15:117db924cf7c	4430	"andl $0xffffff80, %%r13d\n\t"
wolfSSL	15:117db924cf7c	4431
wolfSSL	15:117db924cf7c	4432	CALC_HT_8_AVX()
wolfSSL	15:117db924cf7c	4433
wolfSSL	15:117db924cf7c	4434	"# First 128 bytes of input\n\t"
wolfSSL	15:117db924cf7c	4435	AESENC_CTR()
wolfSSL	15:117db924cf7c	4436	AESENC_XOR()
wolfSSL	15:117db924cf7c	4437	AESENC_SET(16)
wolfSSL	15:117db924cf7c	4438	AESENC_SET(32)
wolfSSL	15:117db924cf7c	4439	AESENC_SET(48)
wolfSSL	15:117db924cf7c	4440	AESENC_SET(64)
wolfSSL	15:117db924cf7c	4441	AESENC_SET(80)
wolfSSL	15:117db924cf7c	4442	AESENC_SET(96)
wolfSSL	15:117db924cf7c	4443	AESENC_SET(112)
wolfSSL	15:117db924cf7c	4444	AESENC_SET(128)
wolfSSL	15:117db924cf7c	4445	AESENC_SET(144)
wolfSSL	15:117db924cf7c	4446	"cmpl $11, %[nr]\n\t"
wolfSSL	15:117db924cf7c	4447	"movdqa 160(%[KEY]), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4448	"jl 1f\n\t"
wolfSSL	15:117db924cf7c	4449	AESENC()
wolfSSL	15:117db924cf7c	4450	AESENC_SET(176)
wolfSSL	15:117db924cf7c	4451	"cmpl $13, %[nr]\n\t"
wolfSSL	15:117db924cf7c	4452	"movdqa 192(%[KEY]), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4453	"jl 1f\n\t"
wolfSSL	15:117db924cf7c	4454	AESENC()
wolfSSL	15:117db924cf7c	4455	AESENC_SET(208)
wolfSSL	15:117db924cf7c	4456	"movdqa 224(%[KEY]), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4457	"\n"
wolfSSL	15:117db924cf7c	4458	"1:\n\t"
wolfSSL	15:117db924cf7c	4459	AESENC_LAST(%[in], %[out])
wolfSSL	15:117db924cf7c	4460
wolfSSL	15:117db924cf7c	4461	"cmpl $128, %%r13d\n\t"
wolfSSL	15:117db924cf7c	4462	"movl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4463	"jle 2f\n\t"
wolfSSL	15:117db924cf7c	4464
wolfSSL	15:117db924cf7c	4465	"# More 128 bytes of input\n\t"
wolfSSL	15:117db924cf7c	4466	"\n"
wolfSSL	15:117db924cf7c	4467	"3:\n\t"
wolfSSL	15:117db924cf7c	4468	AESENC_128_GHASH_AVX(%%rdx, 0)
wolfSSL	15:117db924cf7c	4469	"addl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4470	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4471	"jl 3b\n\t"
wolfSSL	15:117db924cf7c	4472	"\n"
wolfSSL	15:117db924cf7c	4473	"2:\n\t"
wolfSSL	15:117db924cf7c	4474	"movdqa %[BSWAP_MASK], %%xmm13\n\t"
wolfSSL	15:117db924cf7c	4475	"pshufb %%xmm13, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	4476	"pshufb %%xmm13, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	4477	"pshufb %%xmm13, %%xmm6\n\t"
wolfSSL	15:117db924cf7c	4478	"pshufb %%xmm13, %%xmm7\n\t"
wolfSSL	15:117db924cf7c	4479	"pxor %%xmm2, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	4480	"pshufb %%xmm13, %%xmm8\n\t"
wolfSSL	15:117db924cf7c	4481	"pshufb %%xmm13, %%xmm9\n\t"
wolfSSL	15:117db924cf7c	4482	"pshufb %%xmm13, %%xmm10\n\t"
wolfSSL	15:117db924cf7c	4483	"pshufb %%xmm13, %%xmm11\n\t"
wolfSSL	15:117db924cf7c	4484
wolfSSL	15:117db924cf7c	4485	"movdqu 112(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4486	GHASH_GFMUL_AVX(XR, %%xmm13, %%xmm4, %%xmm12)
wolfSSL	15:117db924cf7c	4487	"movdqu 96(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4488	GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm5, %%xmm12)
wolfSSL	15:117db924cf7c	4489	"movdqu 80(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4490	GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm6, %%xmm12)
wolfSSL	15:117db924cf7c	4491	"movdqu 64(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4492	GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm7, %%xmm12)
wolfSSL	15:117db924cf7c	4493	"movdqu 48(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4494	GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm8, %%xmm12)
wolfSSL	15:117db924cf7c	4495	"movdqu 32(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4496	GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm9, %%xmm12)
wolfSSL	15:117db924cf7c	4497	"movdqu 16(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4498	GHASH_GFMUL_XOR_AVX(XR, %%xmm13, %%xmm10, %%xmm12)
wolfSSL	15:117db924cf7c	4499	"movdqu (" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	4500	GHASH_GFMUL_RED_XOR_AVX(XR, %%xmm13, %%xmm11, %%xmm12)
wolfSSL	15:117db924cf7c	4501
wolfSSL	15:117db924cf7c	4502	"movdqu 0(" VAR(HTR) "), " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	4503	"\n"
wolfSSL	15:117db924cf7c	4504	"5:\n\t"
wolfSSL	15:117db924cf7c	4505	"movl %[nbytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	4506	"cmpl %%edx, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4507	"jge 55f\n\t"
wolfSSL	15:117db924cf7c	4508	#endif
wolfSSL	15:117db924cf7c	4509
wolfSSL	15:117db924cf7c	4510	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	4511	"andl $0xfffffff0, %%r13d\n\t"
wolfSSL	15:117db924cf7c	4512	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4513	"jge 14f\n\t"
wolfSSL	15:117db924cf7c	4514
wolfSSL	15:117db924cf7c	4515	"leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t"
wolfSSL	15:117db924cf7c	4516	"leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t"
wolfSSL	15:117db924cf7c	4517	AESENC_BLOCK(%%rcx, %%rdx)
wolfSSL	15:117db924cf7c	4518	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4519	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4520	"jge 13f\n\t"
wolfSSL	15:117db924cf7c	4521	"\n"
wolfSSL	15:117db924cf7c	4522	"12:\n\t"
wolfSSL	15:117db924cf7c	4523	"leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t"
wolfSSL	15:117db924cf7c	4524	"leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t"
wolfSSL	15:117db924cf7c	4525	AESENC_GFMUL(%%rcx, %%rdx, HR, XR)
wolfSSL	15:117db924cf7c	4526	"pshufb %[BSWAP_MASK], %%xmm4\n\t"
wolfSSL	15:117db924cf7c	4527	"pxor %%xmm4, " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	4528	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4529	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	4530	"jl 12b\n\t"
wolfSSL	15:117db924cf7c	4531	"\n"
wolfSSL	15:117db924cf7c	4532	"13:\n\t"
wolfSSL	15:117db924cf7c	4533	GHASH_GFMUL_RED_AVX(XR, HR, XR)
wolfSSL	15:117db924cf7c	4534	"\n"
wolfSSL	15:117db924cf7c	4535	"14:\n\t"
wolfSSL	15:117db924cf7c	4536
wolfSSL	15:117db924cf7c	4537	AESENC_LAST15_ENC_AVX()
wolfSSL	15:117db924cf7c	4538	"\n"
wolfSSL	15:117db924cf7c	4539	"55:\n\t"
wolfSSL	15:117db924cf7c	4540
wolfSSL	15:117db924cf7c	4541	CALC_TAG()
wolfSSL	15:117db924cf7c	4542	STORE_TAG()
wolfSSL	15:117db924cf7c	4543	"addq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	4544
wolfSSL	15:117db924cf7c	4545	:
wolfSSL	15:117db924cf7c	4546	: [KEY] "r" (key),
wolfSSL	15:117db924cf7c	4547	[in] "r" (in), [out] "r" (out), [nr] "r" (nr),
wolfSSL	15:117db924cf7c	4548	[nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt),
wolfSSL	15:117db924cf7c	4549	[ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tbytes),
wolfSSL	15:117db924cf7c	4550	[tag] "r" (tag),
wolfSSL	15:117db924cf7c	4551	[BSWAP_MASK] "m" (BSWAP_MASK),
wolfSSL	15:117db924cf7c	4552	[BSWAP_EPI64] "m" (BSWAP_EPI64),
wolfSSL	15:117db924cf7c	4553	[ONE] "m" (ONE),
wolfSSL	15:117db924cf7c	4554	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	4555	[TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR),
wolfSSL	15:117db924cf7c	4556	[FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN),
wolfSSL	15:117db924cf7c	4557	[EIGHT] "m" (EIGHT),
wolfSSL	15:117db924cf7c	4558	#endif
wolfSSL	15:117db924cf7c	4559	[MOD2_128] "m" (MOD2_128)
wolfSSL	15:117db924cf7c	4560	: "xmm15", "xmm14", "xmm13", "xmm12",
wolfSSL	15:117db924cf7c	4561	"xmm0", "xmm1", "xmm2", "xmm3", "memory",
wolfSSL	15:117db924cf7c	4562	"xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
wolfSSL	15:117db924cf7c	4563	"rcx", "rdx", "r13"
wolfSSL	15:117db924cf7c	4564	);
wolfSSL	15:117db924cf7c	4565	}
wolfSSL	15:117db924cf7c	4566
wolfSSL	15:117db924cf7c	4567	#ifdef HAVE_INTEL_AVX1
wolfSSL	15:117db924cf7c	4568	/* Encrypt with key in xmm12. */
wolfSSL	15:117db924cf7c	4569	#define VAESENC() \
wolfSSL	15:117db924cf7c	4570	"vaesenc %%xmm12, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4571	"vaesenc %%xmm12, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4572	"vaesenc %%xmm12, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4573	"vaesenc %%xmm12, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4574	"vaesenc %%xmm12, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4575	"vaesenc %%xmm12, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4576	"vaesenc %%xmm12, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4577	"vaesenc %%xmm12, %%xmm11, %%xmm11\n\t"
wolfSSL	15:117db924cf7c	4578
wolfSSL	15:117db924cf7c	4579	#define VAESENC_SET(o) \
wolfSSL	15:117db924cf7c	4580	"vmovdqa "#o"(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4581	VAESENC()
wolfSSL	15:117db924cf7c	4582
wolfSSL	15:117db924cf7c	4583	#define VAESENC_CTR() \
wolfSSL	15:117db924cf7c	4584	"vmovdqu " VAR(CTR1) ", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4585	"vmovdqa %[BSWAP_EPI64], %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4586	"vpshufb %%xmm1, %%xmm0, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4587	"vpaddd %[ONE], %%xmm0, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4588	"vpshufb %%xmm1, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4589	"vpaddd %[TWO], %%xmm0, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4590	"vpshufb %%xmm1, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4591	"vpaddd %[THREE], %%xmm0, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4592	"vpshufb %%xmm1, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4593	"vpaddd %[FOUR], %%xmm0, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4594	"vpshufb %%xmm1, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4595	"vpaddd %[FIVE], %%xmm0, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4596	"vpshufb %%xmm1, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4597	"vpaddd %[SIX], %%xmm0, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4598	"vpshufb %%xmm1, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4599	"vpaddd %[SEVEN], %%xmm0, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	4600	"vpshufb %%xmm1, %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	4601	"vpaddd %[EIGHT], %%xmm0, %%xmm0\n\t"
wolfSSL	15:117db924cf7c	4602
wolfSSL	15:117db924cf7c	4603	#define VAESENC_XOR() \
wolfSSL	15:117db924cf7c	4604	"vmovdqa (%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4605	"vmovdqu %%xmm0, " VAR(CTR1) "\n\t" \
wolfSSL	15:117db924cf7c	4606	"vpxor %%xmm12, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4607	"vpxor %%xmm12, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4608	"vpxor %%xmm12, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4609	"vpxor %%xmm12, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4610	"vpxor %%xmm12, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4611	"vpxor %%xmm12, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4612	"vpxor %%xmm12, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4613	"vpxor %%xmm12, %%xmm11, %%xmm11\n\t"
wolfSSL	15:117db924cf7c	4614
wolfSSL	15:117db924cf7c	4615	#define VAESENC_128() \
wolfSSL	15:117db924cf7c	4616	VAESENC_CTR() \
wolfSSL	15:117db924cf7c	4617	VAESENC_XOR() \
wolfSSL	15:117db924cf7c	4618	VAESENC_SET(16) \
wolfSSL	15:117db924cf7c	4619	VAESENC_SET(32) \
wolfSSL	15:117db924cf7c	4620	VAESENC_SET(48) \
wolfSSL	15:117db924cf7c	4621	VAESENC_SET(64) \
wolfSSL	15:117db924cf7c	4622	VAESENC_SET(80) \
wolfSSL	15:117db924cf7c	4623	VAESENC_SET(96) \
wolfSSL	15:117db924cf7c	4624	VAESENC_SET(112) \
wolfSSL	15:117db924cf7c	4625	VAESENC_SET(128) \
wolfSSL	15:117db924cf7c	4626	VAESENC_SET(144) \
wolfSSL	15:117db924cf7c	4627	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4628	"vmovdqa 160(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4629	"jl 1f\n\t" \
wolfSSL	15:117db924cf7c	4630	VAESENC() \
wolfSSL	15:117db924cf7c	4631	VAESENC_SET(176) \
wolfSSL	15:117db924cf7c	4632	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4633	"vmovdqa 192(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4634	"jl 1f\n\t" \
wolfSSL	15:117db924cf7c	4635	VAESENC() \
wolfSSL	15:117db924cf7c	4636	VAESENC_SET(208) \
wolfSSL	15:117db924cf7c	4637	"vmovdqa 224(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4638	"\n" \
wolfSSL	15:117db924cf7c	4639	"1:\n\t" \
wolfSSL	15:117db924cf7c	4640	VAESENC_LAST(%[in], %[out])
wolfSSL	15:117db924cf7c	4641
wolfSSL	15:117db924cf7c	4642	/* Encrypt and carry-less multiply for AVX1. */
wolfSSL	15:117db924cf7c	4643	#define VAESENC_PCLMUL_1(src, o1, o2, o3) \
wolfSSL	15:117db924cf7c	4644	"vmovdqu " #o3 "(" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4645	"vmovdqu " #o2 "(" #src "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4646	"vaesenc " #o1 "(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4647	"vpshufb %[BSWAP_MASK], %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4648	"vpxor %%xmm2, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4649	"vpshufd $0x4e, %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4650	"vpshufd $0x4e, %%xmm0, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4651	"vpxor %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4652	"vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4653	"vpclmulqdq $0x11, %%xmm12, %%xmm0, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4654	"vaesenc " #o1 "(%[KEY]), %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4655	"vaesenc " #o1 "(%[KEY]), %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4656	"vpclmulqdq $0x00, %%xmm12, %%xmm0, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4657	"vaesenc " #o1 "(%[KEY]), %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4658	"vaesenc " #o1 "(%[KEY]), %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4659	"vpclmulqdq $0x00, %%xmm14, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4660	"vaesenc " #o1 "(%[KEY]), %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4661	"vaesenc " #o1 "(%[KEY]), %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4662	"vaesenc " #o1 "(%[KEY]), %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	4663	"vpxor %%xmm2, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4664	"vpxor %%xmm3, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4665
wolfSSL	15:117db924cf7c	4666	#define VAESENC_PCLMUL_N(src, o1, o2, o3) \
wolfSSL	15:117db924cf7c	4667	"vmovdqu " #o3 "(" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4668	"vmovdqu " #o2 "(" #src "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4669	"vpshufd $0x4e, %%xmm12, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4670	"vpshufb %[BSWAP_MASK], %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4671	"vaesenc " #o1 "(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4672	"vpxor %%xmm12, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4673	"vpshufd $0x4e, %%xmm0, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4674	"vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4675	"vpclmulqdq $0x11, %%xmm12, %%xmm0, %%xmm15\n\t" \
wolfSSL	15:117db924cf7c	4676	"vaesenc " #o1 "(%[KEY]), %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4677	"vaesenc " #o1 "(%[KEY]), %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4678	"vpclmulqdq $0x00, %%xmm12, %%xmm0, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4679	"vaesenc " #o1 "(%[KEY]), %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4680	"vaesenc " #o1 "(%[KEY]), %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4681	"vpclmulqdq $0x00, %%xmm14, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4682	"vaesenc " #o1 "(%[KEY]), %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4683	"vaesenc " #o1 "(%[KEY]), %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4684	"vaesenc " #o1 "(%[KEY]), %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	4685	"vpxor %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4686	"vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4687	"vpxor %%xmm15, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4688	"vpxor %%xmm15, %%xmm3, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4689	"vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4690
wolfSSL	15:117db924cf7c	4691	#define VAESENC_PCLMUL_L(o) \
wolfSSL	15:117db924cf7c	4692	"vpslldq $8, %%xmm1, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4693	"vpsrldq $8, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4694	"vaesenc "#o"(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4695	"vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4696	"vpxor %%xmm1, %%xmm3, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4697	"vaesenc "#o"(%[KEY]), %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4698	"vpslld $31, %%xmm2, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4699	"vpslld $30, %%xmm2, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4700	"vpslld $25, %%xmm2, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4701	"vaesenc "#o"(%[KEY]), %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4702	"vpxor %%xmm13, %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4703	"vpxor %%xmm14, %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4704	"vaesenc "#o"(%[KEY]), %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4705	"vpsrldq $4, %%xmm12, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4706	"vpslldq $12, %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4707	"vaesenc "#o"(%[KEY]), %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4708	"vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4709	"vpsrld $1, %%xmm2, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4710	"vaesenc "#o"(%[KEY]), %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4711	"vpsrld $2, %%xmm2, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4712	"vpsrld $7, %%xmm2, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4713	"vaesenc "#o"(%[KEY]), %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4714	"vpxor %%xmm1, %%xmm14, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4715	"vpxor %%xmm0, %%xmm14, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4716	"vaesenc "#o"(%[KEY]), %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	4717	"vpxor %%xmm13, %%xmm14, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	4718	"vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4719	"vpxor %%xmm3, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4720
wolfSSL	15:117db924cf7c	4721
wolfSSL	15:117db924cf7c	4722	/* Encrypt and carry-less multiply with last key. */
wolfSSL	15:117db924cf7c	4723	#define VAESENC_LAST(in, out) \
wolfSSL	15:117db924cf7c	4724	"vaesenclast %%xmm12, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4725	"vaesenclast %%xmm12, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4726	"vmovdqu (" #in "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4727	"vmovdqu 16(" #in "), %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4728	"vpxor %%xmm0, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4729	"vpxor %%xmm1, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4730	"vmovdqu %%xmm4, (" #out ")\n\t" \
wolfSSL	15:117db924cf7c	4731	"vmovdqu %%xmm5, 16(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	4732	"vaesenclast %%xmm12, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4733	"vaesenclast %%xmm12, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4734	"vmovdqu 32(" #in "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4735	"vmovdqu 48(" #in "), %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4736	"vpxor %%xmm0, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4737	"vpxor %%xmm1, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4738	"vmovdqu %%xmm6, 32(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	4739	"vmovdqu %%xmm7, 48(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	4740	"vaesenclast %%xmm12, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4741	"vaesenclast %%xmm12, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4742	"vmovdqu 64(" #in "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4743	"vmovdqu 80(" #in "), %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4744	"vpxor %%xmm0, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4745	"vpxor %%xmm1, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4746	"vmovdqu %%xmm8, 64(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	4747	"vmovdqu %%xmm9, 80(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	4748	"vaesenclast %%xmm12, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4749	"vaesenclast %%xmm12, %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	4750	"vmovdqu 96(" #in "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4751	"vmovdqu 112(" #in "), %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4752	"vpxor %%xmm0, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4753	"vpxor %%xmm1, %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	4754	"vmovdqu %%xmm10, 96(" #out ")\n\t" \
wolfSSL	15:117db924cf7c	4755	"vmovdqu %%xmm11, 112(" #out ")\n\t"
wolfSSL	15:117db924cf7c	4756
wolfSSL	15:117db924cf7c	4757	#define VAESENC_BLOCK() \
wolfSSL	15:117db924cf7c	4758	"vmovdqu " VAR(CTR1) ", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4759	"vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4760	"vpaddd %[ONE], %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4761	"vmovdqu %%xmm5, " VAR(CTR1) "\n\t" \
wolfSSL	15:117db924cf7c	4762	"vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4763	"vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4764	"vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4765	"vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4766	"vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4767	"vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4768	"vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4769	"vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4770	"vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4771	"vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4772	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4773	"vmovdqa 160(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4774	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	4775	"vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4776	"vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4777	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4778	"vmovdqa 192(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4779	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	4780	"vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4781	"vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4782	"vmovdqa 224(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4783	"%=:\n\t" \
wolfSSL	15:117db924cf7c	4784	"vaesenclast %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4785	"vmovdqu (%[in]," VAR(KR64) ",1), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4786	"vpxor %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4787	"vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t" \
wolfSSL	15:117db924cf7c	4788	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4789	"vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	4790
wolfSSL	15:117db924cf7c	4791	#define _VAESENC_GFMUL(in, H, X) \
wolfSSL	15:117db924cf7c	4792	"vmovdqu " VAR(CTR1) ", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4793	"vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4794	"vpaddd %[ONE], %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4795	"vmovdqu %%xmm5, " VAR(CTR1) "\n\t" \
wolfSSL	15:117db924cf7c	4796	"vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4797	"vpclmulqdq $0x10, " #H ", " #X ", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4798	"vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4799	"vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4800	"vpclmulqdq $0x01, " #H ", " #X ", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4801	"vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4802	"vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4803	"vpclmulqdq $0x00, " #H ", " #X ", %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4804	"vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4805	"vpclmulqdq $0x11, " #H ", " #X ", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4806	"vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4807	"vpxor %%xmm7, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4808	"vpslldq $8, %%xmm6, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4809	"vpsrldq $8, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4810	"vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4811	"vpxor %%xmm8, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4812	"vpxor %%xmm6, %%xmm1, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4813	"vmovdqa %[MOD2_128], %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4814	"vpclmulqdq $0x10, %%xmm0, %%xmm2, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4815	"vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4816	"vpshufd $0x4e, %%xmm2, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4817	"vpxor %%xmm7, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4818	"vpclmulqdq $0x10, %%xmm0, %%xmm6, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4819	"vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4820	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4821	"vpxor %%xmm7, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4822	"vpxor %%xmm3, %%xmm6, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	4823	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4824	"vmovdqa 160(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4825	"jl 1f\n\t" \
wolfSSL	15:117db924cf7c	4826	"vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4827	"vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4828	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	4829	"vmovdqa 192(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4830	"jl 1f\n\t" \
wolfSSL	15:117db924cf7c	4831	"vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4832	"vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4833	"vmovdqa 224(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4834	"1:\n\t" \
wolfSSL	15:117db924cf7c	4835	"vaesenclast %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4836	"vmovdqu " #in ", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4837	"vpxor %%xmm0, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4838	"vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t"
wolfSSL	15:117db924cf7c	4839	#define VAESENC_GFMUL(in, H, X) \
wolfSSL	15:117db924cf7c	4840	_VAESENC_GFMUL(in, H, X)
wolfSSL	15:117db924cf7c	4841
wolfSSL	15:117db924cf7c	4842
wolfSSL	15:117db924cf7c	4843	#define _GHASH_GFMUL_AVX1(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4844	"vpshufd $0x4e, "#a", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4845	"vpshufd $0x4e, "#b", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4846	"vpclmulqdq $0x11, "#a", "#b", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4847	"vpclmulqdq $0x00, "#a", "#b", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4848	"vpxor "#a", %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4849	"vpxor "#b", %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4850	"vpclmulqdq $0x00, %%xmm2, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4851	"vpxor %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4852	"vpxor %%xmm3, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4853	"vmovdqa %%xmm0, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	4854	"vmovdqa %%xmm3, " #r "\n\t" \
wolfSSL	15:117db924cf7c	4855	"vpslldq $8, %%xmm1, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4856	"vpsrldq $8, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4857	"vpxor %%xmm2, "#r2", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	4858	"vpxor %%xmm1, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	4859	#define GHASH_GFMUL_AVX1(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4860	_GHASH_GFMUL_AVX1(r, r2, a, b)
wolfSSL	15:117db924cf7c	4861
wolfSSL	15:117db924cf7c	4862	#define _GHASH_GFMUL_XOR_AVX1(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4863	"vpshufd $0x4e, "#a", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4864	"vpshufd $0x4e, "#b", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4865	"vpclmulqdq $0x11, "#a", "#b", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4866	"vpclmulqdq $0x00, "#a", "#b", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4867	"vpxor "#a", %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4868	"vpxor "#b", %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4869	"vpclmulqdq $0x00, %%xmm2, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4870	"vpxor %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4871	"vpxor %%xmm3, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4872	"vpxor %%xmm0, "#r2", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	4873	"vpxor %%xmm3, " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	4874	"vpslldq $8, %%xmm1, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4875	"vpsrldq $8, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4876	"vpxor %%xmm2, "#r2", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	4877	"vpxor %%xmm1, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	4878	#define GHASH_GFMUL_XOR_AVX1(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4879	_GHASH_GFMUL_XOR_AVX1(r, r2, a, b)
wolfSSL	15:117db924cf7c	4880
wolfSSL	15:117db924cf7c	4881	#define GHASH_MID_AVX1(r, r2) \
wolfSSL	15:117db924cf7c	4882	"vpsrld $31, "#r2", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4883	"vpsrld $31, " #r ", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4884	"vpslld $1, "#r2", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	4885	"vpslld $1, " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	4886	"vpsrldq $12, %%xmm0, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4887	"vpslldq $4, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4888	"vpslldq $4, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4889	"vpor %%xmm2, " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	4890	"vpor %%xmm0, "#r2", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	4891	"vpor %%xmm1, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	4892
wolfSSL	15:117db924cf7c	4893	#define _GHASH_GFMUL_RED_AVX1(r, a, b) \
wolfSSL	15:117db924cf7c	4894	"vpshufd $0x4e, "#a", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4895	"vpshufd $0x4e, "#b", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4896	"vpclmulqdq $0x11, "#a", "#b", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	4897	"vpclmulqdq $0x00, "#a", "#b", %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4898	"vpxor "#a", %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4899	"vpxor "#b", %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4900	"vpclmulqdq $0x00, %%xmm6, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4901	"vpxor %%xmm4, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4902	"vpxor %%xmm7, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4903	"vpslldq $8, %%xmm5, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4904	"vpsrldq $8, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4905	"vpxor %%xmm6, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4906	"vpxor %%xmm5, %%xmm7, " #r "\n\t" \
wolfSSL	15:117db924cf7c	4907	"vpslld $31, %%xmm4, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4908	"vpslld $30, %%xmm4, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4909	"vpslld $25, %%xmm4, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4910	"vpxor %%xmm9, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4911	"vpxor %%xmm10, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4912	"vpsrldq $4, %%xmm8, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4913	"vpslldq $12, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4914	"vpxor %%xmm8, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4915	"vpsrld $1, %%xmm4, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4916	"vpsrld $2, %%xmm4, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4917	"vpsrld $7, %%xmm4, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4918	"vpxor %%xmm6, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4919	"vpxor %%xmm5, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4920	"vpxor %%xmm9, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4921	"vpxor %%xmm4, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4922	"vpxor %%xmm10, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	4923	#define GHASH_GFMUL_RED_AVX1(r, a, b) \
wolfSSL	15:117db924cf7c	4924	_GHASH_GFMUL_RED_AVX1(r, a, b)
wolfSSL	15:117db924cf7c	4925
wolfSSL	15:117db924cf7c	4926	#define _GHASH_GFSQR_RED_AVX1(r, a) \
wolfSSL	15:117db924cf7c	4927	"vpclmulqdq $0x00, "#a", "#a", %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4928	"vpclmulqdq $0x11, "#a", "#a", " #r "\n\t" \
wolfSSL	15:117db924cf7c	4929	"vpslld $31, %%xmm4, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4930	"vpslld $30, %%xmm4, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4931	"vpslld $25, %%xmm4, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4932	"vpxor %%xmm9, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4933	"vpxor %%xmm10, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4934	"vpsrldq $4, %%xmm8, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	4935	"vpslldq $12, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	4936	"vpxor %%xmm8, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	4937	"vpsrld $1, %%xmm4, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4938	"vpsrld $2, %%xmm4, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	4939	"vpsrld $7, %%xmm4, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	4940	"vpxor %%xmm6, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4941	"vpxor %%xmm5, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4942	"vpxor %%xmm9, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4943	"vpxor %%xmm4, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	4944	"vpxor %%xmm10, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	4945	#define GHASH_GFSQR_RED_AVX1(r, a) \
wolfSSL	15:117db924cf7c	4946	_GHASH_GFSQR_RED_AVX1(r, a)
wolfSSL	15:117db924cf7c	4947
wolfSSL	15:117db924cf7c	4948	#define GHASH_RED_AVX1(r, r2) \
wolfSSL	15:117db924cf7c	4949	"vpslld $31, "#r2", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4950	"vpslld $30, "#r2", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4951	"vpslld $25, "#r2", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4952	"vpxor %%xmm1, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4953	"vpxor %%xmm2, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4954	"vmovdqa %%xmm0, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4955	"vpsrldq $4, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4956	"vpslldq $12, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4957	"vpxor %%xmm0, "#r2", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	4958	"vpsrld $1, "#r2", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4959	"vpsrld $2, "#r2", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	4960	"vpsrld $7, "#r2", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	4961	"vpxor %%xmm3, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4962	"vpxor %%xmm0, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4963	"vpxor %%xmm1, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4964	"vpxor "#r2", %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	4965	"vpxor %%xmm2, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	4966
wolfSSL	15:117db924cf7c	4967	#define GHASH_GFMUL_RED_XOR_AVX1(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4968	GHASH_GFMUL_XOR_AVX1(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4969	GHASH_RED_AVX1(r, r2)
wolfSSL	15:117db924cf7c	4970
wolfSSL	15:117db924cf7c	4971	#define GHASH_FULL_AVX1(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4972	GHASH_GFMUL_AVX1(r, r2, a, b) \
wolfSSL	15:117db924cf7c	4973	GHASH_MID_AVX1(r, r2) \
wolfSSL	15:117db924cf7c	4974	GHASH_RED_AVX1(r, r2)
wolfSSL	15:117db924cf7c	4975
wolfSSL	15:117db924cf7c	4976	#define CALC_IV_12_AVX1() \
wolfSSL	15:117db924cf7c	4977	"# Calculate values when IV is 12 bytes\n\t" \
wolfSSL	15:117db924cf7c	4978	"# Set counter based on IV\n\t" \
wolfSSL	15:117db924cf7c	4979	"movl $0x01000000, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	4980	"vpinsrq $0, 0(%%rax), %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4981	"vpinsrd $2, 8(%%rax), %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4982	"vpinsrd $3, %%ecx, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	4983	"# H = Encrypt X(=0) and T = Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	4984	"vmovdqa 0(%[KEY]), " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4985	"vpxor " VAR(HR) ", %%xmm13, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4986	"vmovdqa 16(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4987	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4988	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4989	"vmovdqa 32(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4990	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4991	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4992	"vmovdqa 48(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4993	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4994	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4995	"vmovdqa 64(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4996	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	4997	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	4998	"vmovdqa 80(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	4999	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5000	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5001	"vmovdqa 96(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5002	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5003	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5004	"vmovdqa 112(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5005	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5006	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5007	"vmovdqa 128(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5008	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5009	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5010	"vmovdqa 144(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5011	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5012	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5013	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5014	"vmovdqa 160(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5015	"jl 31f\n\t" \
wolfSSL	15:117db924cf7c	5016	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5017	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5018	"vmovdqa 176(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5019	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5020	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5021	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5022	"vmovdqa 192(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5023	"jl 31f\n\t" \
wolfSSL	15:117db924cf7c	5024	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5025	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5026	"vmovdqa 208(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5027	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5028	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5029	"vmovdqu 224(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5030	"31:\n\t" \
wolfSSL	15:117db924cf7c	5031	"vaesenclast %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5032	"vaesenclast %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5033	"vpshufb %[BSWAP_MASK], " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5034	"vmovdqu %%xmm1, " VAR(TR) "\n\t" \
wolfSSL	15:117db924cf7c	5035	"jmp 39f\n\t"
wolfSSL	15:117db924cf7c	5036
wolfSSL	15:117db924cf7c	5037	#define CALC_IV_AVX1() \
wolfSSL	15:117db924cf7c	5038	"# Calculate values when IV is not 12 bytes\n\t" \
wolfSSL	15:117db924cf7c	5039	"# H = Encrypt X(=0)\n\t" \
wolfSSL	15:117db924cf7c	5040	"vmovdqa 0(%[KEY]), " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5041	VAESENC_AVX(HR) \
wolfSSL	15:117db924cf7c	5042	"vpshufb %[BSWAP_MASK], " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5043	"# Calc counter\n\t" \
wolfSSL	15:117db924cf7c	5044	"# Initialization vector\n\t" \
wolfSSL	15:117db924cf7c	5045	"cmpl $0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5046	"movq $0, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	5047	"je 45f\n\t" \
wolfSSL	15:117db924cf7c	5048	"cmpl $16, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5049	"jl 44f\n\t" \
wolfSSL	15:117db924cf7c	5050	"andl $0xfffffff0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5051	"\n" \
wolfSSL	15:117db924cf7c	5052	"43:\n\t" \
wolfSSL	15:117db924cf7c	5053	"vmovdqu (%%rax,%%rcx,1), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5054	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5055	"vpxor %%xmm4, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5056	GHASH_FULL_AVX1(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	5057	"addl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5058	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5059	"jl 43b\n\t" \
wolfSSL	15:117db924cf7c	5060	"movl %[ibytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	5061	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5062	"je 45f\n\t" \
wolfSSL	15:117db924cf7c	5063	"\n" \
wolfSSL	15:117db924cf7c	5064	"44:\n\t" \
wolfSSL	15:117db924cf7c	5065	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5066	"vpxor %%xmm4, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5067	"xorl %%ebx, %%ebx\n\t" \
wolfSSL	15:117db924cf7c	5068	"vmovdqu %%xmm4, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	5069	"42:\n\t" \
wolfSSL	15:117db924cf7c	5070	"movzbl (%%rax,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	5071	"movb %%r13b, (%%rsp,%%rbx,1)\n\t" \
wolfSSL	15:117db924cf7c	5072	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5073	"incl %%ebx\n\t" \
wolfSSL	15:117db924cf7c	5074	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5075	"jl 42b\n\t" \
wolfSSL	15:117db924cf7c	5076	"vmovdqu (%%rsp), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5077	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5078	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5079	"vpxor %%xmm4, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5080	GHASH_FULL_AVX1(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	5081	"\n" \
wolfSSL	15:117db924cf7c	5082	"45:\n\t" \
wolfSSL	15:117db924cf7c	5083	"# T = Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	5084	"vpxor %%xmm0, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5085	"shll $3, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5086	"vpinsrq $0, %%rdx, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5087	"vpxor %%xmm0, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5088	GHASH_FULL_AVX1(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	5089	"vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5090	"# Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	5091	"vmovdqa 0(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5092	"vpxor %%xmm13, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5093	VAESENC_AVX(%%xmm4) \
wolfSSL	15:117db924cf7c	5094	"vmovdqu %%xmm4, " VAR(TR) "\n\t"
wolfSSL	15:117db924cf7c	5095
wolfSSL	15:117db924cf7c	5096	#define CALC_AAD_AVX1() \
wolfSSL	15:117db924cf7c	5097	"# Additional authentication data\n\t" \
wolfSSL	15:117db924cf7c	5098	"movl %[abytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	5099	"cmpl $0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5100	"je 25f\n\t" \
wolfSSL	15:117db924cf7c	5101	"movq %[addt], %%rax\n\t" \
wolfSSL	15:117db924cf7c	5102	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5103	"cmpl $16, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5104	"jl 24f\n\t" \
wolfSSL	15:117db924cf7c	5105	"andl $0xfffffff0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5106	"\n" \
wolfSSL	15:117db924cf7c	5107	"23:\n\t" \
wolfSSL	15:117db924cf7c	5108	"vmovdqu (%%rax,%%rcx,1), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5109	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5110	"vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	5111	GHASH_FULL_AVX1(XR, %%xmm12, XR, HR) \
wolfSSL	15:117db924cf7c	5112	"addl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5113	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5114	"jl 23b\n\t" \
wolfSSL	15:117db924cf7c	5115	"movl %[abytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	5116	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5117	"je 25f\n\t" \
wolfSSL	15:117db924cf7c	5118	"\n" \
wolfSSL	15:117db924cf7c	5119	"24:\n\t" \
wolfSSL	15:117db924cf7c	5120	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5121	"vpxor %%xmm4, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5122	"xorl %%ebx, %%ebx\n\t" \
wolfSSL	15:117db924cf7c	5123	"vmovdqu %%xmm4, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	5124	"22:\n\t" \
wolfSSL	15:117db924cf7c	5125	"movzbl (%%rax,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	5126	"movb %%r13b, (%%rsp,%%rbx,1)\n\t" \
wolfSSL	15:117db924cf7c	5127	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5128	"incl %%ebx\n\t" \
wolfSSL	15:117db924cf7c	5129	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5130	"jl 22b\n\t" \
wolfSSL	15:117db924cf7c	5131	"vmovdqu (%%rsp), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5132	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5133	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5134	"vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	5135	GHASH_FULL_AVX1(XR, %%xmm12, XR, HR) \
wolfSSL	15:117db924cf7c	5136	"\n" \
wolfSSL	15:117db924cf7c	5137	"25:\n\t"
wolfSSL	15:117db924cf7c	5138
wolfSSL	15:117db924cf7c	5139	#define CALC_HT_8_AVX1() \
wolfSSL	15:117db924cf7c	5140	"vmovdqa " VAR(XR) ", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5141	"# H ^ 1\n\t" \
wolfSSL	15:117db924cf7c	5142	"vmovdqu " VAR(HR) ", 0(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5143	"# H ^ 2\n\t" \
wolfSSL	15:117db924cf7c	5144	GHASH_GFSQR_RED_AVX1(%%xmm0, HR) \
wolfSSL	15:117db924cf7c	5145	"vmovdqu %%xmm0 , 16(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5146	"# H ^ 3\n\t" \
wolfSSL	15:117db924cf7c	5147	GHASH_GFMUL_RED_AVX1(%%xmm1, HR, %%xmm0) \
wolfSSL	15:117db924cf7c	5148	"vmovdqu %%xmm1 , 32(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5149	"# H ^ 4\n\t" \
wolfSSL	15:117db924cf7c	5150	GHASH_GFSQR_RED_AVX1(%%xmm3, %%xmm0) \
wolfSSL	15:117db924cf7c	5151	"vmovdqu %%xmm3 , 48(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5152	"# H ^ 5\n\t" \
wolfSSL	15:117db924cf7c	5153	GHASH_GFMUL_RED_AVX1(%%xmm12, %%xmm0, %%xmm1) \
wolfSSL	15:117db924cf7c	5154	"vmovdqu %%xmm12, 64(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5155	"# H ^ 6\n\t" \
wolfSSL	15:117db924cf7c	5156	GHASH_GFSQR_RED_AVX1(%%xmm12, %%xmm1) \
wolfSSL	15:117db924cf7c	5157	"vmovdqu %%xmm12, 80(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5158	"# H ^ 7\n\t" \
wolfSSL	15:117db924cf7c	5159	GHASH_GFMUL_RED_AVX1(%%xmm12, %%xmm1, %%xmm3) \
wolfSSL	15:117db924cf7c	5160	"vmovdqu %%xmm12, 96(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5161	"# H ^ 8\n\t" \
wolfSSL	15:117db924cf7c	5162	GHASH_GFSQR_RED_AVX1(%%xmm12, %%xmm3) \
wolfSSL	15:117db924cf7c	5163	"vmovdqu %%xmm12, 112(" VAR(HTR) ")\n\t"
wolfSSL	15:117db924cf7c	5164
wolfSSL	15:117db924cf7c	5165	#define VAESENC_128_GHASH_AVX1(src, o) \
wolfSSL	15:117db924cf7c	5166	"leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" \
wolfSSL	15:117db924cf7c	5167	"leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" \
wolfSSL	15:117db924cf7c	5168	/* src is either %%rcx or %%rdx */ \
wolfSSL	15:117db924cf7c	5169	VAESENC_CTR() \
wolfSSL	15:117db924cf7c	5170	VAESENC_XOR() \
wolfSSL	15:117db924cf7c	5171	VAESENC_PCLMUL_1(src, 16, (o-128), 112) \
wolfSSL	15:117db924cf7c	5172	VAESENC_PCLMUL_N(src, 32, (o-112), 96) \
wolfSSL	15:117db924cf7c	5173	VAESENC_PCLMUL_N(src, 48, (o- 96), 80) \
wolfSSL	15:117db924cf7c	5174	VAESENC_PCLMUL_N(src, 64, (o- 80), 64) \
wolfSSL	15:117db924cf7c	5175	VAESENC_PCLMUL_N(src, 80, (o- 64), 48) \
wolfSSL	15:117db924cf7c	5176	VAESENC_PCLMUL_N(src, 96, (o- 48), 32) \
wolfSSL	15:117db924cf7c	5177	VAESENC_PCLMUL_N(src, 112, (o- 32), 16) \
wolfSSL	15:117db924cf7c	5178	VAESENC_PCLMUL_N(src, 128, (o- 16), 0) \
wolfSSL	15:117db924cf7c	5179	VAESENC_PCLMUL_L(144) \
wolfSSL	15:117db924cf7c	5180	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5181	"vmovdqa 160(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5182	"jl 4f\n\t" \
wolfSSL	15:117db924cf7c	5183	VAESENC() \
wolfSSL	15:117db924cf7c	5184	VAESENC_SET(176) \
wolfSSL	15:117db924cf7c	5185	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5186	"vmovdqa 192(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5187	"jl 4f\n\t" \
wolfSSL	15:117db924cf7c	5188	VAESENC() \
wolfSSL	15:117db924cf7c	5189	VAESENC_SET(208) \
wolfSSL	15:117db924cf7c	5190	"vmovdqa 224(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5191	"\n" \
wolfSSL	15:117db924cf7c	5192	"4:\n\t" \
wolfSSL	15:117db924cf7c	5193	VAESENC_LAST(%%rcx, %%rdx)
wolfSSL	15:117db924cf7c	5194
wolfSSL	15:117db924cf7c	5195	#define _VAESENC_AVX(r) \
wolfSSL	15:117db924cf7c	5196	"vaesenc 16(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5197	"vaesenc 32(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5198	"vaesenc 48(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5199	"vaesenc 64(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5200	"vaesenc 80(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5201	"vaesenc 96(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5202	"vaesenc 112(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5203	"vaesenc 128(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5204	"vaesenc 144(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5205	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5206	"vmovdqa 160(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5207	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	5208	"vaesenc %%xmm5, " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5209	"vaesenc 176(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5210	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5211	"vmovdqa 192(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5212	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	5213	"vaesenc %%xmm5, " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5214	"vaesenc 208(%[KEY]), " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5215	"vmovdqa 224(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5216	"%=:\n\t" \
wolfSSL	15:117db924cf7c	5217	"vaesenclast %%xmm5, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	5218	#define VAESENC_AVX(r) \
wolfSSL	15:117db924cf7c	5219	_VAESENC_AVX(r)
wolfSSL	15:117db924cf7c	5220
wolfSSL	15:117db924cf7c	5221	#define AESENC_LAST15_ENC_AVX1() \
wolfSSL	15:117db924cf7c	5222	"movl %[nbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5223	"movl %%ecx, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5224	"andl $0x0f, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5225	"jz 55f\n\t" \
wolfSSL	15:117db924cf7c	5226	"vmovdqu " VAR(CTR1) ", %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5227	"vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5228	"vpxor 0(%[KEY]), %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5229	VAESENC_AVX(%%xmm13) \
wolfSSL	15:117db924cf7c	5230	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5231	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5232	"vmovdqu %%xmm13, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	5233	"\n" \
wolfSSL	15:117db924cf7c	5234	"51:\n\t" \
wolfSSL	15:117db924cf7c	5235	"movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	5236	"xorb (%%rsp,%%rcx,1), %%r13b\n\t" \
wolfSSL	15:117db924cf7c	5237	"movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \
wolfSSL	15:117db924cf7c	5238	"movb %%r13b, (%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	5239	"incl " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	5240	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5241	"cmpl %%edx, " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	5242	"jl 51b\n\t" \
wolfSSL	15:117db924cf7c	5243	"xorq %%r13, %%r13\n\t" \
wolfSSL	15:117db924cf7c	5244	"cmpl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5245	"je 53f\n\t" \
wolfSSL	15:117db924cf7c	5246	"\n" \
wolfSSL	15:117db924cf7c	5247	"52:\n\t" \
wolfSSL	15:117db924cf7c	5248	"movb %%r13b, (%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	5249	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5250	"cmpl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5251	"jl 52b\n\t" \
wolfSSL	15:117db924cf7c	5252	"53:\n\t" \
wolfSSL	15:117db924cf7c	5253	"vmovdqu (%%rsp), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5254	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5255	"vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5256	"vpxor %%xmm13, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	5257	GHASH_GFMUL_RED_AVX1(XR, HR, XR) \
wolfSSL	15:117db924cf7c	5258
wolfSSL	15:117db924cf7c	5259	#define AESENC_LAST15_DEC_AVX1() \
wolfSSL	15:117db924cf7c	5260	"movl %[nbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5261	"movl %%ecx, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5262	"andl $0x0f, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5263	"jz 55f\n\t" \
wolfSSL	15:117db924cf7c	5264	"vmovdqu " VAR(CTR1) ", %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5265	"vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5266	"vpxor 0(%[KEY]), %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5267	VAESENC_AVX(%%xmm13) \
wolfSSL	15:117db924cf7c	5268	"subq $32, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5269	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5270	"vmovdqu %%xmm13, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	5271	"vpxor %%xmm0, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5272	"vmovdqu %%xmm0, 16(%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	5273	"\n" \
wolfSSL	15:117db924cf7c	5274	"51:\n\t" \
wolfSSL	15:117db924cf7c	5275	"movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	5276	"movb %%r13b, 16(%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	5277	"xorb (%%rsp,%%rcx,1), %%r13b\n\t" \
wolfSSL	15:117db924cf7c	5278	"movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \
wolfSSL	15:117db924cf7c	5279	"incl " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	5280	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5281	"cmpl %%edx, " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	5282	"jl 51b\n\t" \
wolfSSL	15:117db924cf7c	5283	"53:\n\t" \
wolfSSL	15:117db924cf7c	5284	"vmovdqu 16(%%rsp), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5285	"addq $32, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5286	"vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5287	"vpxor %%xmm13, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	5288	GHASH_GFMUL_RED_AVX1(XR, HR, XR) \
wolfSSL	15:117db924cf7c	5289
wolfSSL	15:117db924cf7c	5290	#define CALC_TAG_AVX1() \
wolfSSL	15:117db924cf7c	5291	"movl %[nbytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	5292	"movl %[abytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5293	"shlq $3, %%rdx\n\t" \
wolfSSL	15:117db924cf7c	5294	"shlq $3, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	5295	"vpinsrq $0, %%rdx, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5296	"vpinsrq $1, %%rcx, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5297	"vpxor %%xmm0, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	5298	GHASH_GFMUL_RED_AVX1(XR, HR, XR) \
wolfSSL	15:117db924cf7c	5299	"vpshufb %[BSWAP_MASK], " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	5300	"vpxor " VAR(TR) ", " VAR(XR) ", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5301
wolfSSL	15:117db924cf7c	5302	#define STORE_TAG_AVX() \
wolfSSL	15:117db924cf7c	5303	"cmpl $16, %[tbytes]\n\t" \
wolfSSL	15:117db924cf7c	5304	"je 71f\n\t" \
wolfSSL	15:117db924cf7c	5305	"xorq %%rcx, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	5306	"vmovdqu %%xmm0, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	5307	"73:\n\t" \
wolfSSL	15:117db924cf7c	5308	"movzbl (%%rsp,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	5309	"movb %%r13b, (%[tag],%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	5310	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5311	"cmpl %[tbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5312	"jne 73b\n\t" \
wolfSSL	15:117db924cf7c	5313	"jmp 72f\n\t" \
wolfSSL	15:117db924cf7c	5314	"\n" \
wolfSSL	15:117db924cf7c	5315	"71:\n\t" \
wolfSSL	15:117db924cf7c	5316	"vmovdqu %%xmm0, (%[tag])\n\t" \
wolfSSL	15:117db924cf7c	5317	"\n" \
wolfSSL	15:117db924cf7c	5318	"72:\n\t"
wolfSSL	15:117db924cf7c	5319
wolfSSL	15:117db924cf7c	5320	#define CMP_TAG_AVX() \
wolfSSL	15:117db924cf7c	5321	"cmpl $16, %[tbytes]\n\t" \
wolfSSL	15:117db924cf7c	5322	"je 71f\n\t" \
wolfSSL	15:117db924cf7c	5323	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5324	"xorq %%rcx, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	5325	"xorq %%rax, %%rax\n\t" \
wolfSSL	15:117db924cf7c	5326	"vmovdqu %%xmm0, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	5327	"\n" \
wolfSSL	15:117db924cf7c	5328	"73:\n\t" \
wolfSSL	15:117db924cf7c	5329	"movzbl (%%rsp,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	5330	"xorb (%[tag],%%rcx,1), %%r13b\n\t" \
wolfSSL	15:117db924cf7c	5331	"orb %%r13b, %%al\n\t" \
wolfSSL	15:117db924cf7c	5332	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5333	"cmpl %[tbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5334	"jne 73b\n\t" \
wolfSSL	15:117db924cf7c	5335	"cmpb $0x00, %%al\n\t" \
wolfSSL	15:117db924cf7c	5336	"sete %%al\n\t" \
wolfSSL	15:117db924cf7c	5337	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5338	"jmp 72f\n\t" \
wolfSSL	15:117db924cf7c	5339	"\n" \
wolfSSL	15:117db924cf7c	5340	"71:\n\t" \
wolfSSL	15:117db924cf7c	5341	"vmovdqu (%[tag]), %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5342	"vpcmpeqb %%xmm1, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5343	"vpmovmskb %%xmm0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5344	"# %%edx == 0xFFFF then return 1 else => return 0\n\t" \
wolfSSL	15:117db924cf7c	5345	"xorl %%eax, %%eax\n\t" \
wolfSSL	15:117db924cf7c	5346	"cmpl $0xffff, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5347	"sete %%al\n\t" \
wolfSSL	15:117db924cf7c	5348	"\n" \
wolfSSL	15:117db924cf7c	5349	"72:\n\t" \
wolfSSL	15:117db924cf7c	5350	"movl %%eax, (%[res])\n\t"
wolfSSL	15:117db924cf7c	5351
wolfSSL	15:117db924cf7c	5352	static void AES_GCM_encrypt_avx1(const unsigned char in, unsigned char out,
wolfSSL	15:117db924cf7c	5353	const unsigned char* addt,
wolfSSL	15:117db924cf7c	5354	const unsigned char* ivec, unsigned char *tag,
wolfSSL	15:117db924cf7c	5355	unsigned int nbytes, unsigned int abytes,
wolfSSL	15:117db924cf7c	5356	unsigned int ibytes, unsigned int tbytes,
wolfSSL	15:117db924cf7c	5357	const unsigned char* key, int nr)
wolfSSL	15:117db924cf7c	5358	{
wolfSSL	15:117db924cf7c	5359	register const unsigned char* iv asm("rax") = ivec;
wolfSSL	15:117db924cf7c	5360	register unsigned int ivLen asm("ebx") = ibytes;
wolfSSL	15:117db924cf7c	5361
wolfSSL	15:117db924cf7c	5362	__asm__ __volatile__ (
wolfSSL	15:117db924cf7c	5363	"subq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	5364	/* Counter is xmm13 */
wolfSSL	15:117db924cf7c	5365	"vpxor %%xmm13, %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	5366	"vpxor " VAR(XR) ", " VAR(XR) ", " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	5367	"movl %[ibytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	5368	"cmpl $12, %%edx\n\t"
wolfSSL	15:117db924cf7c	5369	"jne 35f\n\t"
wolfSSL	15:117db924cf7c	5370	CALC_IV_12_AVX1()
wolfSSL	15:117db924cf7c	5371	"\n"
wolfSSL	15:117db924cf7c	5372	"35:\n\t"
wolfSSL	15:117db924cf7c	5373	CALC_IV_AVX1()
wolfSSL	15:117db924cf7c	5374	"\n"
wolfSSL	15:117db924cf7c	5375	"39:\n\t"
wolfSSL	15:117db924cf7c	5376
wolfSSL	15:117db924cf7c	5377	CALC_AAD_AVX1()
wolfSSL	15:117db924cf7c	5378
wolfSSL	15:117db924cf7c	5379	"# Calculate counter and H\n\t"
wolfSSL	15:117db924cf7c	5380	"vpsrlq $63, " VAR(HR) ", %%xmm5\n\t"
wolfSSL	15:117db924cf7c	5381	"vpsllq $1, " VAR(HR) ", %%xmm4\n\t"
wolfSSL	15:117db924cf7c	5382	"vpslldq $8, %%xmm5, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	5383	"vpor %%xmm5, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	5384	"vpshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	5385	"vpsrad $31, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	5386	"vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	5387	"vpand %[MOD2_128], " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	5388	"vpaddd %[ONE], %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	5389	"vpxor %%xmm4, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	5390	"vmovdqu %%xmm13, " VAR(CTR1) "\n\t"
wolfSSL	15:117db924cf7c	5391
wolfSSL	15:117db924cf7c	5392	"xorl " VAR(KR) ", " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5393
wolfSSL	15:117db924cf7c	5394	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	5395	"cmpl $128, %[nbytes]\n\t"
wolfSSL	15:117db924cf7c	5396	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	5397	"jl 5f\n\t"
wolfSSL	15:117db924cf7c	5398	"andl $0xffffff80, %%r13d\n\t"
wolfSSL	15:117db924cf7c	5399
wolfSSL	15:117db924cf7c	5400	CALC_HT_8_AVX1()
wolfSSL	15:117db924cf7c	5401
wolfSSL	15:117db924cf7c	5402	"# First 128 bytes of input\n\t"
wolfSSL	15:117db924cf7c	5403	VAESENC_128()
wolfSSL	15:117db924cf7c	5404
wolfSSL	15:117db924cf7c	5405	"cmpl $128, %%r13d\n\t"
wolfSSL	15:117db924cf7c	5406	"movl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5407	"jle 2f\n\t"
wolfSSL	15:117db924cf7c	5408
wolfSSL	15:117db924cf7c	5409	"# More 128 bytes of input\n\t"
wolfSSL	15:117db924cf7c	5410	"\n"
wolfSSL	15:117db924cf7c	5411	"3:\n\t"
wolfSSL	15:117db924cf7c	5412	VAESENC_128_GHASH_AVX1(%%rdx, 0)
wolfSSL	15:117db924cf7c	5413	"addl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5414	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5415	"jl 3b\n\t"
wolfSSL	15:117db924cf7c	5416	"\n"
wolfSSL	15:117db924cf7c	5417	"2:\n\t"
wolfSSL	15:117db924cf7c	5418	"vmovdqa %[BSWAP_MASK], %%xmm13\n\t"
wolfSSL	15:117db924cf7c	5419	"vpshufb %%xmm13, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	5420	"vpshufb %%xmm13, %%xmm5, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	5421	"vpshufb %%xmm13, %%xmm6, %%xmm6\n\t"
wolfSSL	15:117db924cf7c	5422	"vpshufb %%xmm13, %%xmm7, %%xmm7\n\t"
wolfSSL	15:117db924cf7c	5423	"vpxor %%xmm2, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	5424	"vpshufb %%xmm13, %%xmm8, %%xmm8\n\t"
wolfSSL	15:117db924cf7c	5425	"vpshufb %%xmm13, %%xmm9, %%xmm9\n\t"
wolfSSL	15:117db924cf7c	5426	"vpshufb %%xmm13, %%xmm10, %%xmm10\n\t"
wolfSSL	15:117db924cf7c	5427	"vpshufb %%xmm13, %%xmm11, %%xmm11\n\t"
wolfSSL	15:117db924cf7c	5428
wolfSSL	15:117db924cf7c	5429	"vmovdqu (" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	5430	"vmovdqu 16(" VAR(HTR) "), %%xmm14\n\t"
wolfSSL	15:117db924cf7c	5431	GHASH_GFMUL_AVX1(XR, %%xmm13, %%xmm11, %%xmm12)
wolfSSL	15:117db924cf7c	5432	GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm10, %%xmm14)
wolfSSL	15:117db924cf7c	5433	"vmovdqu 32(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	5434	"vmovdqu 48(" VAR(HTR) "), %%xmm14\n\t"
wolfSSL	15:117db924cf7c	5435	GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm9, %%xmm12)
wolfSSL	15:117db924cf7c	5436	GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm8, %%xmm14)
wolfSSL	15:117db924cf7c	5437	"vmovdqu 64(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	5438	"vmovdqu 80(" VAR(HTR) "), %%xmm14\n\t"
wolfSSL	15:117db924cf7c	5439	GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm7, %%xmm12)
wolfSSL	15:117db924cf7c	5440	GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm6, %%xmm14)
wolfSSL	15:117db924cf7c	5441	"vmovdqu 96(" VAR(HTR) "), %%xmm12\n\t"
wolfSSL	15:117db924cf7c	5442	"vmovdqu 112(" VAR(HTR) "), %%xmm14\n\t"
wolfSSL	15:117db924cf7c	5443	GHASH_GFMUL_XOR_AVX1(XR, %%xmm13, %%xmm5, %%xmm12)
wolfSSL	15:117db924cf7c	5444	GHASH_GFMUL_RED_XOR_AVX1(XR, %%xmm13, %%xmm4, %%xmm14)
wolfSSL	15:117db924cf7c	5445
wolfSSL	15:117db924cf7c	5446	"vmovdqu 0(" VAR(HTR) "), " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	5447	"\n"
wolfSSL	15:117db924cf7c	5448	"5:\n\t"
wolfSSL	15:117db924cf7c	5449	"movl %[nbytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	5450	"cmpl %%edx, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5451	"jge 55f\n\t"
wolfSSL	15:117db924cf7c	5452	#endif
wolfSSL	15:117db924cf7c	5453
wolfSSL	15:117db924cf7c	5454	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	5455	"andl $0xfffffff0, %%r13d\n\t"
wolfSSL	15:117db924cf7c	5456	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5457	"jge 14f\n\t"
wolfSSL	15:117db924cf7c	5458
wolfSSL	15:117db924cf7c	5459	VAESENC_BLOCK()
wolfSSL	15:117db924cf7c	5460	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5461	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5462	"jge 13f\n\t"
wolfSSL	15:117db924cf7c	5463	"\n"
wolfSSL	15:117db924cf7c	5464	"12:\n\t"
wolfSSL	15:117db924cf7c	5465	"vmovdqu (%[in]," VAR(KR64) ",1), %%xmm9\n\t"
wolfSSL	15:117db924cf7c	5466	VAESENC_GFMUL(%%xmm9, HR, XR)
wolfSSL	15:117db924cf7c	5467	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	5468	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5469	"vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	5470	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	5471	"jl 12b\n\t"
wolfSSL	15:117db924cf7c	5472	"\n"
wolfSSL	15:117db924cf7c	5473	"13:\n\t"
wolfSSL	15:117db924cf7c	5474	GHASH_GFMUL_RED_AVX1(XR, HR, XR)
wolfSSL	15:117db924cf7c	5475	"\n"
wolfSSL	15:117db924cf7c	5476	"14:\n\t"
wolfSSL	15:117db924cf7c	5477
wolfSSL	15:117db924cf7c	5478	AESENC_LAST15_ENC_AVX1()
wolfSSL	15:117db924cf7c	5479	"\n"
wolfSSL	15:117db924cf7c	5480	"55:\n\t"
wolfSSL	15:117db924cf7c	5481
wolfSSL	15:117db924cf7c	5482	CALC_TAG_AVX1()
wolfSSL	15:117db924cf7c	5483	STORE_TAG_AVX()
wolfSSL	15:117db924cf7c	5484	"addq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	5485	"vzeroupper\n\t"
wolfSSL	15:117db924cf7c	5486
wolfSSL	15:117db924cf7c	5487	:
wolfSSL	15:117db924cf7c	5488	: [KEY] "r" (key),
wolfSSL	15:117db924cf7c	5489	[in] "r" (in), [out] "r" (out), [nr] "r" (nr),
wolfSSL	15:117db924cf7c	5490	[nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt),
wolfSSL	15:117db924cf7c	5491	[ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tbytes),
wolfSSL	15:117db924cf7c	5492	[tag] "r" (tag),
wolfSSL	15:117db924cf7c	5493	[BSWAP_MASK] "m" (BSWAP_MASK),
wolfSSL	15:117db924cf7c	5494	[BSWAP_EPI64] "m" (BSWAP_EPI64),
wolfSSL	15:117db924cf7c	5495	[ONE] "m" (ONE),
wolfSSL	15:117db924cf7c	5496	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	5497	[TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR),
wolfSSL	15:117db924cf7c	5498	[FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN),
wolfSSL	15:117db924cf7c	5499	[EIGHT] "m" (EIGHT),
wolfSSL	15:117db924cf7c	5500	#endif
wolfSSL	15:117db924cf7c	5501	[MOD2_128] "m" (MOD2_128)
wolfSSL	15:117db924cf7c	5502	: "xmm15", "xmm14", "xmm13", "xmm12",
wolfSSL	15:117db924cf7c	5503	"xmm0", "xmm1", "xmm2", "xmm3", "memory",
wolfSSL	15:117db924cf7c	5504	"xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
wolfSSL	15:117db924cf7c	5505	"rcx", "rdx", "r13"
wolfSSL	15:117db924cf7c	5506	);
wolfSSL	15:117db924cf7c	5507	}
wolfSSL	15:117db924cf7c	5508
wolfSSL	15:117db924cf7c	5509	#ifdef HAVE_INTEL_AVX2
wolfSSL	15:117db924cf7c	5510	/* Encrypt and carry-less multiply for AVX2. */
wolfSSL	15:117db924cf7c	5511	#define VAESENC_PCLMUL_AVX2_1(src, o1, o2, o3) \
wolfSSL	15:117db924cf7c	5512	"vmovdqu " #o2 "(" #src "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5513	"vmovdqa " #o1 "(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5514	"vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5515	"vmovdqu " #o3 "(" VAR(HTR) "), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5516	"vpxor %%xmm2, %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5517	"vpclmulqdq $0x10, %%xmm13, %%xmm12, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5518	"vpclmulqdq $0x01, %%xmm13, %%xmm12, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	5519	"vpclmulqdq $0x00, %%xmm13, %%xmm12, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5520	"vpclmulqdq $0x11, %%xmm13, %%xmm12, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5521	"vaesenc %%xmm0, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5522	"vaesenc %%xmm0, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5523	"vaesenc %%xmm0, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5524	"vaesenc %%xmm0, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5525	"vaesenc %%xmm0, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5526	"vaesenc %%xmm0, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5527	"vaesenc %%xmm0, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	5528	"vaesenc %%xmm0, %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	5529
wolfSSL	15:117db924cf7c	5530	#define VAESENC_PCLMUL_AVX2_2(src, o1, o2, o3) \
wolfSSL	15:117db924cf7c	5531	"vmovdqu " #o2 "(" #src "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5532	"vmovdqu " #o3 "(" VAR(HTR) "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5533	"vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5534	"vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5535	"vpclmulqdq $0x10, %%xmm0, %%xmm12, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5536	"vpclmulqdq $0x01, %%xmm0, %%xmm12, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	5537	"vpclmulqdq $0x00, %%xmm0, %%xmm12, %%xmm15\n\t" \
wolfSSL	15:117db924cf7c	5538	"vpclmulqdq $0x11, %%xmm0, %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5539	"vmovdqa " #o1 "(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5540	"vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5541	"vpxor %%xmm12, %%xmm3, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5542	"vaesenc %%xmm0, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5543	"vaesenc %%xmm0, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5544	"vaesenc %%xmm0, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5545	"vaesenc %%xmm0, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5546	"vaesenc %%xmm0, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5547	"vaesenc %%xmm0, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5548	"vaesenc %%xmm0, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	5549	"vaesenc %%xmm0, %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	5550
wolfSSL	15:117db924cf7c	5551	#define VAESENC_PCLMUL_AVX2_N(src, o1, o2, o3) \
wolfSSL	15:117db924cf7c	5552	"vmovdqu " #o2 "(" #src "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5553	"vmovdqu " #o3 "(" VAR(HTR) "), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5554	"vpshufb %[BSWAP_MASK], %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5555	"vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5556	"vpxor %%xmm15, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5557	"vpclmulqdq $0x10, %%xmm0, %%xmm12, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5558	"vpclmulqdq $0x01, %%xmm0, %%xmm12, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	5559	"vpclmulqdq $0x00, %%xmm0, %%xmm12, %%xmm15\n\t" \
wolfSSL	15:117db924cf7c	5560	"vpclmulqdq $0x11, %%xmm0, %%xmm12, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5561	"vmovdqa " #o1 "(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5562	"vpxor %%xmm13, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5563	"vpxor %%xmm12, %%xmm3, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5564	"vaesenc %%xmm0, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5565	"vaesenc %%xmm0, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5566	"vaesenc %%xmm0, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5567	"vaesenc %%xmm0, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5568	"vaesenc %%xmm0, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5569	"vaesenc %%xmm0, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5570	"vaesenc %%xmm0, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	5571	"vaesenc %%xmm0, %%xmm11, %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	5572
wolfSSL	15:117db924cf7c	5573	#define VAESENC_PCLMUL_AVX2_L(o) \
wolfSSL	15:117db924cf7c	5574	"vpxor %%xmm14, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5575	"vpxor %%xmm15, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5576	"vpslldq $8, %%xmm1, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5577	"vpsrldq $8, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5578	"vmovdqa "#o"(%[KEY]), %%xmm15\n\t" \
wolfSSL	15:117db924cf7c	5579	"vmovdqa %[MOD2_128], %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5580	"vaesenc %%xmm15, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5581	"vpxor %%xmm12, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5582	"vpxor %%xmm1, %%xmm3, %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5583	"vpclmulqdq $0x10, %%xmm0, %%xmm2, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	5584	"vaesenc %%xmm15, %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5585	"vaesenc %%xmm15, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5586	"vaesenc %%xmm15, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5587	"vpshufd $0x4e, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5588	"vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5589	"vpclmulqdq $0x10, %%xmm0, %%xmm2, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	5590	"vaesenc %%xmm15, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5591	"vaesenc %%xmm15, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5592	"vaesenc %%xmm15, %%xmm10, %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	5593	"vpshufd $0x4e, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5594	"vpxor %%xmm14, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5595	"vpxor %%xmm3, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5596	"vaesenc %%xmm15, %%xmm11, %%xmm11\n\t"
wolfSSL	15:117db924cf7c	5597
wolfSSL	15:117db924cf7c	5598	#define VAESENC_BLOCK_AVX2() \
wolfSSL	15:117db924cf7c	5599	"vmovdqu " VAR(CTR1) ", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5600	"vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5601	"vpaddd %[ONE], %%xmm5, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5602	"vmovdqu %%xmm5, " VAR(CTR1) "\n\t" \
wolfSSL	15:117db924cf7c	5603	"vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5604	"vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5605	"vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5606	"vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5607	"vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5608	"vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5609	"vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5610	"vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5611	"vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5612	"vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5613	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5614	"vmovdqa 160(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5615	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	5616	"vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5617	"vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5618	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5619	"vmovdqa 192(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5620	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	5621	"vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5622	"vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5623	"vmovdqa 224(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5624	"%=:\n\t" \
wolfSSL	15:117db924cf7c	5625	"vaesenclast %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5626	"vmovdqu (%[in]," VAR(KR64) ",1), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5627	"vpxor %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5628	"vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t" \
wolfSSL	15:117db924cf7c	5629	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5630	"vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	5631
wolfSSL	15:117db924cf7c	5632	/* Karatsuba multiplication - slower
wolfSSL	15:117db924cf7c	5633	* H01 = H[1] ^ H[0] (top and bottom 64-bits XORed)
wolfSSL	15:117db924cf7c	5634	*/
wolfSSL	15:117db924cf7c	5635	#define _VAESENC_GFMUL_AVX2(in, H, X, ctr1, H01) \
wolfSSL	15:117db924cf7c	5636	"vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5637	"vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5638	"vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5639	"vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5640	"vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5641	"vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5642	"vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5643	"vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5644	"vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5645	"vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5646	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5647	"vmovdqa 160(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5648	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	5649	"vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5650	"vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5651	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5652	"vmovdqa 192(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5653	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	5654	"vaesenc %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5655	"vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5656	"vmovdqa 224(%[KEY]), %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5657	"%=:\n\t" \
wolfSSL	15:117db924cf7c	5658	"vaesenclast %%xmm5, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5659	"vmovdqu " #in ", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5660	"vpxor %%xmm0, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5661	\
wolfSSL	15:117db924cf7c	5662	"vpsrldq $8, " #X ", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5663	"vpxor " #X ", %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5664	"vpclmulqdq $0x00, " #H ", " #X ", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5665	"vpclmulqdq $0x11, " #H ", " #X ", %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5666	"vpclmulqdq $0x00, "#H01", %%xmm2, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5667	"vpxor %%xmm5, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5668	"vpxor %%xmm8, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5669	"vpslldq $8, %%xmm7, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5670	"vpsrldq $8, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5671	"vpxor %%xmm7, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5672	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5673	\
wolfSSL	15:117db924cf7c	5674	"vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5675	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5676	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5677	"vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5678	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5679	"vpxor %%xmm8, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5680	"vpxor %%xmm5, %%xmm6, " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	5681	#define VAESENC_GFMUL_AVX2(in, H, X, ctr1) \
wolfSSL	15:117db924cf7c	5682	_VAESENC_GFMUL_AVX2(in, H, X, ctr1)
wolfSSL	15:117db924cf7c	5683
wolfSSL	15:117db924cf7c	5684	#define _VAESENC_GFMUL_SB_AVX2(in, H, X, ctr1) \
wolfSSL	15:117db924cf7c	5685	"vpclmulqdq $0x10, " #H ", " #X ", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5686	"vpclmulqdq $0x01, " #H ", " #X ", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5687	"vpclmulqdq $0x00, " #H ", " #X ", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5688	"vpclmulqdq $0x11, " #H ", " #X ", %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5689	"vpxor (%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5690	"vaesenc 16(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5691	"vpxor %%xmm6, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5692	"vpslldq $8, %%xmm7, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5693	"vpsrldq $8, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5694	"vaesenc 32(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5695	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5696	"vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5697	"vaesenc 48(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5698	"vaesenc 64(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5699	"vaesenc 80(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5700	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5701	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5702	"vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5703	"vaesenc 96(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5704	"vaesenc 112(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5705	"vaesenc 128(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5706	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5707	"vaesenc 144(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5708	"vpxor %%xmm7, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5709	"vpxor %%xmm8, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5710	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5711	"vmovdqa 160(%[KEY]), %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5712	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	5713	"vaesenc %%xmm3, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5714	"vaesenc 176(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5715	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5716	"vmovdqa 192(%[KEY]), %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5717	"jl %=f\n\t" \
wolfSSL	15:117db924cf7c	5718	"vaesenc %%xmm3, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5719	"vaesenc 208(%[KEY]), %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5720	"vmovdqa 224(%[KEY]), %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5721	"%=:\n\t" \
wolfSSL	15:117db924cf7c	5722	"vaesenclast %%xmm3, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5723	"vpxor %%xmm5, %%xmm6, " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	5724	"vmovdqu " #in ", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5725	"vpxor %%xmm5, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	5726	#define VAESENC_GFMUL_SB_AVX2(in, H, X, ctr1) \
wolfSSL	15:117db924cf7c	5727	_VAESENC_GFMUL_SB_AVX2(in, H, X, ctr1)
wolfSSL	15:117db924cf7c	5728
wolfSSL	15:117db924cf7c	5729
wolfSSL	15:117db924cf7c	5730	#define _GHASH_GFMUL_AVX2(r, r2, a, b) \
wolfSSL	15:117db924cf7c	5731	"vpclmulqdq $0x10, "#a", "#b", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5732	"vpclmulqdq $0x01, "#a", "#b", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5733	"vpclmulqdq $0x00, "#a", "#b", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5734	"vpclmulqdq $0x11, "#a", "#b", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5735	"vpxor %%xmm1, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5736	"vpslldq $8, %%xmm2, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5737	"vpsrldq $8, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5738	"vpxor %%xmm1, %%xmm0, "#r2"\n\t" \
wolfSSL	15:117db924cf7c	5739	"vpxor %%xmm2, %%xmm3, " #r "\n\t"
wolfSSL	15:117db924cf7c	5740	#define GHASH_GFMUL_AVX2(r, r2, a, b) \
wolfSSL	15:117db924cf7c	5741	_GHASH_GFMUL_AVX2(r, r2, a, b)
wolfSSL	15:117db924cf7c	5742
wolfSSL	15:117db924cf7c	5743	#define GHASH_MID_AVX2(r, r2) \
wolfSSL	15:117db924cf7c	5744	"vpsrld $31, "#r2", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5745	"vpsrld $31, " #r ", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5746	"vpslld $1, "#r2", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	5747	"vpslld $1, " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5748	"vpsrldq $12, %%xmm0, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5749	"vpslldq $4, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5750	"vpslldq $4, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5751	"vpor %%xmm2, " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5752	"vpor %%xmm0, "#r2", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	5753	"vpor %%xmm1, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	5754
wolfSSL	15:117db924cf7c	5755	#define _GHASH_GFMUL_RED_AVX2(r, a, b) \
wolfSSL	15:117db924cf7c	5756	"vpclmulqdq $0x10, "#a", "#b", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5757	"vpclmulqdq $0x01, "#a", "#b", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5758	"vpclmulqdq $0x00, "#a", "#b", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5759	"vpxor %%xmm6, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5760	"vpslldq $8, %%xmm7, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5761	"vpsrldq $8, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5762	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5763	"vpclmulqdq $0x11, "#a", "#b", %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5764	"vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5765	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5766	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5767	"vpclmulqdq $0x10, %[MOD2_128], %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5768	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5769	"vpxor %%xmm7, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5770	"vpxor %%xmm8, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5771	"vpxor %%xmm5, %%xmm6, " #r "\n\t"
wolfSSL	15:117db924cf7c	5772	#define GHASH_GFMUL_RED_AVX2(r, a, b) \
wolfSSL	15:117db924cf7c	5773	_GHASH_GFMUL_RED_AVX2(r, a, b)
wolfSSL	15:117db924cf7c	5774
wolfSSL	15:117db924cf7c	5775	#define _GHASH_GFSQR_RED2_AVX2(r, a, mod128) \
wolfSSL	15:117db924cf7c	5776	"vpclmulqdq $0x00, "#a", "#a", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5777	"vpclmulqdq $0x11, "#a", "#a", %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5778	"vpclmulqdq $0x10, "#mod128", %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5779	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5780	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5781	"vpclmulqdq $0x10, "#mod128", %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5782	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5783	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5784	"vpxor %%xmm6, %%xmm8, " #r "\n\t"
wolfSSL	15:117db924cf7c	5785	#define GHASH_GFSQR_RED2_AVX2(r, a, mod128) \
wolfSSL	15:117db924cf7c	5786	_GHASH_GFSQR_RED2_AVX2(r, a, mod128)
wolfSSL	15:117db924cf7c	5787
wolfSSL	15:117db924cf7c	5788	#define _GHASH_GFMUL_SQR_RED2_AVX2(rm, rs, a, b, mod128) \
wolfSSL	15:117db924cf7c	5789	"vpclmulqdq $0x10, "#a", "#b", %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5790	"vpclmulqdq $0x01, "#a", "#b", %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5791	"vpclmulqdq $0x00, "#a", "#b", %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5792	"vpclmulqdq $0x11, "#a", "#b", %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5793	"vpclmulqdq $0x00, "#b", "#b", %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5794	"vpclmulqdq $0x11, "#b", "#b", %%xmm10\n\t" \
wolfSSL	15:117db924cf7c	5795	"vpxor %%xmm6, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5796	"vpslldq $8, %%xmm7, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5797	"vpsrldq $8, %%xmm7, %%xmm7\n\t" \
wolfSSL	15:117db924cf7c	5798	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5799	"vpclmulqdq $0x10, "#mod128", %%xmm9, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5800	"vpclmulqdq $0x10, "#mod128", %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5801	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5802	"vpshufd $0x4e, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5803	"vpxor %%xmm5, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5804	"vpxor %%xmm4, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5805	"vpclmulqdq $0x10, "#mod128", %%xmm6, %%xmm5\n\t" \
wolfSSL	15:117db924cf7c	5806	"vpclmulqdq $0x10, "#mod128", %%xmm9, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5807	"vpshufd $0x4e, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5808	"vpshufd $0x4e, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5809	"vpxor %%xmm7, %%xmm8, %%xmm8\n\t" \
wolfSSL	15:117db924cf7c	5810	"vpxor %%xmm4, %%xmm9, %%xmm9\n\t" \
wolfSSL	15:117db924cf7c	5811	"vpxor %%xmm8, %%xmm6, %%xmm6\n\t" \
wolfSSL	15:117db924cf7c	5812	"vpxor %%xmm10, %%xmm9, "#rs"\n\t" \
wolfSSL	15:117db924cf7c	5813	"vpxor %%xmm5, %%xmm6, "#rm"\n\t"
wolfSSL	15:117db924cf7c	5814	#define GHASH_GFMUL_SQR_RED2_AVX2(rm, rs, a, b, mod128) \
wolfSSL	15:117db924cf7c	5815	_GHASH_GFMUL_SQR_RED2_AVX2(rm, rs, a, b, mod128)
wolfSSL	15:117db924cf7c	5816
wolfSSL	15:117db924cf7c	5817	#define CALC_HT_8_AVX2() \
wolfSSL	15:117db924cf7c	5818	"vmovdqa %[MOD2_128], %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	5819	"vmovdqa " VAR(XR) ", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5820	"# H ^ 1 and H ^ 2\n\t" \
wolfSSL	15:117db924cf7c	5821	GHASH_GFSQR_RED2_AVX2(%%xmm0, HR, %%xmm11) \
wolfSSL	15:117db924cf7c	5822	"vmovdqu " VAR(HR) ", 0(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5823	"vmovdqu %%xmm0 , 16(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5824	"# H ^ 3 and H ^ 4\n\t" \
wolfSSL	15:117db924cf7c	5825	GHASH_GFMUL_SQR_RED2_AVX2(%%xmm1, %%xmm3, HR, %%xmm0, %%xmm11) \
wolfSSL	15:117db924cf7c	5826	"vmovdqu %%xmm1 , 32(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5827	"vmovdqu %%xmm3 , 48(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5828	"# H ^ 5 and H ^ 6\n\t" \
wolfSSL	15:117db924cf7c	5829	GHASH_GFMUL_SQR_RED2_AVX2(%%xmm12, %%xmm0, %%xmm0, %%xmm1, %%xmm11) \
wolfSSL	15:117db924cf7c	5830	"vmovdqu %%xmm12, 64(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5831	"vmovdqu %%xmm0 , 80(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5832	"# H ^ 7 and H ^ 8\n\t" \
wolfSSL	15:117db924cf7c	5833	GHASH_GFMUL_SQR_RED2_AVX2(%%xmm12, %%xmm0, %%xmm1, %%xmm3, %%xmm11) \
wolfSSL	15:117db924cf7c	5834	"vmovdqu %%xmm12, 96(" VAR(HTR) ")\n\t" \
wolfSSL	15:117db924cf7c	5835	"vmovdqu %%xmm0 , 112(" VAR(HTR) ")\n\t"
wolfSSL	15:117db924cf7c	5836
wolfSSL	15:117db924cf7c	5837	#define _GHASH_RED_AVX2(r, r2) \
wolfSSL	15:117db924cf7c	5838	"vmovdqa %[MOD2_128], %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5839	"vpclmulqdq $0x10, %%xmm2, "#r2", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5840	"vpshufd $0x4e, "#r2", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5841	"vpxor %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5842	"vpclmulqdq $0x10, %%xmm2, %%xmm1, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5843	"vpshufd $0x4e, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5844	"vpxor %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5845	"vpxor %%xmm1, " #r ", " #r "\n\t"
wolfSSL	15:117db924cf7c	5846	#define GHASH_RED_AVX2(r, r2) \
wolfSSL	15:117db924cf7c	5847	_GHASH_RED_AVX2(r, r2)
wolfSSL	15:117db924cf7c	5848
wolfSSL	15:117db924cf7c	5849	#define GHASH_FULL_AVX2(r, r2, a, b) \
wolfSSL	15:117db924cf7c	5850	GHASH_GFMUL_AVX2(r, r2, a, b) \
wolfSSL	15:117db924cf7c	5851	GHASH_MID_AVX2(r, r2) \
wolfSSL	15:117db924cf7c	5852	GHASH_RED_AVX2(r, r2)
wolfSSL	15:117db924cf7c	5853
wolfSSL	15:117db924cf7c	5854	#define _GFMUL_3V_AVX2(r, r2, r3, a, b) \
wolfSSL	15:117db924cf7c	5855	"vpclmulqdq $0x10, "#a", "#b", "#r3"\n\t" \
wolfSSL	15:117db924cf7c	5856	"vpclmulqdq $0x01, "#a", "#b", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5857	"vpclmulqdq $0x00, "#a", "#b", "#r2"\n\t" \
wolfSSL	15:117db924cf7c	5858	"vpclmulqdq $0x11, "#a", "#b", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5859	"vpxor %%xmm1, "#r3", "#r3"\n\t"
wolfSSL	15:117db924cf7c	5860	#define GFMUL_3V_AVX2(r, r2, r3, a, b) \
wolfSSL	15:117db924cf7c	5861	_GFMUL_3V_AVX2(r, r2, r3, a, b)
wolfSSL	15:117db924cf7c	5862
wolfSSL	15:117db924cf7c	5863	#define _GFMUL_XOR_3V_AVX2(r, r2, r3, a, b) \
wolfSSL	15:117db924cf7c	5864	"vpclmulqdq $0x10, "#a", "#b", %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5865	"vpclmulqdq $0x01, "#a", "#b", %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5866	"vpclmulqdq $0x00, "#a", "#b", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5867	"vpclmulqdq $0x11, "#a", "#b", %%xmm3\n\t" \
wolfSSL	15:117db924cf7c	5868	"vpxor %%xmm1, %%xmm2, %%xmm2\n\t" \
wolfSSL	15:117db924cf7c	5869	"vpxor %%xmm3, " #r ", " #r "\n\t" \
wolfSSL	15:117db924cf7c	5870	"vpxor %%xmm2, "#r3", "#r3"\n\t" \
wolfSSL	15:117db924cf7c	5871	"vpxor %%xmm0, "#r2", "#r2"\n\t"
wolfSSL	15:117db924cf7c	5872	#define GFMUL_XOR_3V_AVX2(r, r2, r3, a, b) \
wolfSSL	15:117db924cf7c	5873	_GFMUL_XOR_3V_AVX2(r, r2, r3, a, b)
wolfSSL	15:117db924cf7c	5874
wolfSSL	15:117db924cf7c	5875	#define GHASH_GFMUL_RED_8_AVX2() \
wolfSSL	15:117db924cf7c	5876	"vmovdqu (" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5877	GFMUL_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm11, %%xmm12) \
wolfSSL	15:117db924cf7c	5878	"vmovdqu 16(" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5879	GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm10, %%xmm12) \
wolfSSL	15:117db924cf7c	5880	"vmovdqu 32(" VAR(HTR) "), %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	5881	"vmovdqu 48(" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5882	GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm9, %%xmm11) \
wolfSSL	15:117db924cf7c	5883	GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm8, %%xmm12) \
wolfSSL	15:117db924cf7c	5884	"vmovdqu 64(" VAR(HTR) "), %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	5885	"vmovdqu 80(" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5886	GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm7, %%xmm11) \
wolfSSL	15:117db924cf7c	5887	GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm6, %%xmm12) \
wolfSSL	15:117db924cf7c	5888	"vmovdqu 96(" VAR(HTR) "), %%xmm11\n\t" \
wolfSSL	15:117db924cf7c	5889	"vmovdqu 112(" VAR(HTR) "), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5890	GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm5, %%xmm11) \
wolfSSL	15:117db924cf7c	5891	GFMUL_XOR_3V_AVX2(XR, %%xmm13, %%xmm14, %%xmm4, %%xmm12) \
wolfSSL	15:117db924cf7c	5892	"vpslldq $8, %%xmm14, %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5893	"vpsrldq $8, %%xmm14, %%xmm14\n\t" \
wolfSSL	15:117db924cf7c	5894	"vpxor %%xmm12, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5895	"vpxor %%xmm14, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	5896	GHASH_RED_AVX2(XR, %%xmm13)
wolfSSL	15:117db924cf7c	5897
wolfSSL	15:117db924cf7c	5898	#define CALC_IV_12_AVX2() \
wolfSSL	15:117db924cf7c	5899	"# Calculate values when IV is 12 bytes\n\t" \
wolfSSL	15:117db924cf7c	5900	"# Set counter based on IV\n\t" \
wolfSSL	15:117db924cf7c	5901	"movl $0x01000000, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5902	"vpinsrq $0, 0(%%rax), %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5903	"vpinsrd $2, 8(%%rax), %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5904	"vpinsrd $3, %%ecx, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5905	"# H = Encrypt X(=0) and T = Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	5906	"vmovdqa 0(%[KEY]), " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5907	"vmovdqa 16(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5908	"vpxor " VAR(HR) ", %%xmm13, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5909	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5910	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5911	"vmovdqa 32(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5912	"vmovdqa 48(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5913	"vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5914	"vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5915	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5916	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5917	"vmovdqa 64(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5918	"vmovdqa 80(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5919	"vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5920	"vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5921	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5922	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5923	"vmovdqa 96(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5924	"vmovdqa 112(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5925	"vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5926	"vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5927	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5928	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5929	"vmovdqa 128(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5930	"vmovdqa 144(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5931	"vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5932	"vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5933	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5934	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5935	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5936	"vmovdqa 160(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5937	"jl 31f\n\t" \
wolfSSL	15:117db924cf7c	5938	"vmovdqa 176(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5939	"vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5940	"vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5941	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5942	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5943	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	5944	"vmovdqa 192(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5945	"jl 31f\n\t" \
wolfSSL	15:117db924cf7c	5946	"vmovdqa 208(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	5947	"vaesenc %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5948	"vaesenc %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5949	"vaesenc %%xmm12, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5950	"vaesenc %%xmm12, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5951	"vmovdqu 224(%[KEY]), %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	5952	"31:\n\t" \
wolfSSL	15:117db924cf7c	5953	"vaesenclast %%xmm0, " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5954	"vaesenclast %%xmm0, %%xmm1, %%xmm1\n\t" \
wolfSSL	15:117db924cf7c	5955	"vpshufb %[BSWAP_MASK], " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5956	"vmovdqu %%xmm1, " VAR(TR) "\n\t" \
wolfSSL	15:117db924cf7c	5957
wolfSSL	15:117db924cf7c	5958	#define CALC_IV_AVX2() \
wolfSSL	15:117db924cf7c	5959	"# Calculate values when IV is not 12 bytes\n\t" \
wolfSSL	15:117db924cf7c	5960	"# H = Encrypt X(=0)\n\t" \
wolfSSL	15:117db924cf7c	5961	"vmovdqa 0(%[KEY]), " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5962	VAESENC_AVX(HR) \
wolfSSL	15:117db924cf7c	5963	"vpshufb %[BSWAP_MASK], " VAR(HR) ", " VAR(HR) "\n\t" \
wolfSSL	15:117db924cf7c	5964	"# Calc counter\n\t" \
wolfSSL	15:117db924cf7c	5965	"# Initialization vector\n\t" \
wolfSSL	15:117db924cf7c	5966	"cmpl $0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5967	"movq $0, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	5968	"je 45f\n\t" \
wolfSSL	15:117db924cf7c	5969	"cmpl $16, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5970	"jl 44f\n\t" \
wolfSSL	15:117db924cf7c	5971	"andl $0xfffffff0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	5972	"\n" \
wolfSSL	15:117db924cf7c	5973	"43:\n\t" \
wolfSSL	15:117db924cf7c	5974	"vmovdqu (%%rax,%%rcx,1), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5975	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5976	"vpxor %%xmm4, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	5977	GHASH_FULL_AVX2(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	5978	"addl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5979	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5980	"jl 43b\n\t" \
wolfSSL	15:117db924cf7c	5981	"movl %[ibytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	5982	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5983	"je 45f\n\t" \
wolfSSL	15:117db924cf7c	5984	"\n" \
wolfSSL	15:117db924cf7c	5985	"44:\n\t" \
wolfSSL	15:117db924cf7c	5986	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5987	"vpxor %%xmm4, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5988	"xorl %%ebx, %%ebx\n\t" \
wolfSSL	15:117db924cf7c	5989	"vmovdqu %%xmm4, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	5990	"42:\n\t" \
wolfSSL	15:117db924cf7c	5991	"movzbl (%%rax,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	5992	"movb %%r13b, (%%rsp,%%rbx,1)\n\t" \
wolfSSL	15:117db924cf7c	5993	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5994	"incl %%ebx\n\t" \
wolfSSL	15:117db924cf7c	5995	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	5996	"jl 42b\n\t" \
wolfSSL	15:117db924cf7c	5997	"vmovdqu (%%rsp), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	5998	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	5999	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	6000	"vpxor %%xmm4, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6001	GHASH_FULL_AVX2(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	6002	"\n" \
wolfSSL	15:117db924cf7c	6003	"45:\n\t" \
wolfSSL	15:117db924cf7c	6004	"# T = Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	6005	"vpxor %%xmm0, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	6006	"shll $3, %%edx\n\t" \
wolfSSL	15:117db924cf7c	6007	"vpinsrq $0, %%rdx, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	6008	"vpxor %%xmm0, %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6009	GHASH_FULL_AVX2(%%xmm13, %%xmm12, %%xmm13, HR) \
wolfSSL	15:117db924cf7c	6010	"vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6011	"# Encrypt counter\n\t" \
wolfSSL	15:117db924cf7c	6012	"vmovdqa 0(%[KEY]), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	6013	"vpxor %%xmm13, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	6014	VAESENC_AVX(%%xmm4) \
wolfSSL	15:117db924cf7c	6015	"vmovdqu %%xmm4, " VAR(TR) "\n\t"
wolfSSL	15:117db924cf7c	6016
wolfSSL	15:117db924cf7c	6017	#define CALC_AAD_AVX2() \
wolfSSL	15:117db924cf7c	6018	"# Additional authentication data\n\t" \
wolfSSL	15:117db924cf7c	6019	"movl %[abytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	6020	"cmpl $0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	6021	"je 25f\n\t" \
wolfSSL	15:117db924cf7c	6022	"movq %[addt], %%rax\n\t" \
wolfSSL	15:117db924cf7c	6023	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6024	"cmpl $16, %%edx\n\t" \
wolfSSL	15:117db924cf7c	6025	"jl 24f\n\t" \
wolfSSL	15:117db924cf7c	6026	"andl $0xfffffff0, %%edx\n\t" \
wolfSSL	15:117db924cf7c	6027	"\n" \
wolfSSL	15:117db924cf7c	6028	"23:\n\t" \
wolfSSL	15:117db924cf7c	6029	"vmovdqu (%%rax,%%rcx,1), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	6030	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	6031	"vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	6032	GHASH_FULL_AVX2(XR, %%xmm12, XR, HR) \
wolfSSL	15:117db924cf7c	6033	"addl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6034	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6035	"jl 23b\n\t" \
wolfSSL	15:117db924cf7c	6036	"movl %[abytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	6037	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6038	"je 25f\n\t" \
wolfSSL	15:117db924cf7c	6039	"\n" \
wolfSSL	15:117db924cf7c	6040	"24:\n\t" \
wolfSSL	15:117db924cf7c	6041	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	6042	"vpxor %%xmm4, %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	6043	"xorl %%ebx, %%ebx\n\t" \
wolfSSL	15:117db924cf7c	6044	"vmovdqu %%xmm4, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	6045	"22:\n\t" \
wolfSSL	15:117db924cf7c	6046	"movzbl (%%rax,%%rcx,1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	6047	"movb %%r13b, (%%rsp,%%rbx,1)\n\t" \
wolfSSL	15:117db924cf7c	6048	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6049	"incl %%ebx\n\t" \
wolfSSL	15:117db924cf7c	6050	"cmpl %%edx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6051	"jl 22b\n\t" \
wolfSSL	15:117db924cf7c	6052	"vmovdqu (%%rsp), %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	6053	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	6054	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t" \
wolfSSL	15:117db924cf7c	6055	"vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	6056	GHASH_FULL_AVX2(XR, %%xmm12, XR, HR) \
wolfSSL	15:117db924cf7c	6057	"\n" \
wolfSSL	15:117db924cf7c	6058	"25:\n\t"
wolfSSL	15:117db924cf7c	6059
wolfSSL	15:117db924cf7c	6060	#define VAESENC_128_GHASH_AVX2(src, o) \
wolfSSL	15:117db924cf7c	6061	"leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t" \
wolfSSL	15:117db924cf7c	6062	"leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t" \
wolfSSL	15:117db924cf7c	6063	/* src is either %%rcx or %%rdx */ \
wolfSSL	15:117db924cf7c	6064	VAESENC_CTR() \
wolfSSL	15:117db924cf7c	6065	VAESENC_XOR() \
wolfSSL	15:117db924cf7c	6066	VAESENC_PCLMUL_AVX2_1(src, 16, (o-128), 112) \
wolfSSL	15:117db924cf7c	6067	VAESENC_PCLMUL_AVX2_2(src, 32, (o-112), 96) \
wolfSSL	15:117db924cf7c	6068	VAESENC_PCLMUL_AVX2_N(src, 48, (o- 96), 80) \
wolfSSL	15:117db924cf7c	6069	VAESENC_PCLMUL_AVX2_N(src, 64, (o- 80), 64) \
wolfSSL	15:117db924cf7c	6070	VAESENC_PCLMUL_AVX2_N(src, 80, (o- 64), 48) \
wolfSSL	15:117db924cf7c	6071	VAESENC_PCLMUL_AVX2_N(src, 96, (o- 48), 32) \
wolfSSL	15:117db924cf7c	6072	VAESENC_PCLMUL_AVX2_N(src, 112, (o- 32), 16) \
wolfSSL	15:117db924cf7c	6073	VAESENC_PCLMUL_AVX2_N(src, 128, (o- 16), 0) \
wolfSSL	15:117db924cf7c	6074	VAESENC_PCLMUL_AVX2_L(144) \
wolfSSL	15:117db924cf7c	6075	"cmpl $11, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	6076	"vmovdqa 160(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	6077	"jl 4f\n\t" \
wolfSSL	15:117db924cf7c	6078	VAESENC() \
wolfSSL	15:117db924cf7c	6079	VAESENC_SET(176) \
wolfSSL	15:117db924cf7c	6080	"cmpl $13, %[nr]\n\t" \
wolfSSL	15:117db924cf7c	6081	"vmovdqa 192(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	6082	"jl 4f\n\t" \
wolfSSL	15:117db924cf7c	6083	VAESENC() \
wolfSSL	15:117db924cf7c	6084	VAESENC_SET(208) \
wolfSSL	15:117db924cf7c	6085	"vmovdqa 224(%[KEY]), %%xmm12\n\t" \
wolfSSL	15:117db924cf7c	6086	"\n" \
wolfSSL	15:117db924cf7c	6087	"4:\n\t" \
wolfSSL	15:117db924cf7c	6088	VAESENC_LAST(%%rcx, %%rdx)
wolfSSL	15:117db924cf7c	6089
wolfSSL	15:117db924cf7c	6090	#define AESENC_LAST15_ENC_AVX2() \
wolfSSL	15:117db924cf7c	6091	"movl %[nbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6092	"movl %%ecx, %%edx\n\t" \
wolfSSL	15:117db924cf7c	6093	"andl $0x0f, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6094	"jz 55f\n\t" \
wolfSSL	15:117db924cf7c	6095	"vmovdqu " VAR(CTR1) ", %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6096	"vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6097	"vpxor 0(%[KEY]), %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6098	VAESENC_AVX(%%xmm13) \
wolfSSL	15:117db924cf7c	6099	"subq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	6100	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6101	"vmovdqu %%xmm13, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	6102	"\n" \
wolfSSL	15:117db924cf7c	6103	"51:\n\t" \
wolfSSL	15:117db924cf7c	6104	"movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	6105	"xorb (%%rsp,%%rcx,1), %%r13b\n\t" \
wolfSSL	15:117db924cf7c	6106	"movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \
wolfSSL	15:117db924cf7c	6107	"movb %%r13b, (%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	6108	"incl " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	6109	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6110	"cmpl %%edx, " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	6111	"jl 51b\n\t" \
wolfSSL	15:117db924cf7c	6112	"xorq %%r13, %%r13\n\t" \
wolfSSL	15:117db924cf7c	6113	"cmpl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6114	"je 53f\n\t" \
wolfSSL	15:117db924cf7c	6115	"\n" \
wolfSSL	15:117db924cf7c	6116	"52:\n\t" \
wolfSSL	15:117db924cf7c	6117	"movb %%r13b, (%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	6118	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6119	"cmpl $16, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6120	"jl 52b\n\t" \
wolfSSL	15:117db924cf7c	6121	"53:\n\t" \
wolfSSL	15:117db924cf7c	6122	"vmovdqu (%%rsp), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6123	"addq $16, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	6124	"vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6125	"vpxor %%xmm13, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	6126	GHASH_GFMUL_RED_AVX2(XR, HR, XR) \
wolfSSL	15:117db924cf7c	6127
wolfSSL	15:117db924cf7c	6128	#define AESENC_LAST15_DEC_AVX2() \
wolfSSL	15:117db924cf7c	6129	"movl %[nbytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6130	"movl %%ecx, %%edx\n\t" \
wolfSSL	15:117db924cf7c	6131	"andl $0x0f, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6132	"jz 55f\n\t" \
wolfSSL	15:117db924cf7c	6133	"vmovdqu " VAR(CTR1) ", %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6134	"vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6135	"vpxor 0(%[KEY]), %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6136	VAESENC_AVX(%%xmm13) \
wolfSSL	15:117db924cf7c	6137	"subq $32, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	6138	"xorl %%ecx, %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6139	"vmovdqu %%xmm13, (%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	6140	"vpxor %%xmm0, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	6141	"vmovdqu %%xmm0, 16(%%rsp)\n\t" \
wolfSSL	15:117db924cf7c	6142	"\n" \
wolfSSL	15:117db924cf7c	6143	"51:\n\t" \
wolfSSL	15:117db924cf7c	6144	"movzbl (%[in]," VAR(KR64) ",1), %%r13d\n\t" \
wolfSSL	15:117db924cf7c	6145	"movb %%r13b, 16(%%rsp,%%rcx,1)\n\t" \
wolfSSL	15:117db924cf7c	6146	"xorb (%%rsp,%%rcx,1), %%r13b\n\t" \
wolfSSL	15:117db924cf7c	6147	"movb %%r13b, (%[out]," VAR(KR64) ",1)\n\t" \
wolfSSL	15:117db924cf7c	6148	"incl " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	6149	"incl %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6150	"cmpl %%edx, " VAR(KR) "\n\t" \
wolfSSL	15:117db924cf7c	6151	"jl 51b\n\t" \
wolfSSL	15:117db924cf7c	6152	"53:\n\t" \
wolfSSL	15:117db924cf7c	6153	"vmovdqu 16(%%rsp), %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6154	"addq $32, %%rsp\n\t" \
wolfSSL	15:117db924cf7c	6155	"vpshufb %[BSWAP_MASK], %%xmm13, %%xmm13\n\t" \
wolfSSL	15:117db924cf7c	6156	"vpxor %%xmm13, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	6157	GHASH_GFMUL_RED_AVX2(XR, HR, XR) \
wolfSSL	15:117db924cf7c	6158
wolfSSL	15:117db924cf7c	6159	#define CALC_TAG_AVX2() \
wolfSSL	15:117db924cf7c	6160	"movl %[nbytes], %%edx\n\t" \
wolfSSL	15:117db924cf7c	6161	"movl %[abytes], %%ecx\n\t" \
wolfSSL	15:117db924cf7c	6162	"shlq $3, %%rdx\n\t" \
wolfSSL	15:117db924cf7c	6163	"shlq $3, %%rcx\n\t" \
wolfSSL	15:117db924cf7c	6164	"vpinsrq $0, %%rdx, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	6165	"vpinsrq $1, %%rcx, %%xmm0, %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	6166	"vpxor %%xmm0, " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	6167	GHASH_GFMUL_RED_AVX2(XR, HR, XR) \
wolfSSL	15:117db924cf7c	6168	"vpshufb %[BSWAP_MASK], " VAR(XR) ", " VAR(XR) "\n\t" \
wolfSSL	15:117db924cf7c	6169	"vpxor " VAR(TR) ", " VAR(XR) ", %%xmm0\n\t" \
wolfSSL	15:117db924cf7c	6170
wolfSSL	15:117db924cf7c	6171
wolfSSL	15:117db924cf7c	6172	static void AES_GCM_encrypt_avx2(const unsigned char in, unsigned char out,
wolfSSL	15:117db924cf7c	6173	const unsigned char* addt,
wolfSSL	15:117db924cf7c	6174	const unsigned char* ivec, unsigned char *tag,
wolfSSL	15:117db924cf7c	6175	unsigned int nbytes, unsigned int abytes,
wolfSSL	15:117db924cf7c	6176	unsigned int ibytes, unsigned int tbytes,
wolfSSL	15:117db924cf7c	6177	const unsigned char* key, int nr)
wolfSSL	15:117db924cf7c	6178	{
wolfSSL	15:117db924cf7c	6179	register const unsigned char* iv asm("rax") = ivec;
wolfSSL	15:117db924cf7c	6180	register unsigned int ivLen asm("ebx") = ibytes;
wolfSSL	15:117db924cf7c	6181
wolfSSL	15:117db924cf7c	6182	__asm__ __volatile__ (
wolfSSL	15:117db924cf7c	6183	"subq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	6184	/* Counter is xmm13 */
wolfSSL	15:117db924cf7c	6185	"vpxor %%xmm13, %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6186	"vpxor " VAR(XR) ", " VAR(XR) ", " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	6187	"movl %[ibytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	6188	"cmpl $12, %%edx\n\t"
wolfSSL	15:117db924cf7c	6189	"jne 35f\n\t"
wolfSSL	15:117db924cf7c	6190	CALC_IV_12_AVX2()
wolfSSL	15:117db924cf7c	6191	"jmp 39f\n\t"
wolfSSL	15:117db924cf7c	6192	"\n"
wolfSSL	15:117db924cf7c	6193	"35:\n\t"
wolfSSL	15:117db924cf7c	6194	CALC_IV_AVX2()
wolfSSL	15:117db924cf7c	6195	"\n"
wolfSSL	15:117db924cf7c	6196	"39:\n\t"
wolfSSL	15:117db924cf7c	6197
wolfSSL	15:117db924cf7c	6198	CALC_AAD_AVX2()
wolfSSL	15:117db924cf7c	6199
wolfSSL	15:117db924cf7c	6200	"# Calculate counter and H\n\t"
wolfSSL	15:117db924cf7c	6201	"vpsrlq $63, " VAR(HR) ", %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6202	"vpsllq $1, " VAR(HR) ", %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6203	"vpslldq $8, %%xmm5, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6204	"vpor %%xmm5, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6205	"vpshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6206	"vpsrad $31, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6207	"vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6208	"vpand %[MOD2_128], " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6209	"vpaddd %[ONE], %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6210	"vpxor %%xmm4, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6211	"vmovdqu %%xmm13, " VAR(CTR1) "\n\t"
wolfSSL	15:117db924cf7c	6212
wolfSSL	15:117db924cf7c	6213	"xorl " VAR(KR) ", " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6214
wolfSSL	15:117db924cf7c	6215	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL)
wolfSSL	15:117db924cf7c	6216	"cmpl $128, %[nbytes]\n\t"
wolfSSL	15:117db924cf7c	6217	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	6218	"jl 5f\n\t"
wolfSSL	15:117db924cf7c	6219	"andl $0xffffff80, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6220
wolfSSL	15:117db924cf7c	6221	CALC_HT_8_AVX2()
wolfSSL	15:117db924cf7c	6222
wolfSSL	15:117db924cf7c	6223	"# First 128 bytes of input\n\t"
wolfSSL	15:117db924cf7c	6224	VAESENC_128()
wolfSSL	15:117db924cf7c	6225
wolfSSL	15:117db924cf7c	6226	"cmpl $128, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6227	"movl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6228	"jle 2f\n\t"
wolfSSL	15:117db924cf7c	6229
wolfSSL	15:117db924cf7c	6230	"# More 128 bytes of input\n\t"
wolfSSL	15:117db924cf7c	6231	"\n"
wolfSSL	15:117db924cf7c	6232	"3:\n\t"
wolfSSL	15:117db924cf7c	6233	VAESENC_128_GHASH_AVX2(%%rdx, 0)
wolfSSL	15:117db924cf7c	6234	"addl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6235	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6236	"jl 3b\n\t"
wolfSSL	15:117db924cf7c	6237	"\n"
wolfSSL	15:117db924cf7c	6238	"2:\n\t"
wolfSSL	15:117db924cf7c	6239	"vmovdqa %[BSWAP_MASK], %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6240	"vpshufb %%xmm13, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6241	"vpshufb %%xmm13, %%xmm5, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6242	"vpshufb %%xmm13, %%xmm6, %%xmm6\n\t"
wolfSSL	15:117db924cf7c	6243	"vpshufb %%xmm13, %%xmm7, %%xmm7\n\t"
wolfSSL	15:117db924cf7c	6244	"vpshufb %%xmm13, %%xmm8, %%xmm8\n\t"
wolfSSL	15:117db924cf7c	6245	"vpshufb %%xmm13, %%xmm9, %%xmm9\n\t"
wolfSSL	15:117db924cf7c	6246	"vpshufb %%xmm13, %%xmm10, %%xmm10\n\t"
wolfSSL	15:117db924cf7c	6247	"vpshufb %%xmm13, %%xmm11, %%xmm11\n\t"
wolfSSL	15:117db924cf7c	6248	"vpxor %%xmm2, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6249
wolfSSL	15:117db924cf7c	6250	GHASH_GFMUL_RED_8_AVX2()
wolfSSL	15:117db924cf7c	6251
wolfSSL	15:117db924cf7c	6252	"vmovdqu 0(" VAR(HTR) "), " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6253	"\n"
wolfSSL	15:117db924cf7c	6254	"5:\n\t"
wolfSSL	15:117db924cf7c	6255	"movl %[nbytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	6256	"cmpl %%edx, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6257	"jge 55f\n\t"
wolfSSL	15:117db924cf7c	6258	#endif
wolfSSL	15:117db924cf7c	6259
wolfSSL	15:117db924cf7c	6260	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	6261	"andl $0xfffffff0, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6262	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6263	"jge 14f\n\t"
wolfSSL	15:117db924cf7c	6264
wolfSSL	15:117db924cf7c	6265	VAESENC_BLOCK_AVX2()
wolfSSL	15:117db924cf7c	6266	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6267	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6268	"jge 13f\n\t"
wolfSSL	15:117db924cf7c	6269	"vmovdqa %[MOD2_128], %%xmm0\n\t"
wolfSSL	15:117db924cf7c	6270	"\n"
wolfSSL	15:117db924cf7c	6271	"12:\n\t"
wolfSSL	15:117db924cf7c	6272	"vmovdqu (%[in]," VAR(KR64) ",1), %%xmm9\n\t"
wolfSSL	15:117db924cf7c	6273	"vmovdqu " VAR(CTR1) ", %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6274	"vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6275	"vpaddd %[ONE], %%xmm5, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6276	"vmovdqu %%xmm5, " VAR(CTR1) "\n\t"
wolfSSL	15:117db924cf7c	6277	VAESENC_GFMUL_SB_AVX2(%%xmm9, HR, XR, CTR1)
wolfSSL	15:117db924cf7c	6278	"vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t"
wolfSSL	15:117db924cf7c	6279	"vpshufb %[BSWAP_MASK], %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6280	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6281	"vpxor %%xmm4, " VAR(XR) ", " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	6282	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6283	"jl 12b\n\t"
wolfSSL	15:117db924cf7c	6284	"\n"
wolfSSL	15:117db924cf7c	6285	"13:\n\t"
wolfSSL	15:117db924cf7c	6286	GHASH_GFMUL_RED_AVX2(XR, HR, XR)
wolfSSL	15:117db924cf7c	6287	"\n"
wolfSSL	15:117db924cf7c	6288	"14:\n\t"
wolfSSL	15:117db924cf7c	6289
wolfSSL	15:117db924cf7c	6290	AESENC_LAST15_ENC_AVX2()
wolfSSL	15:117db924cf7c	6291	"\n"
wolfSSL	15:117db924cf7c	6292	"55:\n\t"
wolfSSL	15:117db924cf7c	6293
wolfSSL	15:117db924cf7c	6294	CALC_TAG_AVX2()
wolfSSL	15:117db924cf7c	6295	STORE_TAG_AVX()
wolfSSL	15:117db924cf7c	6296	"addq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	6297	"vzeroupper\n\t"
wolfSSL	15:117db924cf7c	6298
wolfSSL	15:117db924cf7c	6299	:
wolfSSL	15:117db924cf7c	6300	: [KEY] "r" (key),
wolfSSL	15:117db924cf7c	6301	[in] "r" (in), [out] "r" (out), [nr] "r" (nr),
wolfSSL	15:117db924cf7c	6302	[nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt),
wolfSSL	15:117db924cf7c	6303	[ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tbytes),
wolfSSL	15:117db924cf7c	6304	[tag] "r" (tag),
wolfSSL	15:117db924cf7c	6305	[BSWAP_MASK] "m" (BSWAP_MASK),
wolfSSL	15:117db924cf7c	6306	[BSWAP_EPI64] "m" (BSWAP_EPI64),
wolfSSL	15:117db924cf7c	6307	[ONE] "m" (ONE),
wolfSSL	15:117db924cf7c	6308	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL)
wolfSSL	15:117db924cf7c	6309	[TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR),
wolfSSL	15:117db924cf7c	6310	[FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN),
wolfSSL	15:117db924cf7c	6311	[EIGHT] "m" (EIGHT),
wolfSSL	15:117db924cf7c	6312	#endif
wolfSSL	15:117db924cf7c	6313	[MOD2_128] "m" (MOD2_128)
wolfSSL	15:117db924cf7c	6314	: "xmm15", "xmm14", "xmm13", "xmm12",
wolfSSL	15:117db924cf7c	6315	"xmm0", "xmm1", "xmm2", "xmm3", "memory",
wolfSSL	15:117db924cf7c	6316	"xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
wolfSSL	15:117db924cf7c	6317	"rcx", "rdx", "r13"
wolfSSL	15:117db924cf7c	6318	);
wolfSSL	15:117db924cf7c	6319	}
wolfSSL	15:117db924cf7c	6320	#endif /* HAVE_INTEL_AVX2 */
wolfSSL	15:117db924cf7c	6321	#endif /* HAVE_INTEL_AVX1 */
wolfSSL	15:117db924cf7c	6322
wolfSSL	15:117db924cf7c	6323	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	6324	/* Figure 10. AES-GCM – Decrypt With Single Block Ghash at a Time */
wolfSSL	15:117db924cf7c	6325
wolfSSL	15:117db924cf7c	6326	static void AES_GCM_decrypt(const unsigned char in, unsigned char out,
wolfSSL	15:117db924cf7c	6327	const unsigned char* addt,
wolfSSL	15:117db924cf7c	6328	const unsigned char* ivec, const unsigned char *tag,
wolfSSL	15:117db924cf7c	6329	int nbytes, int abytes, int ibytes, int tbytes,
wolfSSL	15:117db924cf7c	6330	const unsigned char* key, int nr, int* res)
wolfSSL	15:117db924cf7c	6331	{
wolfSSL	15:117db924cf7c	6332	register const unsigned char* iv asm("rax") = ivec;
wolfSSL	15:117db924cf7c	6333	register int ivLen asm("ebx") = ibytes;
wolfSSL	15:117db924cf7c	6334	register int tagLen asm("edx") = tbytes;
wolfSSL	15:117db924cf7c	6335
wolfSSL	15:117db924cf7c	6336	__asm__ __volatile__ (
wolfSSL	15:117db924cf7c	6337	"pushq %%rdx\n\t"
wolfSSL	15:117db924cf7c	6338	"subq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	6339	/* Counter is xmm13 */
wolfSSL	15:117db924cf7c	6340	"pxor %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6341	"pxor %%xmm15, %%xmm15\n\t"
wolfSSL	15:117db924cf7c	6342	"movl %[ibytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	6343	"cmpl $12, %%edx\n\t"
wolfSSL	15:117db924cf7c	6344	"jne 35f\n\t"
wolfSSL	15:117db924cf7c	6345	CALC_IV_12()
wolfSSL	15:117db924cf7c	6346	"\n"
wolfSSL	15:117db924cf7c	6347	"35:\n\t"
wolfSSL	15:117db924cf7c	6348	CALC_IV()
wolfSSL	15:117db924cf7c	6349	"\n"
wolfSSL	15:117db924cf7c	6350	"39:\n\t"
wolfSSL	15:117db924cf7c	6351
wolfSSL	15:117db924cf7c	6352	CALC_AAD()
wolfSSL	15:117db924cf7c	6353
wolfSSL	15:117db924cf7c	6354	"# Calculate counter and H\n\t"
wolfSSL	15:117db924cf7c	6355	"pshufb %[BSWAP_EPI64], %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6356	"movdqa " VAR(HR) ", %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6357	"paddd %[ONE], %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6358	"movdqa " VAR(HR) ", %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6359	"movdqu %%xmm13, " VAR(CTR1) "\n\t"
wolfSSL	15:117db924cf7c	6360	"psrlq $63, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6361	"psllq $1, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6362	"pslldq $8, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6363	"por %%xmm5, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6364	"pshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6365	"psrad $31, " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6366	"pand %[MOD2_128], " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6367	"pxor %%xmm4, " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6368
wolfSSL	15:117db924cf7c	6369	"xorl " VAR(KR) ", " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6370
wolfSSL	15:117db924cf7c	6371	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	6372	"cmpl $128, %[nbytes]\n\t"
wolfSSL	15:117db924cf7c	6373	"jl 5f\n\t"
wolfSSL	15:117db924cf7c	6374
wolfSSL	15:117db924cf7c	6375	CALC_HT_8_AVX()
wolfSSL	15:117db924cf7c	6376
wolfSSL	15:117db924cf7c	6377	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	6378	"andl $0xffffff80, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6379	"\n"
wolfSSL	15:117db924cf7c	6380	"2:\n\t"
wolfSSL	15:117db924cf7c	6381	AESENC_128_GHASH_AVX(%%rcx, 128)
wolfSSL	15:117db924cf7c	6382	"addl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6383	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6384	"jl 2b\n\t"
wolfSSL	15:117db924cf7c	6385
wolfSSL	15:117db924cf7c	6386	"movdqa %%xmm2, " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	6387	"movdqu (%%rsp), " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6388	"5:\n\t"
wolfSSL	15:117db924cf7c	6389	"movl %[nbytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	6390	"cmpl %%edx, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6391	"jge 55f\n\t"
wolfSSL	15:117db924cf7c	6392	#endif
wolfSSL	15:117db924cf7c	6393	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	6394	"andl $0xfffffff0, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6395	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6396	"jge 13f\n\t"
wolfSSL	15:117db924cf7c	6397
wolfSSL	15:117db924cf7c	6398	"\n"
wolfSSL	15:117db924cf7c	6399	"12:\n\t"
wolfSSL	15:117db924cf7c	6400	"leaq (%[in]," VAR(KR64) ",1), %%rcx\n\t"
wolfSSL	15:117db924cf7c	6401	"leaq (%[out]," VAR(KR64) ",1), %%rdx\n\t"
wolfSSL	15:117db924cf7c	6402	"movdqu (%%rcx), %%xmm1\n\t"
wolfSSL	15:117db924cf7c	6403	"movdqa " VAR(HR) ", %%xmm0\n\t"
wolfSSL	15:117db924cf7c	6404	"pshufb %[BSWAP_MASK], %%xmm1\n\t"
wolfSSL	15:117db924cf7c	6405	"pxor " VAR(XR) ", %%xmm1\n\t"
wolfSSL	15:117db924cf7c	6406	AESENC_GFMUL(%%rcx, %%rdx, %%xmm0, %%xmm1)
wolfSSL	15:117db924cf7c	6407	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6408	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6409	"jl 12b\n\t"
wolfSSL	15:117db924cf7c	6410	"\n"
wolfSSL	15:117db924cf7c	6411	"13:\n\t"
wolfSSL	15:117db924cf7c	6412
wolfSSL	15:117db924cf7c	6413	AESENC_LAST15_DEC_AVX()
wolfSSL	15:117db924cf7c	6414	"\n"
wolfSSL	15:117db924cf7c	6415	"55:\n\t"
wolfSSL	15:117db924cf7c	6416
wolfSSL	15:117db924cf7c	6417	CALC_TAG()
wolfSSL	15:117db924cf7c	6418	"addq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	6419	"popq %%rdx\n\t"
wolfSSL	15:117db924cf7c	6420	CMP_TAG()
wolfSSL	15:117db924cf7c	6421
wolfSSL	15:117db924cf7c	6422	:
wolfSSL	15:117db924cf7c	6423	: [KEY] "r" (key),
wolfSSL	15:117db924cf7c	6424	[in] "r" (in), [out] "r" (out), [nr] "r" (nr),
wolfSSL	15:117db924cf7c	6425	[nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt),
wolfSSL	15:117db924cf7c	6426	[ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tagLen),
wolfSSL	15:117db924cf7c	6427	[tag] "r" (tag), [res] "r" (res),
wolfSSL	15:117db924cf7c	6428	[BSWAP_MASK] "m" (BSWAP_MASK),
wolfSSL	15:117db924cf7c	6429	[BSWAP_EPI64] "m" (BSWAP_EPI64),
wolfSSL	15:117db924cf7c	6430	[ONE] "m" (ONE),
wolfSSL	15:117db924cf7c	6431	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	6432	[TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR),
wolfSSL	15:117db924cf7c	6433	[FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN),
wolfSSL	15:117db924cf7c	6434	[EIGHT] "m" (EIGHT),
wolfSSL	15:117db924cf7c	6435	#endif
wolfSSL	15:117db924cf7c	6436	[MOD2_128] "m" (MOD2_128)
wolfSSL	15:117db924cf7c	6437	: "xmm15", "xmm14", "xmm13", "xmm12",
wolfSSL	15:117db924cf7c	6438	"xmm0", "xmm1", "xmm2", "xmm3", "memory",
wolfSSL	15:117db924cf7c	6439	"xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
wolfSSL	15:117db924cf7c	6440	"rcx", "r13"
wolfSSL	15:117db924cf7c	6441	);
wolfSSL	15:117db924cf7c	6442	}
wolfSSL	15:117db924cf7c	6443
wolfSSL	15:117db924cf7c	6444	#ifdef HAVE_INTEL_AVX1
wolfSSL	15:117db924cf7c	6445	static void AES_GCM_decrypt_avx1(const unsigned char in, unsigned char out,
wolfSSL	15:117db924cf7c	6446	const unsigned char* addt,
wolfSSL	15:117db924cf7c	6447	const unsigned char* ivec,
wolfSSL	15:117db924cf7c	6448	const unsigned char *tag, int nbytes,
wolfSSL	15:117db924cf7c	6449	int abytes, int ibytes, int tbytes,
wolfSSL	15:117db924cf7c	6450	const unsigned char* key, int nr, int* res)
wolfSSL	15:117db924cf7c	6451	{
wolfSSL	15:117db924cf7c	6452	register const unsigned char* iv asm("rax") = ivec;
wolfSSL	15:117db924cf7c	6453	register int ivLen asm("ebx") = ibytes;
wolfSSL	15:117db924cf7c	6454	register int tagLen asm("edx") = tbytes;
wolfSSL	15:117db924cf7c	6455
wolfSSL	15:117db924cf7c	6456	__asm__ __volatile__ (
wolfSSL	15:117db924cf7c	6457	"pushq %%rdx\n\t"
wolfSSL	15:117db924cf7c	6458	"subq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	6459	/* Counter is xmm13 */
wolfSSL	15:117db924cf7c	6460	"vpxor %%xmm13, %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6461	"vpxor %%xmm15, %%xmm15, %%xmm15\n\t"
wolfSSL	15:117db924cf7c	6462	"movl %[ibytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	6463	"cmpl $12, %%edx\n\t"
wolfSSL	15:117db924cf7c	6464	"jne 35f\n\t"
wolfSSL	15:117db924cf7c	6465	CALC_IV_12_AVX1()
wolfSSL	15:117db924cf7c	6466	"\n"
wolfSSL	15:117db924cf7c	6467	"35:\n\t"
wolfSSL	15:117db924cf7c	6468	CALC_IV_AVX1()
wolfSSL	15:117db924cf7c	6469	"\n"
wolfSSL	15:117db924cf7c	6470	"39:\n\t"
wolfSSL	15:117db924cf7c	6471
wolfSSL	15:117db924cf7c	6472	CALC_AAD_AVX1()
wolfSSL	15:117db924cf7c	6473
wolfSSL	15:117db924cf7c	6474	"# Calculate counter and H\n\t"
wolfSSL	15:117db924cf7c	6475	"vpsrlq $63, " VAR(HR) ", %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6476	"vpsllq $1, " VAR(HR) ", %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6477	"vpslldq $8, %%xmm5, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6478	"vpor %%xmm5, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6479	"vpshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6480	"vpsrad $31, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6481	"vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6482	"vpand %[MOD2_128], " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6483	"vpaddd %[ONE], %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6484	"vpxor %%xmm4, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6485	"vmovdqu %%xmm13, " VAR(CTR1) "\n\t"
wolfSSL	15:117db924cf7c	6486
wolfSSL	15:117db924cf7c	6487	"xorl " VAR(KR) ", " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6488
wolfSSL	15:117db924cf7c	6489	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	6490	"cmpl $128, %[nbytes]\n\t"
wolfSSL	15:117db924cf7c	6491	"jl 5f\n\t"
wolfSSL	15:117db924cf7c	6492
wolfSSL	15:117db924cf7c	6493	CALC_HT_8_AVX1()
wolfSSL	15:117db924cf7c	6494
wolfSSL	15:117db924cf7c	6495	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	6496	"andl $0xffffff80, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6497	"\n"
wolfSSL	15:117db924cf7c	6498	"2:\n\t"
wolfSSL	15:117db924cf7c	6499	VAESENC_128_GHASH_AVX1(%%rcx, 128)
wolfSSL	15:117db924cf7c	6500	"addl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6501	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6502	"jl 2b\n\t"
wolfSSL	15:117db924cf7c	6503
wolfSSL	15:117db924cf7c	6504	"vmovdqa %%xmm2, " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	6505	"vmovdqu (%%rsp), " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6506	"5:\n\t"
wolfSSL	15:117db924cf7c	6507	"movl %[nbytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	6508	"cmpl %%edx, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6509	"jge 55f\n\t"
wolfSSL	15:117db924cf7c	6510	#endif
wolfSSL	15:117db924cf7c	6511	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	6512	"andl $0xfffffff0, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6513	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6514	"jge 13f\n\t"
wolfSSL	15:117db924cf7c	6515
wolfSSL	15:117db924cf7c	6516	"\n"
wolfSSL	15:117db924cf7c	6517	"12:\n\t"
wolfSSL	15:117db924cf7c	6518	"vmovdqu (%[in]," VAR(KR64) ",1), %%xmm9\n\t"
wolfSSL	15:117db924cf7c	6519	"vmovdqa " VAR(HR) ", %%xmm0\n\t"
wolfSSL	15:117db924cf7c	6520	"vpshufb %[BSWAP_MASK], %%xmm9, %%xmm1\n\t"
wolfSSL	15:117db924cf7c	6521	"vpxor " VAR(XR) ", %%xmm1, %%xmm1\n\t"
wolfSSL	15:117db924cf7c	6522	VAESENC_GFMUL(%%xmm9, %%xmm0, %%xmm1)
wolfSSL	15:117db924cf7c	6523	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6524	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6525	"jl 12b\n\t"
wolfSSL	15:117db924cf7c	6526	"\n"
wolfSSL	15:117db924cf7c	6527	"13:\n\t"
wolfSSL	15:117db924cf7c	6528
wolfSSL	15:117db924cf7c	6529	AESENC_LAST15_DEC_AVX1()
wolfSSL	15:117db924cf7c	6530	"\n"
wolfSSL	15:117db924cf7c	6531	"55:\n\t"
wolfSSL	15:117db924cf7c	6532
wolfSSL	15:117db924cf7c	6533	CALC_TAG_AVX1()
wolfSSL	15:117db924cf7c	6534	"addq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	6535	"popq %%rdx\n\t"
wolfSSL	15:117db924cf7c	6536	CMP_TAG_AVX()
wolfSSL	15:117db924cf7c	6537	"vzeroupper\n\t"
wolfSSL	15:117db924cf7c	6538
wolfSSL	15:117db924cf7c	6539	:
wolfSSL	15:117db924cf7c	6540	: [KEY] "r" (key),
wolfSSL	15:117db924cf7c	6541	[in] "r" (in), [out] "r" (out), [nr] "r" (nr),
wolfSSL	15:117db924cf7c	6542	[nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt),
wolfSSL	15:117db924cf7c	6543	[ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tagLen),
wolfSSL	15:117db924cf7c	6544	[tag] "r" (tag), [res] "r" (res),
wolfSSL	15:117db924cf7c	6545	[BSWAP_MASK] "m" (BSWAP_MASK),
wolfSSL	15:117db924cf7c	6546	[BSWAP_EPI64] "m" (BSWAP_EPI64),
wolfSSL	15:117db924cf7c	6547	[ONE] "m" (ONE),
wolfSSL	15:117db924cf7c	6548	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX1_NO_UNROLL)
wolfSSL	15:117db924cf7c	6549	[TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR),
wolfSSL	15:117db924cf7c	6550	[FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN),
wolfSSL	15:117db924cf7c	6551	[EIGHT] "m" (EIGHT),
wolfSSL	15:117db924cf7c	6552	#endif
wolfSSL	15:117db924cf7c	6553	[MOD2_128] "m" (MOD2_128)
wolfSSL	15:117db924cf7c	6554	: "xmm15", "xmm14", "xmm13", "xmm12",
wolfSSL	15:117db924cf7c	6555	"xmm0", "xmm1", "xmm2", "xmm3", "memory",
wolfSSL	15:117db924cf7c	6556	"xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
wolfSSL	15:117db924cf7c	6557	"rcx", "r13"
wolfSSL	15:117db924cf7c	6558	);
wolfSSL	15:117db924cf7c	6559	}
wolfSSL	15:117db924cf7c	6560
wolfSSL	15:117db924cf7c	6561	#ifdef HAVE_INTEL_AVX2
wolfSSL	15:117db924cf7c	6562	static void AES_GCM_decrypt_avx2(const unsigned char in, unsigned char out,
wolfSSL	15:117db924cf7c	6563	const unsigned char* addt,
wolfSSL	15:117db924cf7c	6564	const unsigned char* ivec,
wolfSSL	15:117db924cf7c	6565	const unsigned char *tag, int nbytes,
wolfSSL	15:117db924cf7c	6566	int abytes, int ibytes, int tbytes,
wolfSSL	15:117db924cf7c	6567	const unsigned char* key, int nr, int* res)
wolfSSL	15:117db924cf7c	6568	{
wolfSSL	15:117db924cf7c	6569	register const unsigned char* iv asm("rax") = ivec;
wolfSSL	15:117db924cf7c	6570	register int ivLen asm("ebx") = ibytes;
wolfSSL	15:117db924cf7c	6571	register int tagLen asm("edx") = tbytes;
wolfSSL	15:117db924cf7c	6572
wolfSSL	15:117db924cf7c	6573	__asm__ __volatile__ (
wolfSSL	15:117db924cf7c	6574	"pushq %%rdx\n\t"
wolfSSL	15:117db924cf7c	6575	"subq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	6576	/* Counter is xmm13 */
wolfSSL	15:117db924cf7c	6577	"vpxor %%xmm13, %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6578	"vpxor %%xmm15, %%xmm15, %%xmm15\n\t"
wolfSSL	15:117db924cf7c	6579	"movl %[ibytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	6580	"cmpl $12, %%edx\n\t"
wolfSSL	15:117db924cf7c	6581	"jne 35f\n\t"
wolfSSL	15:117db924cf7c	6582	CALC_IV_12_AVX2()
wolfSSL	15:117db924cf7c	6583	"jmp 39f\n\t"
wolfSSL	15:117db924cf7c	6584	"\n"
wolfSSL	15:117db924cf7c	6585	"35:\n\t"
wolfSSL	15:117db924cf7c	6586	CALC_IV_AVX2()
wolfSSL	15:117db924cf7c	6587	"\n"
wolfSSL	15:117db924cf7c	6588	"39:\n\t"
wolfSSL	15:117db924cf7c	6589
wolfSSL	15:117db924cf7c	6590	CALC_AAD_AVX2()
wolfSSL	15:117db924cf7c	6591
wolfSSL	15:117db924cf7c	6592	"# Calculate counter and H\n\t"
wolfSSL	15:117db924cf7c	6593	"vpsrlq $63, " VAR(HR) ", %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6594	"vpsllq $1, " VAR(HR) ", %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6595	"vpslldq $8, %%xmm5, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6596	"vpor %%xmm5, %%xmm4, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6597	"vpshufd $0xff, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6598	"vpsrad $31, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6599	"vpshufb %[BSWAP_EPI64], %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6600	"vpand %[MOD2_128], " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6601	"vpaddd %[ONE], %%xmm13, %%xmm13\n\t"
wolfSSL	15:117db924cf7c	6602	"vpxor %%xmm4, " VAR(HR) ", " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6603	"vmovdqu %%xmm13, " VAR(CTR1) "\n\t"
wolfSSL	15:117db924cf7c	6604
wolfSSL	15:117db924cf7c	6605	"xorl " VAR(KR) ", " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6606
wolfSSL	15:117db924cf7c	6607	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL)
wolfSSL	15:117db924cf7c	6608	"cmpl $128, %[nbytes]\n\t"
wolfSSL	15:117db924cf7c	6609	"jl 5f\n\t"
wolfSSL	15:117db924cf7c	6610
wolfSSL	15:117db924cf7c	6611	CALC_HT_8_AVX2()
wolfSSL	15:117db924cf7c	6612
wolfSSL	15:117db924cf7c	6613	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	6614	"andl $0xffffff80, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6615	"\n"
wolfSSL	15:117db924cf7c	6616	"2:\n\t"
wolfSSL	15:117db924cf7c	6617	VAESENC_128_GHASH_AVX2(%%rcx, 128)
wolfSSL	15:117db924cf7c	6618	"addl $128, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6619	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6620	"jl 2b\n\t"
wolfSSL	15:117db924cf7c	6621
wolfSSL	15:117db924cf7c	6622	"vmovdqa %%xmm2, " VAR(XR) "\n\t"
wolfSSL	15:117db924cf7c	6623	"vmovdqu (%%rsp), " VAR(HR) "\n\t"
wolfSSL	15:117db924cf7c	6624	"5:\n\t"
wolfSSL	15:117db924cf7c	6625	"movl %[nbytes], %%edx\n\t"
wolfSSL	15:117db924cf7c	6626	"cmpl %%edx, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6627	"jge 55f\n\t"
wolfSSL	15:117db924cf7c	6628	#endif
wolfSSL	15:117db924cf7c	6629	"movl %[nbytes], %%r13d\n\t"
wolfSSL	15:117db924cf7c	6630	"andl $0xfffffff0, %%r13d\n\t"
wolfSSL	15:117db924cf7c	6631	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6632	"jge 13f\n\t"
wolfSSL	15:117db924cf7c	6633
wolfSSL	15:117db924cf7c	6634	"vmovdqa %[MOD2_128], %%xmm0\n\t"
wolfSSL	15:117db924cf7c	6635	"\n"
wolfSSL	15:117db924cf7c	6636	"12:\n\t"
wolfSSL	15:117db924cf7c	6637	"vmovdqu (%[in]," VAR(KR64) ",1), %%xmm9\n\t"
wolfSSL	15:117db924cf7c	6638	"vmovdqu " VAR(CTR1) ", %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6639	"vpshufb %[BSWAP_MASK], %%xmm9, %%xmm1\n\t"
wolfSSL	15:117db924cf7c	6640	"vpshufb %[BSWAP_EPI64], %%xmm5, %%xmm4\n\t"
wolfSSL	15:117db924cf7c	6641	"vpaddd %[ONE], %%xmm5, %%xmm5\n\t"
wolfSSL	15:117db924cf7c	6642	"vpxor " VAR(XR) ", %%xmm1, %%xmm1\n\t"
wolfSSL	15:117db924cf7c	6643	"vmovdqu %%xmm5, " VAR(CTR1) "\n\t"
wolfSSL	15:117db924cf7c	6644	VAESENC_GFMUL_SB_AVX2(%%xmm9, HR, %%xmm1, CTR1)
wolfSSL	15:117db924cf7c	6645	"vmovdqu %%xmm4, (%[out]," VAR(KR64) ",1)\n\t"
wolfSSL	15:117db924cf7c	6646	"addl $16, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6647	"cmpl %%r13d, " VAR(KR) "\n\t"
wolfSSL	15:117db924cf7c	6648	"jl 12b\n\t"
wolfSSL	15:117db924cf7c	6649	"\n"
wolfSSL	15:117db924cf7c	6650	"13:\n\t"
wolfSSL	15:117db924cf7c	6651
wolfSSL	15:117db924cf7c	6652	AESENC_LAST15_DEC_AVX2()
wolfSSL	15:117db924cf7c	6653	"\n"
wolfSSL	15:117db924cf7c	6654	"55:\n\t"
wolfSSL	15:117db924cf7c	6655
wolfSSL	15:117db924cf7c	6656	CALC_TAG_AVX2()
wolfSSL	15:117db924cf7c	6657	"addq $" VAR(STACK_OFFSET) ", %%rsp\n\t"
wolfSSL	15:117db924cf7c	6658	"popq %%rdx\n\t"
wolfSSL	15:117db924cf7c	6659	CMP_TAG_AVX()
wolfSSL	15:117db924cf7c	6660	"vzeroupper\n\t"
wolfSSL	15:117db924cf7c	6661
wolfSSL	15:117db924cf7c	6662	:
wolfSSL	15:117db924cf7c	6663	: [KEY] "r" (key),
wolfSSL	15:117db924cf7c	6664	[in] "r" (in), [out] "r" (out), [nr] "r" (nr),
wolfSSL	15:117db924cf7c	6665	[nbytes] "r" (nbytes), [abytes] "r" (abytes), [addt] "r" (addt),
wolfSSL	15:117db924cf7c	6666	[ivec] "r" (iv), [ibytes] "r" (ivLen), [tbytes] "r" (tagLen),
wolfSSL	15:117db924cf7c	6667	[tag] "r" (tag), [res] "r" (res),
wolfSSL	15:117db924cf7c	6668	[BSWAP_MASK] "m" (BSWAP_MASK),
wolfSSL	15:117db924cf7c	6669	[BSWAP_EPI64] "m" (BSWAP_EPI64),
wolfSSL	15:117db924cf7c	6670	[ONE] "m" (ONE),
wolfSSL	15:117db924cf7c	6671	#if !defined(AES_GCM_AESNI_NO_UNROLL) && !defined(AES_GCM_AVX2_NO_UNROLL)
wolfSSL	15:117db924cf7c	6672	[TWO] "m" (TWO), [THREE] "m" (THREE), [FOUR] "m" (FOUR),
wolfSSL	15:117db924cf7c	6673	[FIVE] "m" (FIVE), [SIX] "m" (SIX), [SEVEN] "m" (SEVEN),
wolfSSL	15:117db924cf7c	6674	[EIGHT] "m" (EIGHT),
wolfSSL	15:117db924cf7c	6675	#endif
wolfSSL	15:117db924cf7c	6676	[MOD2_128] "m" (MOD2_128)
wolfSSL	15:117db924cf7c	6677	: "xmm15", "xmm14", "xmm13", "xmm12",
wolfSSL	15:117db924cf7c	6678	"xmm0", "xmm1", "xmm2", "xmm3", "memory",
wolfSSL	15:117db924cf7c	6679	"xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11",
wolfSSL	15:117db924cf7c	6680	"rcx", "r13"
wolfSSL	15:117db924cf7c	6681	);
wolfSSL	15:117db924cf7c	6682	}
wolfSSL	15:117db924cf7c	6683	#endif /* HAVE_INTEL_AVX2 */
wolfSSL	15:117db924cf7c	6684	#endif /* HAVE_INTEL_AVX1 */
wolfSSL	15:117db924cf7c	6685	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	6686
wolfSSL	15:117db924cf7c	6687	#else /* _MSC_VER */
wolfSSL	15:117db924cf7c	6688	/* The following are for MSC based builds which do not allow
wolfSSL	15:117db924cf7c	6689	* inline assembly. Intrinsic functions are used instead. */
wolfSSL	15:117db924cf7c	6690
wolfSSL	15:117db924cf7c	6691	#define aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T) \
wolfSSL	15:117db924cf7c	6692	do \
wolfSSL	15:117db924cf7c	6693	{ \
wolfSSL	15:117db924cf7c	6694	word32 iv12[4]; \
wolfSSL	15:117db924cf7c	6695	iv12[0] = (word32)&ivec[0]; \
wolfSSL	15:117db924cf7c	6696	iv12[1] = (word32)&ivec[4]; \
wolfSSL	15:117db924cf7c	6697	iv12[2] = (word32)&ivec[8]; \
wolfSSL	15:117db924cf7c	6698	iv12[3] = 0x01000000; \
wolfSSL	15:117db924cf7c	6699	Y = _mm_loadu_si128((__m128i*)iv12); \
wolfSSL	15:117db924cf7c	6700	\
wolfSSL	15:117db924cf7c	6701	/* (Compute E[ZERO, KS] and E[Y0, KS] together */ \
wolfSSL	15:117db924cf7c	6702	tmp1 = _mm_load_si128(&KEY[0]); \
wolfSSL	15:117db924cf7c	6703	tmp2 = _mm_xor_si128(Y, KEY[0]); \
wolfSSL	15:117db924cf7c	6704	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \
wolfSSL	15:117db924cf7c	6705	tmp2 = _mm_aesenc_si128(tmp2, KEY[1]); \
wolfSSL	15:117db924cf7c	6706	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \
wolfSSL	15:117db924cf7c	6707	tmp2 = _mm_aesenc_si128(tmp2, KEY[2]); \
wolfSSL	15:117db924cf7c	6708	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \
wolfSSL	15:117db924cf7c	6709	tmp2 = _mm_aesenc_si128(tmp2, KEY[3]); \
wolfSSL	15:117db924cf7c	6710	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \
wolfSSL	15:117db924cf7c	6711	tmp2 = _mm_aesenc_si128(tmp2, KEY[4]); \
wolfSSL	15:117db924cf7c	6712	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \
wolfSSL	15:117db924cf7c	6713	tmp2 = _mm_aesenc_si128(tmp2, KEY[5]); \
wolfSSL	15:117db924cf7c	6714	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \
wolfSSL	15:117db924cf7c	6715	tmp2 = _mm_aesenc_si128(tmp2, KEY[6]); \
wolfSSL	15:117db924cf7c	6716	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \
wolfSSL	15:117db924cf7c	6717	tmp2 = _mm_aesenc_si128(tmp2, KEY[7]); \
wolfSSL	15:117db924cf7c	6718	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \
wolfSSL	15:117db924cf7c	6719	tmp2 = _mm_aesenc_si128(tmp2, KEY[8]); \
wolfSSL	15:117db924cf7c	6720	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \
wolfSSL	15:117db924cf7c	6721	tmp2 = _mm_aesenc_si128(tmp2, KEY[9]); \
wolfSSL	15:117db924cf7c	6722	lastKey = KEY[10]; \
wolfSSL	15:117db924cf7c	6723	if (nr > 10) { \
wolfSSL	15:117db924cf7c	6724	tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6725	tmp2 = _mm_aesenc_si128(tmp2, lastKey); \
wolfSSL	15:117db924cf7c	6726	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \
wolfSSL	15:117db924cf7c	6727	tmp2 = _mm_aesenc_si128(tmp2, KEY[11]); \
wolfSSL	15:117db924cf7c	6728	lastKey = KEY[12]; \
wolfSSL	15:117db924cf7c	6729	if (nr > 12) { \
wolfSSL	15:117db924cf7c	6730	tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6731	tmp2 = _mm_aesenc_si128(tmp2, lastKey); \
wolfSSL	15:117db924cf7c	6732	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \
wolfSSL	15:117db924cf7c	6733	tmp2 = _mm_aesenc_si128(tmp2, KEY[13]); \
wolfSSL	15:117db924cf7c	6734	lastKey = KEY[14]; \
wolfSSL	15:117db924cf7c	6735	} \
wolfSSL	15:117db924cf7c	6736	} \
wolfSSL	15:117db924cf7c	6737	H = _mm_aesenclast_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6738	T = _mm_aesenclast_si128(tmp2, lastKey); \
wolfSSL	15:117db924cf7c	6739	H = _mm_shuffle_epi8(H, BSWAP_MASK); \
wolfSSL	15:117db924cf7c	6740	} \
wolfSSL	15:117db924cf7c	6741	while (0)
wolfSSL	15:117db924cf7c	6742
wolfSSL	15:117db924cf7c	6743	#define aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T) \
wolfSSL	15:117db924cf7c	6744	do \
wolfSSL	15:117db924cf7c	6745	{ \
wolfSSL	15:117db924cf7c	6746	if (ibytes % 16) { \
wolfSSL	15:117db924cf7c	6747	i = ibytes / 16; \
wolfSSL	15:117db924cf7c	6748	for (j=0; j < (int)(ibytes%16); j++) \
wolfSSL	15:117db924cf7c	6749	((unsigned char)&last_block)[j] = ivec[i16+j]; \
wolfSSL	15:117db924cf7c	6750	} \
wolfSSL	15:117db924cf7c	6751	tmp1 = _mm_load_si128(&KEY[0]); \
wolfSSL	15:117db924cf7c	6752	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \
wolfSSL	15:117db924cf7c	6753	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \
wolfSSL	15:117db924cf7c	6754	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \
wolfSSL	15:117db924cf7c	6755	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \
wolfSSL	15:117db924cf7c	6756	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \
wolfSSL	15:117db924cf7c	6757	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \
wolfSSL	15:117db924cf7c	6758	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \
wolfSSL	15:117db924cf7c	6759	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \
wolfSSL	15:117db924cf7c	6760	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \
wolfSSL	15:117db924cf7c	6761	lastKey = KEY[10]; \
wolfSSL	15:117db924cf7c	6762	if (nr > 10) { \
wolfSSL	15:117db924cf7c	6763	tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6764	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \
wolfSSL	15:117db924cf7c	6765	lastKey = KEY[12]; \
wolfSSL	15:117db924cf7c	6766	if (nr > 12) { \
wolfSSL	15:117db924cf7c	6767	tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6768	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \
wolfSSL	15:117db924cf7c	6769	lastKey = KEY[14]; \
wolfSSL	15:117db924cf7c	6770	} \
wolfSSL	15:117db924cf7c	6771	} \
wolfSSL	15:117db924cf7c	6772	H = _mm_aesenclast_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6773	H = _mm_shuffle_epi8(H, BSWAP_MASK); \
wolfSSL	15:117db924cf7c	6774	Y = _mm_setzero_si128(); \
wolfSSL	15:117db924cf7c	6775	for (i=0; i < (int)(ibytes/16); i++) { \
wolfSSL	15:117db924cf7c	6776	tmp1 = _mm_loadu_si128(&((__m128i*)ivec)[i]); \
wolfSSL	15:117db924cf7c	6777	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); \
wolfSSL	15:117db924cf7c	6778	Y = _mm_xor_si128(Y, tmp1); \
wolfSSL	15:117db924cf7c	6779	Y = gfmul_sw(Y, H); \
wolfSSL	15:117db924cf7c	6780	} \
wolfSSL	15:117db924cf7c	6781	if (ibytes % 16) { \
wolfSSL	15:117db924cf7c	6782	tmp1 = last_block; \
wolfSSL	15:117db924cf7c	6783	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK); \
wolfSSL	15:117db924cf7c	6784	Y = _mm_xor_si128(Y, tmp1); \
wolfSSL	15:117db924cf7c	6785	Y = gfmul_sw(Y, H); \
wolfSSL	15:117db924cf7c	6786	} \
wolfSSL	15:117db924cf7c	6787	tmp1 = _mm_insert_epi64(tmp1, ibytes*8, 0); \
wolfSSL	15:117db924cf7c	6788	tmp1 = _mm_insert_epi64(tmp1, 0, 1); \
wolfSSL	15:117db924cf7c	6789	Y = _mm_xor_si128(Y, tmp1); \
wolfSSL	15:117db924cf7c	6790	Y = gfmul_sw(Y, H); \
wolfSSL	15:117db924cf7c	6791	Y = _mm_shuffle_epi8(Y, BSWAP_MASK); /* Compute E(K, Y0) */ \
wolfSSL	15:117db924cf7c	6792	tmp1 = _mm_xor_si128(Y, KEY[0]); \
wolfSSL	15:117db924cf7c	6793	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]); \
wolfSSL	15:117db924cf7c	6794	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]); \
wolfSSL	15:117db924cf7c	6795	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]); \
wolfSSL	15:117db924cf7c	6796	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]); \
wolfSSL	15:117db924cf7c	6797	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]); \
wolfSSL	15:117db924cf7c	6798	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]); \
wolfSSL	15:117db924cf7c	6799	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]); \
wolfSSL	15:117db924cf7c	6800	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]); \
wolfSSL	15:117db924cf7c	6801	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]); \
wolfSSL	15:117db924cf7c	6802	lastKey = KEY[10]; \
wolfSSL	15:117db924cf7c	6803	if (nr > 10) { \
wolfSSL	15:117db924cf7c	6804	tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6805	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]); \
wolfSSL	15:117db924cf7c	6806	lastKey = KEY[12]; \
wolfSSL	15:117db924cf7c	6807	if (nr > 12) { \
wolfSSL	15:117db924cf7c	6808	tmp1 = _mm_aesenc_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6809	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]); \
wolfSSL	15:117db924cf7c	6810	lastKey = KEY[14]; \
wolfSSL	15:117db924cf7c	6811	} \
wolfSSL	15:117db924cf7c	6812	} \
wolfSSL	15:117db924cf7c	6813	T = _mm_aesenclast_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6814	} \
wolfSSL	15:117db924cf7c	6815	while (0)
wolfSSL	15:117db924cf7c	6816
wolfSSL	15:117db924cf7c	6817	#define AES_ENC_8(j) \
wolfSSL	15:117db924cf7c	6818	tmp1 = _mm_aesenc_si128(tmp1, KEY[j]); \
wolfSSL	15:117db924cf7c	6819	tmp2 = _mm_aesenc_si128(tmp2, KEY[j]); \
wolfSSL	15:117db924cf7c	6820	tmp3 = _mm_aesenc_si128(tmp3, KEY[j]); \
wolfSSL	15:117db924cf7c	6821	tmp4 = _mm_aesenc_si128(tmp4, KEY[j]); \
wolfSSL	15:117db924cf7c	6822	tmp5 = _mm_aesenc_si128(tmp5, KEY[j]); \
wolfSSL	15:117db924cf7c	6823	tmp6 = _mm_aesenc_si128(tmp6, KEY[j]); \
wolfSSL	15:117db924cf7c	6824	tmp7 = _mm_aesenc_si128(tmp7, KEY[j]); \
wolfSSL	15:117db924cf7c	6825	tmp8 = _mm_aesenc_si128(tmp8, KEY[j]);
wolfSSL	15:117db924cf7c	6826
wolfSSL	15:117db924cf7c	6827	#define AES_ENC_LAST_8() \
wolfSSL	15:117db924cf7c	6828	tmp1 =_mm_aesenclast_si128(tmp1, lastKey); \
wolfSSL	15:117db924cf7c	6829	tmp2 =_mm_aesenclast_si128(tmp2, lastKey); \
wolfSSL	15:117db924cf7c	6830	tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i)in)[i8+0])); \
wolfSSL	15:117db924cf7c	6831	tmp2 = _mm_xor_si128(tmp2, _mm_loadu_si128(&((__m128i)in)[i8+1])); \
wolfSSL	15:117db924cf7c	6832	_mm_storeu_si128(&((__m128i)out)[i8+0], tmp1); \
wolfSSL	15:117db924cf7c	6833	_mm_storeu_si128(&((__m128i)out)[i8+1], tmp2); \
wolfSSL	15:117db924cf7c	6834	tmp3 =_mm_aesenclast_si128(tmp3, lastKey); \
wolfSSL	15:117db924cf7c	6835	tmp4 =_mm_aesenclast_si128(tmp4, lastKey); \
wolfSSL	15:117db924cf7c	6836	tmp3 = _mm_xor_si128(tmp3, _mm_loadu_si128(&((__m128i)in)[i8+2])); \
wolfSSL	15:117db924cf7c	6837	tmp4 = _mm_xor_si128(tmp4, _mm_loadu_si128(&((__m128i)in)[i8+3])); \
wolfSSL	15:117db924cf7c	6838	_mm_storeu_si128(&((__m128i)out)[i8+2], tmp3); \
wolfSSL	15:117db924cf7c	6839	_mm_storeu_si128(&((__m128i)out)[i8+3], tmp4); \
wolfSSL	15:117db924cf7c	6840	tmp5 =_mm_aesenclast_si128(tmp5, lastKey); \
wolfSSL	15:117db924cf7c	6841	tmp6 =_mm_aesenclast_si128(tmp6, lastKey); \
wolfSSL	15:117db924cf7c	6842	tmp5 = _mm_xor_si128(tmp5, _mm_loadu_si128(&((__m128i)in)[i8+4])); \
wolfSSL	15:117db924cf7c	6843	tmp6 = _mm_xor_si128(tmp6, _mm_loadu_si128(&((__m128i)in)[i8+5])); \
wolfSSL	15:117db924cf7c	6844	_mm_storeu_si128(&((__m128i)out)[i8+4], tmp5); \
wolfSSL	15:117db924cf7c	6845	_mm_storeu_si128(&((__m128i)out)[i8+5], tmp6); \
wolfSSL	15:117db924cf7c	6846	tmp7 =_mm_aesenclast_si128(tmp7, lastKey); \
wolfSSL	15:117db924cf7c	6847	tmp8 =_mm_aesenclast_si128(tmp8, lastKey); \
wolfSSL	15:117db924cf7c	6848	tmp7 = _mm_xor_si128(tmp7, _mm_loadu_si128(&((__m128i)in)[i8+6])); \
wolfSSL	15:117db924cf7c	6849	tmp8 = _mm_xor_si128(tmp8, _mm_loadu_si128(&((__m128i)in)[i8+7])); \
wolfSSL	15:117db924cf7c	6850	_mm_storeu_si128(&((__m128i)out)[i8+6], tmp7); \
wolfSSL	15:117db924cf7c	6851	_mm_storeu_si128(&((__m128i)out)[i8+7], tmp8);
wolfSSL	15:117db924cf7c	6852
wolfSSL	15:117db924cf7c	6853
wolfSSL	15:117db924cf7c	6854	static __m128i gfmul_sw(__m128i a, __m128i b)
wolfSSL	15:117db924cf7c	6855	{
wolfSSL	15:117db924cf7c	6856	__m128i r, t1, t2, t3, t4, t5, t6, t7;
wolfSSL	15:117db924cf7c	6857	t2 = _mm_shuffle_epi32(b, 78);
wolfSSL	15:117db924cf7c	6858	t3 = _mm_shuffle_epi32(a, 78);
wolfSSL	15:117db924cf7c	6859	t2 = _mm_xor_si128(t2, b);
wolfSSL	15:117db924cf7c	6860	t3 = _mm_xor_si128(t3, a);
wolfSSL	15:117db924cf7c	6861	t4 = _mm_clmulepi64_si128(b, a, 0x11);
wolfSSL	15:117db924cf7c	6862	t1 = _mm_clmulepi64_si128(b, a, 0x00);
wolfSSL	15:117db924cf7c	6863	t2 = _mm_clmulepi64_si128(t2, t3, 0x00);
wolfSSL	15:117db924cf7c	6864	t2 = _mm_xor_si128(t2, t1);
wolfSSL	15:117db924cf7c	6865	t2 = _mm_xor_si128(t2, t4);
wolfSSL	15:117db924cf7c	6866	t3 = _mm_slli_si128(t2, 8);
wolfSSL	15:117db924cf7c	6867	t2 = _mm_srli_si128(t2, 8);
wolfSSL	15:117db924cf7c	6868	t1 = _mm_xor_si128(t1, t3);
wolfSSL	15:117db924cf7c	6869	t4 = _mm_xor_si128(t4, t2);
wolfSSL	15:117db924cf7c	6870
wolfSSL	15:117db924cf7c	6871	t5 = _mm_srli_epi32(t1, 31);
wolfSSL	15:117db924cf7c	6872	t6 = _mm_srli_epi32(t4, 31);
wolfSSL	15:117db924cf7c	6873	t1 = _mm_slli_epi32(t1, 1);
wolfSSL	15:117db924cf7c	6874	t4 = _mm_slli_epi32(t4, 1);
wolfSSL	15:117db924cf7c	6875	t7 = _mm_srli_si128(t5, 12);
wolfSSL	15:117db924cf7c	6876	t5 = _mm_slli_si128(t5, 4);
wolfSSL	15:117db924cf7c	6877	t6 = _mm_slli_si128(t6, 4);
wolfSSL	15:117db924cf7c	6878	t4 = _mm_or_si128(t4, t7);
wolfSSL	15:117db924cf7c	6879	t1 = _mm_or_si128(t1, t5);
wolfSSL	15:117db924cf7c	6880	t4 = _mm_or_si128(t4, t6);
wolfSSL	15:117db924cf7c	6881
wolfSSL	15:117db924cf7c	6882	t5 = _mm_slli_epi32(t1, 31);
wolfSSL	15:117db924cf7c	6883	t6 = _mm_slli_epi32(t1, 30);
wolfSSL	15:117db924cf7c	6884	t7 = _mm_slli_epi32(t1, 25);
wolfSSL	15:117db924cf7c	6885	t5 = _mm_xor_si128(t5, t6);
wolfSSL	15:117db924cf7c	6886	t5 = _mm_xor_si128(t5, t7);
wolfSSL	15:117db924cf7c	6887
wolfSSL	15:117db924cf7c	6888	t6 = _mm_srli_si128(t5, 4);
wolfSSL	15:117db924cf7c	6889	t5 = _mm_slli_si128(t5, 12);
wolfSSL	15:117db924cf7c	6890	t1 = _mm_xor_si128(t1, t5);
wolfSSL	15:117db924cf7c	6891	t7 = _mm_srli_epi32(t1, 1);
wolfSSL	15:117db924cf7c	6892	t3 = _mm_srli_epi32(t1, 2);
wolfSSL	15:117db924cf7c	6893	t2 = _mm_srli_epi32(t1, 7);
wolfSSL	15:117db924cf7c	6894
wolfSSL	15:117db924cf7c	6895	t7 = _mm_xor_si128(t7, t3);
wolfSSL	15:117db924cf7c	6896	t7 = _mm_xor_si128(t7, t2);
wolfSSL	15:117db924cf7c	6897	t7 = _mm_xor_si128(t7, t6);
wolfSSL	15:117db924cf7c	6898	t7 = _mm_xor_si128(t7, t1);
wolfSSL	15:117db924cf7c	6899	r = _mm_xor_si128(t4, t7);
wolfSSL	15:117db924cf7c	6900
wolfSSL	15:117db924cf7c	6901	return r;
wolfSSL	15:117db924cf7c	6902	}
wolfSSL	15:117db924cf7c	6903
wolfSSL	15:117db924cf7c	6904	static void gfmul_only(__m128i a, __m128i b, __m128i* r0, __m128i* r1)
wolfSSL	15:117db924cf7c	6905	{
wolfSSL	15:117db924cf7c	6906	__m128i t1, t2, t3, t4;
wolfSSL	15:117db924cf7c	6907
wolfSSL	15:117db924cf7c	6908	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	6909	t2 = _mm_shuffle_epi32(b, 78);
wolfSSL	15:117db924cf7c	6910	t3 = _mm_shuffle_epi32(a, 78);
wolfSSL	15:117db924cf7c	6911	t2 = _mm_xor_si128(t2, b);
wolfSSL	15:117db924cf7c	6912	t3 = _mm_xor_si128(t3, a);
wolfSSL	15:117db924cf7c	6913	t4 = _mm_clmulepi64_si128(b, a, 0x11);
wolfSSL	15:117db924cf7c	6914	t1 = _mm_clmulepi64_si128(b, a, 0x00);
wolfSSL	15:117db924cf7c	6915	t2 = _mm_clmulepi64_si128(t2, t3, 0x00);
wolfSSL	15:117db924cf7c	6916	t2 = _mm_xor_si128(t2, t1);
wolfSSL	15:117db924cf7c	6917	t2 = _mm_xor_si128(t2, t4);
wolfSSL	15:117db924cf7c	6918	t3 = _mm_slli_si128(t2, 8);
wolfSSL	15:117db924cf7c	6919	t2 = _mm_srli_si128(t2, 8);
wolfSSL	15:117db924cf7c	6920	t1 = _mm_xor_si128(t1, t3);
wolfSSL	15:117db924cf7c	6921	t4 = _mm_xor_si128(t4, t2);
wolfSSL	15:117db924cf7c	6922	r0 = _mm_xor_si128(t1, r0);
wolfSSL	15:117db924cf7c	6923	r1 = _mm_xor_si128(t4, r1);
wolfSSL	15:117db924cf7c	6924	}
wolfSSL	15:117db924cf7c	6925
wolfSSL	15:117db924cf7c	6926	static __m128i gfmul_shl1(__m128i a)
wolfSSL	15:117db924cf7c	6927	{
wolfSSL	15:117db924cf7c	6928	__m128i t1 = a, t2;
wolfSSL	15:117db924cf7c	6929	t2 = _mm_srli_epi64(t1, 63);
wolfSSL	15:117db924cf7c	6930	t1 = _mm_slli_epi64(t1, 1);
wolfSSL	15:117db924cf7c	6931	t2 = _mm_slli_si128(t2, 8);
wolfSSL	15:117db924cf7c	6932	t1 = _mm_or_si128(t1, t2);
wolfSSL	15:117db924cf7c	6933	/* if (a[1] >> 63) t1 = _mm_xor_si128(t1, MOD2_128); */
wolfSSL	15:117db924cf7c	6934	a = _mm_shuffle_epi32(a, 0xff);
wolfSSL	15:117db924cf7c	6935	a = _mm_srai_epi32(a, 31);
wolfSSL	15:117db924cf7c	6936	a = _mm_and_si128(a, MOD2_128);
wolfSSL	15:117db924cf7c	6937	t1 = _mm_xor_si128(t1, a);
wolfSSL	15:117db924cf7c	6938	return t1;
wolfSSL	15:117db924cf7c	6939	}
wolfSSL	15:117db924cf7c	6940
wolfSSL	15:117db924cf7c	6941	static __m128i ghash_red(__m128i r0, __m128i r1)
wolfSSL	15:117db924cf7c	6942	{
wolfSSL	15:117db924cf7c	6943	__m128i t2, t3;
wolfSSL	15:117db924cf7c	6944	__m128i t5, t6, t7;
wolfSSL	15:117db924cf7c	6945
wolfSSL	15:117db924cf7c	6946	t5 = _mm_slli_epi32(r0, 31);
wolfSSL	15:117db924cf7c	6947	t6 = _mm_slli_epi32(r0, 30);
wolfSSL	15:117db924cf7c	6948	t7 = _mm_slli_epi32(r0, 25);
wolfSSL	15:117db924cf7c	6949	t5 = _mm_xor_si128(t5, t6);
wolfSSL	15:117db924cf7c	6950	t5 = _mm_xor_si128(t5, t7);
wolfSSL	15:117db924cf7c	6951
wolfSSL	15:117db924cf7c	6952	t6 = _mm_srli_si128(t5, 4);
wolfSSL	15:117db924cf7c	6953	t5 = _mm_slli_si128(t5, 12);
wolfSSL	15:117db924cf7c	6954	r0 = _mm_xor_si128(r0, t5);
wolfSSL	15:117db924cf7c	6955	t7 = _mm_srli_epi32(r0, 1);
wolfSSL	15:117db924cf7c	6956	t3 = _mm_srli_epi32(r0, 2);
wolfSSL	15:117db924cf7c	6957	t2 = _mm_srli_epi32(r0, 7);
wolfSSL	15:117db924cf7c	6958
wolfSSL	15:117db924cf7c	6959	t7 = _mm_xor_si128(t7, t3);
wolfSSL	15:117db924cf7c	6960	t7 = _mm_xor_si128(t7, t2);
wolfSSL	15:117db924cf7c	6961	t7 = _mm_xor_si128(t7, t6);
wolfSSL	15:117db924cf7c	6962	t7 = _mm_xor_si128(t7, r0);
wolfSSL	15:117db924cf7c	6963	return _mm_xor_si128(r1, t7);
wolfSSL	15:117db924cf7c	6964	}
wolfSSL	15:117db924cf7c	6965
wolfSSL	15:117db924cf7c	6966	static __m128i gfmul_shifted(__m128i a, __m128i b)
wolfSSL	15:117db924cf7c	6967	{
wolfSSL	15:117db924cf7c	6968	__m128i t0 = _mm_setzero_si128(), t1 = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	6969	gfmul_only(a, b, &t0, &t1);
wolfSSL	15:117db924cf7c	6970	return ghash_red(t0, t1);
wolfSSL	15:117db924cf7c	6971	}
wolfSSL	15:117db924cf7c	6972
wolfSSL	15:117db924cf7c	6973	#ifndef AES_GCM_AESNI_NO_UNROLL
wolfSSL	15:117db924cf7c	6974	static __m128i gfmul8(__m128i a1, __m128i a2, __m128i a3, __m128i a4,
wolfSSL	15:117db924cf7c	6975	__m128i a5, __m128i a6, __m128i a7, __m128i a8,
wolfSSL	15:117db924cf7c	6976	__m128i b1, __m128i b2, __m128i b3, __m128i b4,
wolfSSL	15:117db924cf7c	6977	__m128i b5, __m128i b6, __m128i b7, __m128i b8)
wolfSSL	15:117db924cf7c	6978	{
wolfSSL	15:117db924cf7c	6979	__m128i t0 = _mm_setzero_si128(), t1 = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	6980	gfmul_only(a1, b8, &t0, &t1);
wolfSSL	15:117db924cf7c	6981	gfmul_only(a2, b7, &t0, &t1);
wolfSSL	15:117db924cf7c	6982	gfmul_only(a3, b6, &t0, &t1);
wolfSSL	15:117db924cf7c	6983	gfmul_only(a4, b5, &t0, &t1);
wolfSSL	15:117db924cf7c	6984	gfmul_only(a5, b4, &t0, &t1);
wolfSSL	15:117db924cf7c	6985	gfmul_only(a6, b3, &t0, &t1);
wolfSSL	15:117db924cf7c	6986	gfmul_only(a7, b2, &t0, &t1);
wolfSSL	15:117db924cf7c	6987	gfmul_only(a8, b1, &t0, &t1);
wolfSSL	15:117db924cf7c	6988	return ghash_red(t0, t1);
wolfSSL	15:117db924cf7c	6989	}
wolfSSL	15:117db924cf7c	6990	#endif
wolfSSL	15:117db924cf7c	6991
wolfSSL	15:117db924cf7c	6992
wolfSSL	15:117db924cf7c	6993	static void AES_GCM_encrypt(const unsigned char *in,
wolfSSL	15:117db924cf7c	6994	unsigned char *out,
wolfSSL	15:117db924cf7c	6995	const unsigned char* addt,
wolfSSL	15:117db924cf7c	6996	const unsigned char* ivec,
wolfSSL	15:117db924cf7c	6997	unsigned char *tag, unsigned int nbytes,
wolfSSL	15:117db924cf7c	6998	unsigned int abytes, unsigned int ibytes,
wolfSSL	15:117db924cf7c	6999	unsigned int tbytes,
wolfSSL	15:117db924cf7c	7000	const unsigned char* key, int nr)
wolfSSL	15:117db924cf7c	7001	{
wolfSSL	15:117db924cf7c	7002	int i, j ,k;
wolfSSL	15:117db924cf7c	7003	__m128i ctr1;
wolfSSL	15:117db924cf7c	7004	__m128i H, Y, T;
wolfSSL	15:117db924cf7c	7005	__m128i X = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7006	__m128i KEY = (__m128i)key, lastKey;
wolfSSL	15:117db924cf7c	7007	__m128i last_block = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7008	__m128i tmp1, tmp2;
wolfSSL	15:117db924cf7c	7009	#ifndef AES_GCM_AESNI_NO_UNROLL
wolfSSL	15:117db924cf7c	7010	__m128i HT[8];
wolfSSL	15:117db924cf7c	7011	__m128i r0, r1;
wolfSSL	15:117db924cf7c	7012	__m128i XV;
wolfSSL	15:117db924cf7c	7013	__m128i tmp3, tmp4, tmp5, tmp6, tmp7, tmp8;
wolfSSL	15:117db924cf7c	7014	#endif
wolfSSL	15:117db924cf7c	7015
wolfSSL	15:117db924cf7c	7016	if (ibytes == 12)
wolfSSL	15:117db924cf7c	7017	aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T);
wolfSSL	15:117db924cf7c	7018	else
wolfSSL	15:117db924cf7c	7019	aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T);
wolfSSL	15:117db924cf7c	7020
wolfSSL	15:117db924cf7c	7021	for (i=0; i < (int)(abytes/16); i++) {
wolfSSL	15:117db924cf7c	7022	tmp1 = _mm_loadu_si128(&((__m128i*)addt)[i]);
wolfSSL	15:117db924cf7c	7023	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7024	X = _mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7025	X = gfmul_sw(X, H);
wolfSSL	15:117db924cf7c	7026	}
wolfSSL	15:117db924cf7c	7027	if (abytes%16) {
wolfSSL	15:117db924cf7c	7028	last_block = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7029	for (j=0; j < (int)(abytes%16); j++)
wolfSSL	15:117db924cf7c	7030	((unsigned char)&last_block)[j] = addt[i16+j];
wolfSSL	15:117db924cf7c	7031	tmp1 = last_block;
wolfSSL	15:117db924cf7c	7032	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7033	X = _mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7034	X = gfmul_sw(X, H);
wolfSSL	15:117db924cf7c	7035	}
wolfSSL	15:117db924cf7c	7036	tmp1 = _mm_shuffle_epi8(Y, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7037	ctr1 = _mm_add_epi32(tmp1, ONE);
wolfSSL	15:117db924cf7c	7038	H = gfmul_shl1(H);
wolfSSL	15:117db924cf7c	7039
wolfSSL	15:117db924cf7c	7040	#ifndef AES_GCM_AESNI_NO_UNROLL
wolfSSL	15:117db924cf7c	7041	i = 0;
wolfSSL	15:117db924cf7c	7042	if (nbytes >= 16*8) {
wolfSSL	15:117db924cf7c	7043	HT[0] = H;
wolfSSL	15:117db924cf7c	7044	HT[1] = gfmul_shifted(H, H);
wolfSSL	15:117db924cf7c	7045	HT[2] = gfmul_shifted(H, HT[1]);
wolfSSL	15:117db924cf7c	7046	HT[3] = gfmul_shifted(HT[1], HT[1]);
wolfSSL	15:117db924cf7c	7047	HT[4] = gfmul_shifted(HT[1], HT[2]);
wolfSSL	15:117db924cf7c	7048	HT[5] = gfmul_shifted(HT[2], HT[2]);
wolfSSL	15:117db924cf7c	7049	HT[6] = gfmul_shifted(HT[2], HT[3]);
wolfSSL	15:117db924cf7c	7050	HT[7] = gfmul_shifted(HT[3], HT[3]);
wolfSSL	15:117db924cf7c	7051
wolfSSL	15:117db924cf7c	7052	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7053	tmp2 = _mm_add_epi32(ctr1, ONE);
wolfSSL	15:117db924cf7c	7054	tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7055	tmp3 = _mm_add_epi32(ctr1, TWO);
wolfSSL	15:117db924cf7c	7056	tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7057	tmp4 = _mm_add_epi32(ctr1, THREE);
wolfSSL	15:117db924cf7c	7058	tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7059	tmp5 = _mm_add_epi32(ctr1, FOUR);
wolfSSL	15:117db924cf7c	7060	tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7061	tmp6 = _mm_add_epi32(ctr1, FIVE);
wolfSSL	15:117db924cf7c	7062	tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7063	tmp7 = _mm_add_epi32(ctr1, SIX);
wolfSSL	15:117db924cf7c	7064	tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7065	tmp8 = _mm_add_epi32(ctr1, SEVEN);
wolfSSL	15:117db924cf7c	7066	tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7067	ctr1 = _mm_add_epi32(ctr1, EIGHT);
wolfSSL	15:117db924cf7c	7068	tmp1 =_mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7069	tmp2 =_mm_xor_si128(tmp2, KEY[0]);
wolfSSL	15:117db924cf7c	7070	tmp3 =_mm_xor_si128(tmp3, KEY[0]);
wolfSSL	15:117db924cf7c	7071	tmp4 =_mm_xor_si128(tmp4, KEY[0]);
wolfSSL	15:117db924cf7c	7072	tmp5 =_mm_xor_si128(tmp5, KEY[0]);
wolfSSL	15:117db924cf7c	7073	tmp6 =_mm_xor_si128(tmp6, KEY[0]);
wolfSSL	15:117db924cf7c	7074	tmp7 =_mm_xor_si128(tmp7, KEY[0]);
wolfSSL	15:117db924cf7c	7075	tmp8 =_mm_xor_si128(tmp8, KEY[0]);
wolfSSL	15:117db924cf7c	7076	AES_ENC_8(1);
wolfSSL	15:117db924cf7c	7077	AES_ENC_8(2);
wolfSSL	15:117db924cf7c	7078	AES_ENC_8(3);
wolfSSL	15:117db924cf7c	7079	AES_ENC_8(4);
wolfSSL	15:117db924cf7c	7080	AES_ENC_8(5);
wolfSSL	15:117db924cf7c	7081	AES_ENC_8(6);
wolfSSL	15:117db924cf7c	7082	AES_ENC_8(7);
wolfSSL	15:117db924cf7c	7083	AES_ENC_8(8);
wolfSSL	15:117db924cf7c	7084	AES_ENC_8(9);
wolfSSL	15:117db924cf7c	7085	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7086	if (nr > 10) {
wolfSSL	15:117db924cf7c	7087	AES_ENC_8(10);
wolfSSL	15:117db924cf7c	7088	AES_ENC_8(11);
wolfSSL	15:117db924cf7c	7089	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7090	if (nr > 12) {
wolfSSL	15:117db924cf7c	7091	AES_ENC_8(12);
wolfSSL	15:117db924cf7c	7092	AES_ENC_8(13);
wolfSSL	15:117db924cf7c	7093	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7094	}
wolfSSL	15:117db924cf7c	7095	}
wolfSSL	15:117db924cf7c	7096	AES_ENC_LAST_8();
wolfSSL	15:117db924cf7c	7097
wolfSSL	15:117db924cf7c	7098	for (i=1; i < (int)(nbytes/16/8); i++) {
wolfSSL	15:117db924cf7c	7099	r0 = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7100	r1 = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7101	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7102	tmp2 = _mm_add_epi32(ctr1, ONE);
wolfSSL	15:117db924cf7c	7103	tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7104	tmp3 = _mm_add_epi32(ctr1, TWO);
wolfSSL	15:117db924cf7c	7105	tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7106	tmp4 = _mm_add_epi32(ctr1, THREE);
wolfSSL	15:117db924cf7c	7107	tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7108	tmp5 = _mm_add_epi32(ctr1, FOUR);
wolfSSL	15:117db924cf7c	7109	tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7110	tmp6 = _mm_add_epi32(ctr1, FIVE);
wolfSSL	15:117db924cf7c	7111	tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7112	tmp7 = _mm_add_epi32(ctr1, SIX);
wolfSSL	15:117db924cf7c	7113	tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7114	tmp8 = _mm_add_epi32(ctr1, SEVEN);
wolfSSL	15:117db924cf7c	7115	tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7116	ctr1 = _mm_add_epi32(ctr1, EIGHT);
wolfSSL	15:117db924cf7c	7117	tmp1 =_mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7118	tmp2 =_mm_xor_si128(tmp2, KEY[0]);
wolfSSL	15:117db924cf7c	7119	tmp3 =_mm_xor_si128(tmp3, KEY[0]);
wolfSSL	15:117db924cf7c	7120	tmp4 =_mm_xor_si128(tmp4, KEY[0]);
wolfSSL	15:117db924cf7c	7121	tmp5 =_mm_xor_si128(tmp5, KEY[0]);
wolfSSL	15:117db924cf7c	7122	tmp6 =_mm_xor_si128(tmp6, KEY[0]);
wolfSSL	15:117db924cf7c	7123	tmp7 =_mm_xor_si128(tmp7, KEY[0]);
wolfSSL	15:117db924cf7c	7124	tmp8 =_mm_xor_si128(tmp8, KEY[0]);
wolfSSL	15:117db924cf7c	7125	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7126	XV = _mm_loadu_si128(&((__m128i)out)[(i-1)8+0]);
wolfSSL	15:117db924cf7c	7127	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7128	XV = _mm_xor_si128(XV, X);
wolfSSL	15:117db924cf7c	7129	gfmul_only(XV, HT[7], &r0, &r1);
wolfSSL	15:117db924cf7c	7130	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
wolfSSL	15:117db924cf7c	7131	tmp2 = _mm_aesenc_si128(tmp2, KEY[1]);
wolfSSL	15:117db924cf7c	7132	tmp3 = _mm_aesenc_si128(tmp3, KEY[1]);
wolfSSL	15:117db924cf7c	7133	tmp4 = _mm_aesenc_si128(tmp4, KEY[1]);
wolfSSL	15:117db924cf7c	7134	tmp5 = _mm_aesenc_si128(tmp5, KEY[1]);
wolfSSL	15:117db924cf7c	7135	tmp6 = _mm_aesenc_si128(tmp6, KEY[1]);
wolfSSL	15:117db924cf7c	7136	tmp7 = _mm_aesenc_si128(tmp7, KEY[1]);
wolfSSL	15:117db924cf7c	7137	tmp8 = _mm_aesenc_si128(tmp8, KEY[1]);
wolfSSL	15:117db924cf7c	7138	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7139	XV = _mm_loadu_si128(&((__m128i)out)[(i-1)8+1]);
wolfSSL	15:117db924cf7c	7140	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7141	gfmul_only(XV, HT[6], &r0, &r1);
wolfSSL	15:117db924cf7c	7142	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
wolfSSL	15:117db924cf7c	7143	tmp2 = _mm_aesenc_si128(tmp2, KEY[2]);
wolfSSL	15:117db924cf7c	7144	tmp3 = _mm_aesenc_si128(tmp3, KEY[2]);
wolfSSL	15:117db924cf7c	7145	tmp4 = _mm_aesenc_si128(tmp4, KEY[2]);
wolfSSL	15:117db924cf7c	7146	tmp5 = _mm_aesenc_si128(tmp5, KEY[2]);
wolfSSL	15:117db924cf7c	7147	tmp6 = _mm_aesenc_si128(tmp6, KEY[2]);
wolfSSL	15:117db924cf7c	7148	tmp7 = _mm_aesenc_si128(tmp7, KEY[2]);
wolfSSL	15:117db924cf7c	7149	tmp8 = _mm_aesenc_si128(tmp8, KEY[2]);
wolfSSL	15:117db924cf7c	7150	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7151	XV = _mm_loadu_si128(&((__m128i)out)[(i-1)8+2]);
wolfSSL	15:117db924cf7c	7152	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7153	gfmul_only(XV, HT[5], &r0, &r1);
wolfSSL	15:117db924cf7c	7154	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
wolfSSL	15:117db924cf7c	7155	tmp2 = _mm_aesenc_si128(tmp2, KEY[3]);
wolfSSL	15:117db924cf7c	7156	tmp3 = _mm_aesenc_si128(tmp3, KEY[3]);
wolfSSL	15:117db924cf7c	7157	tmp4 = _mm_aesenc_si128(tmp4, KEY[3]);
wolfSSL	15:117db924cf7c	7158	tmp5 = _mm_aesenc_si128(tmp5, KEY[3]);
wolfSSL	15:117db924cf7c	7159	tmp6 = _mm_aesenc_si128(tmp6, KEY[3]);
wolfSSL	15:117db924cf7c	7160	tmp7 = _mm_aesenc_si128(tmp7, KEY[3]);
wolfSSL	15:117db924cf7c	7161	tmp8 = _mm_aesenc_si128(tmp8, KEY[3]);
wolfSSL	15:117db924cf7c	7162	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7163	XV = _mm_loadu_si128(&((__m128i)out)[(i-1)8+3]);
wolfSSL	15:117db924cf7c	7164	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7165	gfmul_only(XV, HT[4], &r0, &r1);
wolfSSL	15:117db924cf7c	7166	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
wolfSSL	15:117db924cf7c	7167	tmp2 = _mm_aesenc_si128(tmp2, KEY[4]);
wolfSSL	15:117db924cf7c	7168	tmp3 = _mm_aesenc_si128(tmp3, KEY[4]);
wolfSSL	15:117db924cf7c	7169	tmp4 = _mm_aesenc_si128(tmp4, KEY[4]);
wolfSSL	15:117db924cf7c	7170	tmp5 = _mm_aesenc_si128(tmp5, KEY[4]);
wolfSSL	15:117db924cf7c	7171	tmp6 = _mm_aesenc_si128(tmp6, KEY[4]);
wolfSSL	15:117db924cf7c	7172	tmp7 = _mm_aesenc_si128(tmp7, KEY[4]);
wolfSSL	15:117db924cf7c	7173	tmp8 = _mm_aesenc_si128(tmp8, KEY[4]);
wolfSSL	15:117db924cf7c	7174	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7175	XV = _mm_loadu_si128(&((__m128i)out)[(i-1)8+4]);
wolfSSL	15:117db924cf7c	7176	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7177	gfmul_only(XV, HT[3], &r0, &r1);
wolfSSL	15:117db924cf7c	7178	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
wolfSSL	15:117db924cf7c	7179	tmp2 = _mm_aesenc_si128(tmp2, KEY[5]);
wolfSSL	15:117db924cf7c	7180	tmp3 = _mm_aesenc_si128(tmp3, KEY[5]);
wolfSSL	15:117db924cf7c	7181	tmp4 = _mm_aesenc_si128(tmp4, KEY[5]);
wolfSSL	15:117db924cf7c	7182	tmp5 = _mm_aesenc_si128(tmp5, KEY[5]);
wolfSSL	15:117db924cf7c	7183	tmp6 = _mm_aesenc_si128(tmp6, KEY[5]);
wolfSSL	15:117db924cf7c	7184	tmp7 = _mm_aesenc_si128(tmp7, KEY[5]);
wolfSSL	15:117db924cf7c	7185	tmp8 = _mm_aesenc_si128(tmp8, KEY[5]);
wolfSSL	15:117db924cf7c	7186	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7187	XV = _mm_loadu_si128(&((__m128i)out)[(i-1)8+5]);
wolfSSL	15:117db924cf7c	7188	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7189	gfmul_only(XV, HT[2], &r0, &r1);
wolfSSL	15:117db924cf7c	7190	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
wolfSSL	15:117db924cf7c	7191	tmp2 = _mm_aesenc_si128(tmp2, KEY[6]);
wolfSSL	15:117db924cf7c	7192	tmp3 = _mm_aesenc_si128(tmp3, KEY[6]);
wolfSSL	15:117db924cf7c	7193	tmp4 = _mm_aesenc_si128(tmp4, KEY[6]);
wolfSSL	15:117db924cf7c	7194	tmp5 = _mm_aesenc_si128(tmp5, KEY[6]);
wolfSSL	15:117db924cf7c	7195	tmp6 = _mm_aesenc_si128(tmp6, KEY[6]);
wolfSSL	15:117db924cf7c	7196	tmp7 = _mm_aesenc_si128(tmp7, KEY[6]);
wolfSSL	15:117db924cf7c	7197	tmp8 = _mm_aesenc_si128(tmp8, KEY[6]);
wolfSSL	15:117db924cf7c	7198	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7199	XV = _mm_loadu_si128(&((__m128i)out)[(i-1)8+6]);
wolfSSL	15:117db924cf7c	7200	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7201	gfmul_only(XV, HT[1], &r0, &r1);
wolfSSL	15:117db924cf7c	7202	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
wolfSSL	15:117db924cf7c	7203	tmp2 = _mm_aesenc_si128(tmp2, KEY[7]);
wolfSSL	15:117db924cf7c	7204	tmp3 = _mm_aesenc_si128(tmp3, KEY[7]);
wolfSSL	15:117db924cf7c	7205	tmp4 = _mm_aesenc_si128(tmp4, KEY[7]);
wolfSSL	15:117db924cf7c	7206	tmp5 = _mm_aesenc_si128(tmp5, KEY[7]);
wolfSSL	15:117db924cf7c	7207	tmp6 = _mm_aesenc_si128(tmp6, KEY[7]);
wolfSSL	15:117db924cf7c	7208	tmp7 = _mm_aesenc_si128(tmp7, KEY[7]);
wolfSSL	15:117db924cf7c	7209	tmp8 = _mm_aesenc_si128(tmp8, KEY[7]);
wolfSSL	15:117db924cf7c	7210	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7211	XV = _mm_loadu_si128(&((__m128i)out)[(i-1)8+7]);
wolfSSL	15:117db924cf7c	7212	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7213	gfmul_only(XV, HT[0], &r0, &r1);
wolfSSL	15:117db924cf7c	7214	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
wolfSSL	15:117db924cf7c	7215	tmp2 = _mm_aesenc_si128(tmp2, KEY[8]);
wolfSSL	15:117db924cf7c	7216	tmp3 = _mm_aesenc_si128(tmp3, KEY[8]);
wolfSSL	15:117db924cf7c	7217	tmp4 = _mm_aesenc_si128(tmp4, KEY[8]);
wolfSSL	15:117db924cf7c	7218	tmp5 = _mm_aesenc_si128(tmp5, KEY[8]);
wolfSSL	15:117db924cf7c	7219	tmp6 = _mm_aesenc_si128(tmp6, KEY[8]);
wolfSSL	15:117db924cf7c	7220	tmp7 = _mm_aesenc_si128(tmp7, KEY[8]);
wolfSSL	15:117db924cf7c	7221	tmp8 = _mm_aesenc_si128(tmp8, KEY[8]);
wolfSSL	15:117db924cf7c	7222	/* Reduction */
wolfSSL	15:117db924cf7c	7223	X = ghash_red(r0, r1);
wolfSSL	15:117db924cf7c	7224	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
wolfSSL	15:117db924cf7c	7225	tmp2 = _mm_aesenc_si128(tmp2, KEY[9]);
wolfSSL	15:117db924cf7c	7226	tmp3 = _mm_aesenc_si128(tmp3, KEY[9]);
wolfSSL	15:117db924cf7c	7227	tmp4 = _mm_aesenc_si128(tmp4, KEY[9]);
wolfSSL	15:117db924cf7c	7228	tmp5 = _mm_aesenc_si128(tmp5, KEY[9]);
wolfSSL	15:117db924cf7c	7229	tmp6 = _mm_aesenc_si128(tmp6, KEY[9]);
wolfSSL	15:117db924cf7c	7230	tmp7 = _mm_aesenc_si128(tmp7, KEY[9]);
wolfSSL	15:117db924cf7c	7231	tmp8 = _mm_aesenc_si128(tmp8, KEY[9]);
wolfSSL	15:117db924cf7c	7232	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7233	if (nr > 10) {
wolfSSL	15:117db924cf7c	7234	tmp1 = _mm_aesenc_si128(tmp1, KEY[10]);
wolfSSL	15:117db924cf7c	7235	tmp2 = _mm_aesenc_si128(tmp2, KEY[10]);
wolfSSL	15:117db924cf7c	7236	tmp3 = _mm_aesenc_si128(tmp3, KEY[10]);
wolfSSL	15:117db924cf7c	7237	tmp4 = _mm_aesenc_si128(tmp4, KEY[10]);
wolfSSL	15:117db924cf7c	7238	tmp5 = _mm_aesenc_si128(tmp5, KEY[10]);
wolfSSL	15:117db924cf7c	7239	tmp6 = _mm_aesenc_si128(tmp6, KEY[10]);
wolfSSL	15:117db924cf7c	7240	tmp7 = _mm_aesenc_si128(tmp7, KEY[10]);
wolfSSL	15:117db924cf7c	7241	tmp8 = _mm_aesenc_si128(tmp8, KEY[10]);
wolfSSL	15:117db924cf7c	7242	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
wolfSSL	15:117db924cf7c	7243	tmp2 = _mm_aesenc_si128(tmp2, KEY[11]);
wolfSSL	15:117db924cf7c	7244	tmp3 = _mm_aesenc_si128(tmp3, KEY[11]);
wolfSSL	15:117db924cf7c	7245	tmp4 = _mm_aesenc_si128(tmp4, KEY[11]);
wolfSSL	15:117db924cf7c	7246	tmp5 = _mm_aesenc_si128(tmp5, KEY[11]);
wolfSSL	15:117db924cf7c	7247	tmp6 = _mm_aesenc_si128(tmp6, KEY[11]);
wolfSSL	15:117db924cf7c	7248	tmp7 = _mm_aesenc_si128(tmp7, KEY[11]);
wolfSSL	15:117db924cf7c	7249	tmp8 = _mm_aesenc_si128(tmp8, KEY[11]);
wolfSSL	15:117db924cf7c	7250	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7251	if (nr > 12) {
wolfSSL	15:117db924cf7c	7252	tmp1 = _mm_aesenc_si128(tmp1, KEY[12]);
wolfSSL	15:117db924cf7c	7253	tmp2 = _mm_aesenc_si128(tmp2, KEY[12]);
wolfSSL	15:117db924cf7c	7254	tmp3 = _mm_aesenc_si128(tmp3, KEY[12]);
wolfSSL	15:117db924cf7c	7255	tmp4 = _mm_aesenc_si128(tmp4, KEY[12]);
wolfSSL	15:117db924cf7c	7256	tmp5 = _mm_aesenc_si128(tmp5, KEY[12]);
wolfSSL	15:117db924cf7c	7257	tmp6 = _mm_aesenc_si128(tmp6, KEY[12]);
wolfSSL	15:117db924cf7c	7258	tmp7 = _mm_aesenc_si128(tmp7, KEY[12]);
wolfSSL	15:117db924cf7c	7259	tmp8 = _mm_aesenc_si128(tmp8, KEY[12]);
wolfSSL	15:117db924cf7c	7260	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
wolfSSL	15:117db924cf7c	7261	tmp2 = _mm_aesenc_si128(tmp2, KEY[13]);
wolfSSL	15:117db924cf7c	7262	tmp3 = _mm_aesenc_si128(tmp3, KEY[13]);
wolfSSL	15:117db924cf7c	7263	tmp4 = _mm_aesenc_si128(tmp4, KEY[13]);
wolfSSL	15:117db924cf7c	7264	tmp5 = _mm_aesenc_si128(tmp5, KEY[13]);
wolfSSL	15:117db924cf7c	7265	tmp6 = _mm_aesenc_si128(tmp6, KEY[13]);
wolfSSL	15:117db924cf7c	7266	tmp7 = _mm_aesenc_si128(tmp7, KEY[13]);
wolfSSL	15:117db924cf7c	7267	tmp8 = _mm_aesenc_si128(tmp8, KEY[13]);
wolfSSL	15:117db924cf7c	7268	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7269	}
wolfSSL	15:117db924cf7c	7270	}
wolfSSL	15:117db924cf7c	7271	AES_ENC_LAST_8();
wolfSSL	15:117db924cf7c	7272	}
wolfSSL	15:117db924cf7c	7273
wolfSSL	15:117db924cf7c	7274	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7275	tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7276	tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7277	tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7278	tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7279	tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7280	tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7281	tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7282	tmp1 = _mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7283	X = gfmul8(tmp1, tmp2, tmp3, tmp4, tmp5, tmp6, tmp7, tmp8,
wolfSSL	15:117db924cf7c	7284	HT[0], HT[1], HT[2], HT[3], HT[4], HT[5], HT[6], HT[7]);
wolfSSL	15:117db924cf7c	7285	}
wolfSSL	15:117db924cf7c	7286	for (k = i*8; k < (int)(nbytes/16); k++) {
wolfSSL	15:117db924cf7c	7287	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7288	ctr1 = _mm_add_epi32(ctr1, ONE);
wolfSSL	15:117db924cf7c	7289	tmp1 = _mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7290	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
wolfSSL	15:117db924cf7c	7291	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
wolfSSL	15:117db924cf7c	7292	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
wolfSSL	15:117db924cf7c	7293	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
wolfSSL	15:117db924cf7c	7294	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
wolfSSL	15:117db924cf7c	7295	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
wolfSSL	15:117db924cf7c	7296	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
wolfSSL	15:117db924cf7c	7297	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
wolfSSL	15:117db924cf7c	7298	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
wolfSSL	15:117db924cf7c	7299	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7300	if (nr > 10) {
wolfSSL	15:117db924cf7c	7301	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7302	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
wolfSSL	15:117db924cf7c	7303	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7304	if (nr > 12) {
wolfSSL	15:117db924cf7c	7305	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7306	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
wolfSSL	15:117db924cf7c	7307	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7308	}
wolfSSL	15:117db924cf7c	7309	}
wolfSSL	15:117db924cf7c	7310	tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7311	tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[k]));
wolfSSL	15:117db924cf7c	7312	_mm_storeu_si128(&((__m128i*)out)[k], tmp1);
wolfSSL	15:117db924cf7c	7313	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7314	X =_mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7315	X = gfmul_shifted(X, H);
wolfSSL	15:117db924cf7c	7316	}
wolfSSL	15:117db924cf7c	7317	#else /* AES_GCM_AESNI_NO_UNROLL */
wolfSSL	15:117db924cf7c	7318	for (k = 0; k < (int)(nbytes/16) && k < 1; k++) {
wolfSSL	15:117db924cf7c	7319	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7320	ctr1 = _mm_add_epi32(ctr1, ONE);
wolfSSL	15:117db924cf7c	7321	tmp1 = _mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7322	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
wolfSSL	15:117db924cf7c	7323	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
wolfSSL	15:117db924cf7c	7324	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
wolfSSL	15:117db924cf7c	7325	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
wolfSSL	15:117db924cf7c	7326	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
wolfSSL	15:117db924cf7c	7327	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
wolfSSL	15:117db924cf7c	7328	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
wolfSSL	15:117db924cf7c	7329	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
wolfSSL	15:117db924cf7c	7330	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
wolfSSL	15:117db924cf7c	7331	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7332	if (nr > 10) {
wolfSSL	15:117db924cf7c	7333	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7334	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
wolfSSL	15:117db924cf7c	7335	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7336	if (nr > 12) {
wolfSSL	15:117db924cf7c	7337	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7338	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
wolfSSL	15:117db924cf7c	7339	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7340	}
wolfSSL	15:117db924cf7c	7341	}
wolfSSL	15:117db924cf7c	7342	tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7343	tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[k]));
wolfSSL	15:117db924cf7c	7344	_mm_storeu_si128(&((__m128i*)out)[k], tmp1);
wolfSSL	15:117db924cf7c	7345	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7346	X =_mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7347	}
wolfSSL	15:117db924cf7c	7348	for (; k < (int)(nbytes/16); k++) {
wolfSSL	15:117db924cf7c	7349	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7350	ctr1 = _mm_add_epi32(ctr1, ONE);
wolfSSL	15:117db924cf7c	7351	tmp1 = _mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7352	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
wolfSSL	15:117db924cf7c	7353	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
wolfSSL	15:117db924cf7c	7354	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
wolfSSL	15:117db924cf7c	7355	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
wolfSSL	15:117db924cf7c	7356	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
wolfSSL	15:117db924cf7c	7357	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
wolfSSL	15:117db924cf7c	7358	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
wolfSSL	15:117db924cf7c	7359	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
wolfSSL	15:117db924cf7c	7360	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
wolfSSL	15:117db924cf7c	7361	X = gfmul_shifted(X, H);
wolfSSL	15:117db924cf7c	7362	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7363	if (nr > 10) {
wolfSSL	15:117db924cf7c	7364	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7365	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
wolfSSL	15:117db924cf7c	7366	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7367	if (nr > 12) {
wolfSSL	15:117db924cf7c	7368	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7369	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
wolfSSL	15:117db924cf7c	7370	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7371	}
wolfSSL	15:117db924cf7c	7372	}
wolfSSL	15:117db924cf7c	7373	tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7374	tmp1 = _mm_xor_si128(tmp1, _mm_loadu_si128(&((__m128i*)in)[k]));
wolfSSL	15:117db924cf7c	7375	_mm_storeu_si128(&((__m128i*)out)[k], tmp1);
wolfSSL	15:117db924cf7c	7376	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7377	X =_mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7378	}
wolfSSL	15:117db924cf7c	7379	if (k > 0) {
wolfSSL	15:117db924cf7c	7380	X = gfmul_shifted(X, H);
wolfSSL	15:117db924cf7c	7381	}
wolfSSL	15:117db924cf7c	7382	#endif /* AES_GCM_AESNI_NO_UNROLL */
wolfSSL	15:117db924cf7c	7383
wolfSSL	15:117db924cf7c	7384	/* If one partial block remains */
wolfSSL	15:117db924cf7c	7385	if (nbytes % 16) {
wolfSSL	15:117db924cf7c	7386	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7387	tmp1 = _mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7388	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
wolfSSL	15:117db924cf7c	7389	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
wolfSSL	15:117db924cf7c	7390	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
wolfSSL	15:117db924cf7c	7391	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
wolfSSL	15:117db924cf7c	7392	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
wolfSSL	15:117db924cf7c	7393	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
wolfSSL	15:117db924cf7c	7394	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
wolfSSL	15:117db924cf7c	7395	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
wolfSSL	15:117db924cf7c	7396	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
wolfSSL	15:117db924cf7c	7397	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7398	if (nr > 10) {
wolfSSL	15:117db924cf7c	7399	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7400	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
wolfSSL	15:117db924cf7c	7401	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7402	if (nr > 12) {
wolfSSL	15:117db924cf7c	7403	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7404	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
wolfSSL	15:117db924cf7c	7405	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7406	}
wolfSSL	15:117db924cf7c	7407	}
wolfSSL	15:117db924cf7c	7408	tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7409	last_block = tmp1;
wolfSSL	15:117db924cf7c	7410	for (j=0; j < (int)(nbytes%16); j++)
wolfSSL	15:117db924cf7c	7411	((unsigned char)&last_block)[j] = in[k16+j];
wolfSSL	15:117db924cf7c	7412	tmp1 = _mm_xor_si128(tmp1, last_block);
wolfSSL	15:117db924cf7c	7413	last_block = tmp1;
wolfSSL	15:117db924cf7c	7414	for (j=0; j < (int)(nbytes%16); j++)
wolfSSL	15:117db924cf7c	7415	out[k16+j] = ((unsigned char)&last_block)[j];
wolfSSL	15:117db924cf7c	7416	tmp1 = last_block;
wolfSSL	15:117db924cf7c	7417	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7418	X =_mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7419	X = gfmul_shifted(X, H);
wolfSSL	15:117db924cf7c	7420	}
wolfSSL	15:117db924cf7c	7421	tmp1 = _mm_insert_epi64(tmp1, nbytes*8, 0);
wolfSSL	15:117db924cf7c	7422	tmp1 = _mm_insert_epi64(tmp1, abytes*8, 1);
wolfSSL	15:117db924cf7c	7423	X = _mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7424	X = gfmul_shifted(X, H);
wolfSSL	15:117db924cf7c	7425	X = _mm_shuffle_epi8(X, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7426	T = _mm_xor_si128(X, T);
wolfSSL	15:117db924cf7c	7427	/_mm_storeu_si128((__m128i)tag, T);*/
wolfSSL	15:117db924cf7c	7428	XMEMCPY(tag, &T, tbytes);
wolfSSL	15:117db924cf7c	7429	}
wolfSSL	15:117db924cf7c	7430
wolfSSL	15:117db924cf7c	7431	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	7432
wolfSSL	15:117db924cf7c	7433	static void AES_GCM_decrypt(const unsigned char *in,
wolfSSL	15:117db924cf7c	7434	unsigned char *out,
wolfSSL	15:117db924cf7c	7435	const unsigned char* addt,
wolfSSL	15:117db924cf7c	7436	const unsigned char* ivec,
wolfSSL	15:117db924cf7c	7437	const unsigned char *tag, int nbytes, int abytes,
wolfSSL	15:117db924cf7c	7438	int ibytes, word32 tbytes, const unsigned char* key,
wolfSSL	15:117db924cf7c	7439	int nr, int* res)
wolfSSL	15:117db924cf7c	7440	{
wolfSSL	15:117db924cf7c	7441	int i, j ,k;
wolfSSL	15:117db924cf7c	7442	__m128i H, Y, T;
wolfSSL	15:117db924cf7c	7443	__m128i KEY = (__m128i)key, lastKey;
wolfSSL	15:117db924cf7c	7444	__m128i ctr1;
wolfSSL	15:117db924cf7c	7445	__m128i last_block = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7446	__m128i X = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7447	__m128i tmp1, tmp2, XV;
wolfSSL	15:117db924cf7c	7448	#ifndef AES_GCM_AESNI_NO_UNROLL
wolfSSL	15:117db924cf7c	7449	__m128i HT[8];
wolfSSL	15:117db924cf7c	7450	__m128i r0, r1;
wolfSSL	15:117db924cf7c	7451	__m128i tmp3, tmp4, tmp5, tmp6, tmp7, tmp8;
wolfSSL	15:117db924cf7c	7452	#endif /* AES_GCM_AESNI_NO_UNROLL */
wolfSSL	15:117db924cf7c	7453
wolfSSL	15:117db924cf7c	7454	if (ibytes == 12)
wolfSSL	15:117db924cf7c	7455	aes_gcm_calc_iv_12(KEY, ivec, nr, H, Y, T);
wolfSSL	15:117db924cf7c	7456	else
wolfSSL	15:117db924cf7c	7457	aes_gcm_calc_iv(KEY, ivec, ibytes, nr, H, Y, T);
wolfSSL	15:117db924cf7c	7458
wolfSSL	15:117db924cf7c	7459	for (i=0; i<abytes/16; i++) {
wolfSSL	15:117db924cf7c	7460	tmp1 = _mm_loadu_si128(&((__m128i*)addt)[i]);
wolfSSL	15:117db924cf7c	7461	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7462	X = _mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7463	X = gfmul_sw(X, H);
wolfSSL	15:117db924cf7c	7464	}
wolfSSL	15:117db924cf7c	7465	if (abytes%16) {
wolfSSL	15:117db924cf7c	7466	last_block = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7467	for (j=0; j<abytes%16; j++)
wolfSSL	15:117db924cf7c	7468	((unsigned char)&last_block)[j] = addt[i16+j];
wolfSSL	15:117db924cf7c	7469	tmp1 = last_block;
wolfSSL	15:117db924cf7c	7470	tmp1 = _mm_shuffle_epi8(tmp1, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7471	X = _mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7472	X = gfmul_sw(X, H);
wolfSSL	15:117db924cf7c	7473	}
wolfSSL	15:117db924cf7c	7474
wolfSSL	15:117db924cf7c	7475	tmp1 = _mm_shuffle_epi8(Y, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7476	ctr1 = _mm_add_epi32(tmp1, ONE);
wolfSSL	15:117db924cf7c	7477	H = gfmul_shl1(H);
wolfSSL	15:117db924cf7c	7478	i = 0;
wolfSSL	15:117db924cf7c	7479
wolfSSL	15:117db924cf7c	7480	#ifndef AES_GCM_AESNI_NO_UNROLL
wolfSSL	15:117db924cf7c	7481
wolfSSL	15:117db924cf7c	7482	if (0 < nbytes/16/8) {
wolfSSL	15:117db924cf7c	7483	HT[0] = H;
wolfSSL	15:117db924cf7c	7484	HT[1] = gfmul_shifted(H, H);
wolfSSL	15:117db924cf7c	7485	HT[2] = gfmul_shifted(H, HT[1]);
wolfSSL	15:117db924cf7c	7486	HT[3] = gfmul_shifted(HT[1], HT[1]);
wolfSSL	15:117db924cf7c	7487	HT[4] = gfmul_shifted(HT[1], HT[2]);
wolfSSL	15:117db924cf7c	7488	HT[5] = gfmul_shifted(HT[2], HT[2]);
wolfSSL	15:117db924cf7c	7489	HT[6] = gfmul_shifted(HT[2], HT[3]);
wolfSSL	15:117db924cf7c	7490	HT[7] = gfmul_shifted(HT[3], HT[3]);
wolfSSL	15:117db924cf7c	7491
wolfSSL	15:117db924cf7c	7492	for (; i < nbytes/16/8; i++) {
wolfSSL	15:117db924cf7c	7493	r0 = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7494	r1 = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7495
wolfSSL	15:117db924cf7c	7496	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7497	tmp2 = _mm_add_epi32(ctr1, ONE);
wolfSSL	15:117db924cf7c	7498	tmp2 = _mm_shuffle_epi8(tmp2, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7499	tmp3 = _mm_add_epi32(ctr1, TWO);
wolfSSL	15:117db924cf7c	7500	tmp3 = _mm_shuffle_epi8(tmp3, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7501	tmp4 = _mm_add_epi32(ctr1, THREE);
wolfSSL	15:117db924cf7c	7502	tmp4 = _mm_shuffle_epi8(tmp4, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7503	tmp5 = _mm_add_epi32(ctr1, FOUR);
wolfSSL	15:117db924cf7c	7504	tmp5 = _mm_shuffle_epi8(tmp5, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7505	tmp6 = _mm_add_epi32(ctr1, FIVE);
wolfSSL	15:117db924cf7c	7506	tmp6 = _mm_shuffle_epi8(tmp6, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7507	tmp7 = _mm_add_epi32(ctr1, SIX);
wolfSSL	15:117db924cf7c	7508	tmp7 = _mm_shuffle_epi8(tmp7, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7509	tmp8 = _mm_add_epi32(ctr1, SEVEN);
wolfSSL	15:117db924cf7c	7510	tmp8 = _mm_shuffle_epi8(tmp8, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7511	ctr1 = _mm_add_epi32(ctr1, EIGHT);
wolfSSL	15:117db924cf7c	7512	tmp1 =_mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7513	tmp2 =_mm_xor_si128(tmp2, KEY[0]);
wolfSSL	15:117db924cf7c	7514	tmp3 =_mm_xor_si128(tmp3, KEY[0]);
wolfSSL	15:117db924cf7c	7515	tmp4 =_mm_xor_si128(tmp4, KEY[0]);
wolfSSL	15:117db924cf7c	7516	tmp5 =_mm_xor_si128(tmp5, KEY[0]);
wolfSSL	15:117db924cf7c	7517	tmp6 =_mm_xor_si128(tmp6, KEY[0]);
wolfSSL	15:117db924cf7c	7518	tmp7 =_mm_xor_si128(tmp7, KEY[0]);
wolfSSL	15:117db924cf7c	7519	tmp8 =_mm_xor_si128(tmp8, KEY[0]);
wolfSSL	15:117db924cf7c	7520	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7521	XV = _mm_loadu_si128(&((__m128i)in)[i8+0]);
wolfSSL	15:117db924cf7c	7522	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7523	XV = _mm_xor_si128(XV, X);
wolfSSL	15:117db924cf7c	7524	gfmul_only(XV, HT[7], &r0, &r1);
wolfSSL	15:117db924cf7c	7525	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
wolfSSL	15:117db924cf7c	7526	tmp2 = _mm_aesenc_si128(tmp2, KEY[1]);
wolfSSL	15:117db924cf7c	7527	tmp3 = _mm_aesenc_si128(tmp3, KEY[1]);
wolfSSL	15:117db924cf7c	7528	tmp4 = _mm_aesenc_si128(tmp4, KEY[1]);
wolfSSL	15:117db924cf7c	7529	tmp5 = _mm_aesenc_si128(tmp5, KEY[1]);
wolfSSL	15:117db924cf7c	7530	tmp6 = _mm_aesenc_si128(tmp6, KEY[1]);
wolfSSL	15:117db924cf7c	7531	tmp7 = _mm_aesenc_si128(tmp7, KEY[1]);
wolfSSL	15:117db924cf7c	7532	tmp8 = _mm_aesenc_si128(tmp8, KEY[1]);
wolfSSL	15:117db924cf7c	7533	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7534	XV = _mm_loadu_si128(&((__m128i)in)[i8+1]);
wolfSSL	15:117db924cf7c	7535	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7536	gfmul_only(XV, HT[6], &r0, &r1);
wolfSSL	15:117db924cf7c	7537	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
wolfSSL	15:117db924cf7c	7538	tmp2 = _mm_aesenc_si128(tmp2, KEY[2]);
wolfSSL	15:117db924cf7c	7539	tmp3 = _mm_aesenc_si128(tmp3, KEY[2]);
wolfSSL	15:117db924cf7c	7540	tmp4 = _mm_aesenc_si128(tmp4, KEY[2]);
wolfSSL	15:117db924cf7c	7541	tmp5 = _mm_aesenc_si128(tmp5, KEY[2]);
wolfSSL	15:117db924cf7c	7542	tmp6 = _mm_aesenc_si128(tmp6, KEY[2]);
wolfSSL	15:117db924cf7c	7543	tmp7 = _mm_aesenc_si128(tmp7, KEY[2]);
wolfSSL	15:117db924cf7c	7544	tmp8 = _mm_aesenc_si128(tmp8, KEY[2]);
wolfSSL	15:117db924cf7c	7545	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7546	XV = _mm_loadu_si128(&((__m128i)in)[i8+2]);
wolfSSL	15:117db924cf7c	7547	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7548	gfmul_only(XV, HT[5], &r0, &r1);
wolfSSL	15:117db924cf7c	7549	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
wolfSSL	15:117db924cf7c	7550	tmp2 = _mm_aesenc_si128(tmp2, KEY[3]);
wolfSSL	15:117db924cf7c	7551	tmp3 = _mm_aesenc_si128(tmp3, KEY[3]);
wolfSSL	15:117db924cf7c	7552	tmp4 = _mm_aesenc_si128(tmp4, KEY[3]);
wolfSSL	15:117db924cf7c	7553	tmp5 = _mm_aesenc_si128(tmp5, KEY[3]);
wolfSSL	15:117db924cf7c	7554	tmp6 = _mm_aesenc_si128(tmp6, KEY[3]);
wolfSSL	15:117db924cf7c	7555	tmp7 = _mm_aesenc_si128(tmp7, KEY[3]);
wolfSSL	15:117db924cf7c	7556	tmp8 = _mm_aesenc_si128(tmp8, KEY[3]);
wolfSSL	15:117db924cf7c	7557	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7558	XV = _mm_loadu_si128(&((__m128i)in)[i8+3]);
wolfSSL	15:117db924cf7c	7559	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7560	gfmul_only(XV, HT[4], &r0, &r1);
wolfSSL	15:117db924cf7c	7561	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
wolfSSL	15:117db924cf7c	7562	tmp2 = _mm_aesenc_si128(tmp2, KEY[4]);
wolfSSL	15:117db924cf7c	7563	tmp3 = _mm_aesenc_si128(tmp3, KEY[4]);
wolfSSL	15:117db924cf7c	7564	tmp4 = _mm_aesenc_si128(tmp4, KEY[4]);
wolfSSL	15:117db924cf7c	7565	tmp5 = _mm_aesenc_si128(tmp5, KEY[4]);
wolfSSL	15:117db924cf7c	7566	tmp6 = _mm_aesenc_si128(tmp6, KEY[4]);
wolfSSL	15:117db924cf7c	7567	tmp7 = _mm_aesenc_si128(tmp7, KEY[4]);
wolfSSL	15:117db924cf7c	7568	tmp8 = _mm_aesenc_si128(tmp8, KEY[4]);
wolfSSL	15:117db924cf7c	7569	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7570	XV = _mm_loadu_si128(&((__m128i)in)[i8+4]);
wolfSSL	15:117db924cf7c	7571	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7572	gfmul_only(XV, HT[3], &r0, &r1);
wolfSSL	15:117db924cf7c	7573	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
wolfSSL	15:117db924cf7c	7574	tmp2 = _mm_aesenc_si128(tmp2, KEY[5]);
wolfSSL	15:117db924cf7c	7575	tmp3 = _mm_aesenc_si128(tmp3, KEY[5]);
wolfSSL	15:117db924cf7c	7576	tmp4 = _mm_aesenc_si128(tmp4, KEY[5]);
wolfSSL	15:117db924cf7c	7577	tmp5 = _mm_aesenc_si128(tmp5, KEY[5]);
wolfSSL	15:117db924cf7c	7578	tmp6 = _mm_aesenc_si128(tmp6, KEY[5]);
wolfSSL	15:117db924cf7c	7579	tmp7 = _mm_aesenc_si128(tmp7, KEY[5]);
wolfSSL	15:117db924cf7c	7580	tmp8 = _mm_aesenc_si128(tmp8, KEY[5]);
wolfSSL	15:117db924cf7c	7581	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7582	XV = _mm_loadu_si128(&((__m128i)in)[i8+5]);
wolfSSL	15:117db924cf7c	7583	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7584	gfmul_only(XV, HT[2], &r0, &r1);
wolfSSL	15:117db924cf7c	7585	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
wolfSSL	15:117db924cf7c	7586	tmp2 = _mm_aesenc_si128(tmp2, KEY[6]);
wolfSSL	15:117db924cf7c	7587	tmp3 = _mm_aesenc_si128(tmp3, KEY[6]);
wolfSSL	15:117db924cf7c	7588	tmp4 = _mm_aesenc_si128(tmp4, KEY[6]);
wolfSSL	15:117db924cf7c	7589	tmp5 = _mm_aesenc_si128(tmp5, KEY[6]);
wolfSSL	15:117db924cf7c	7590	tmp6 = _mm_aesenc_si128(tmp6, KEY[6]);
wolfSSL	15:117db924cf7c	7591	tmp7 = _mm_aesenc_si128(tmp7, KEY[6]);
wolfSSL	15:117db924cf7c	7592	tmp8 = _mm_aesenc_si128(tmp8, KEY[6]);
wolfSSL	15:117db924cf7c	7593	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7594	XV = _mm_loadu_si128(&((__m128i)in)[i8+6]);
wolfSSL	15:117db924cf7c	7595	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7596	gfmul_only(XV, HT[1], &r0, &r1);
wolfSSL	15:117db924cf7c	7597	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
wolfSSL	15:117db924cf7c	7598	tmp2 = _mm_aesenc_si128(tmp2, KEY[7]);
wolfSSL	15:117db924cf7c	7599	tmp3 = _mm_aesenc_si128(tmp3, KEY[7]);
wolfSSL	15:117db924cf7c	7600	tmp4 = _mm_aesenc_si128(tmp4, KEY[7]);
wolfSSL	15:117db924cf7c	7601	tmp5 = _mm_aesenc_si128(tmp5, KEY[7]);
wolfSSL	15:117db924cf7c	7602	tmp6 = _mm_aesenc_si128(tmp6, KEY[7]);
wolfSSL	15:117db924cf7c	7603	tmp7 = _mm_aesenc_si128(tmp7, KEY[7]);
wolfSSL	15:117db924cf7c	7604	tmp8 = _mm_aesenc_si128(tmp8, KEY[7]);
wolfSSL	15:117db924cf7c	7605	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7606	XV = _mm_loadu_si128(&((__m128i)in)[i8+7]);
wolfSSL	15:117db924cf7c	7607	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7608	gfmul_only(XV, HT[0], &r0, &r1);
wolfSSL	15:117db924cf7c	7609	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
wolfSSL	15:117db924cf7c	7610	tmp2 = _mm_aesenc_si128(tmp2, KEY[8]);
wolfSSL	15:117db924cf7c	7611	tmp3 = _mm_aesenc_si128(tmp3, KEY[8]);
wolfSSL	15:117db924cf7c	7612	tmp4 = _mm_aesenc_si128(tmp4, KEY[8]);
wolfSSL	15:117db924cf7c	7613	tmp5 = _mm_aesenc_si128(tmp5, KEY[8]);
wolfSSL	15:117db924cf7c	7614	tmp6 = _mm_aesenc_si128(tmp6, KEY[8]);
wolfSSL	15:117db924cf7c	7615	tmp7 = _mm_aesenc_si128(tmp7, KEY[8]);
wolfSSL	15:117db924cf7c	7616	tmp8 = _mm_aesenc_si128(tmp8, KEY[8]);
wolfSSL	15:117db924cf7c	7617	/* Reduction */
wolfSSL	15:117db924cf7c	7618	X = ghash_red(r0, r1);
wolfSSL	15:117db924cf7c	7619	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
wolfSSL	15:117db924cf7c	7620	tmp2 = _mm_aesenc_si128(tmp2, KEY[9]);
wolfSSL	15:117db924cf7c	7621	tmp3 = _mm_aesenc_si128(tmp3, KEY[9]);
wolfSSL	15:117db924cf7c	7622	tmp4 = _mm_aesenc_si128(tmp4, KEY[9]);
wolfSSL	15:117db924cf7c	7623	tmp5 = _mm_aesenc_si128(tmp5, KEY[9]);
wolfSSL	15:117db924cf7c	7624	tmp6 = _mm_aesenc_si128(tmp6, KEY[9]);
wolfSSL	15:117db924cf7c	7625	tmp7 = _mm_aesenc_si128(tmp7, KEY[9]);
wolfSSL	15:117db924cf7c	7626	tmp8 = _mm_aesenc_si128(tmp8, KEY[9]);
wolfSSL	15:117db924cf7c	7627	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7628	if (nr > 10) {
wolfSSL	15:117db924cf7c	7629	tmp1 = _mm_aesenc_si128(tmp1, KEY[10]);
wolfSSL	15:117db924cf7c	7630	tmp2 = _mm_aesenc_si128(tmp2, KEY[10]);
wolfSSL	15:117db924cf7c	7631	tmp3 = _mm_aesenc_si128(tmp3, KEY[10]);
wolfSSL	15:117db924cf7c	7632	tmp4 = _mm_aesenc_si128(tmp4, KEY[10]);
wolfSSL	15:117db924cf7c	7633	tmp5 = _mm_aesenc_si128(tmp5, KEY[10]);
wolfSSL	15:117db924cf7c	7634	tmp6 = _mm_aesenc_si128(tmp6, KEY[10]);
wolfSSL	15:117db924cf7c	7635	tmp7 = _mm_aesenc_si128(tmp7, KEY[10]);
wolfSSL	15:117db924cf7c	7636	tmp8 = _mm_aesenc_si128(tmp8, KEY[10]);
wolfSSL	15:117db924cf7c	7637	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
wolfSSL	15:117db924cf7c	7638	tmp2 = _mm_aesenc_si128(tmp2, KEY[11]);
wolfSSL	15:117db924cf7c	7639	tmp3 = _mm_aesenc_si128(tmp3, KEY[11]);
wolfSSL	15:117db924cf7c	7640	tmp4 = _mm_aesenc_si128(tmp4, KEY[11]);
wolfSSL	15:117db924cf7c	7641	tmp5 = _mm_aesenc_si128(tmp5, KEY[11]);
wolfSSL	15:117db924cf7c	7642	tmp6 = _mm_aesenc_si128(tmp6, KEY[11]);
wolfSSL	15:117db924cf7c	7643	tmp7 = _mm_aesenc_si128(tmp7, KEY[11]);
wolfSSL	15:117db924cf7c	7644	tmp8 = _mm_aesenc_si128(tmp8, KEY[11]);
wolfSSL	15:117db924cf7c	7645	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7646	if (nr > 12) {
wolfSSL	15:117db924cf7c	7647	tmp1 = _mm_aesenc_si128(tmp1, KEY[12]);
wolfSSL	15:117db924cf7c	7648	tmp2 = _mm_aesenc_si128(tmp2, KEY[12]);
wolfSSL	15:117db924cf7c	7649	tmp3 = _mm_aesenc_si128(tmp3, KEY[12]);
wolfSSL	15:117db924cf7c	7650	tmp4 = _mm_aesenc_si128(tmp4, KEY[12]);
wolfSSL	15:117db924cf7c	7651	tmp5 = _mm_aesenc_si128(tmp5, KEY[12]);
wolfSSL	15:117db924cf7c	7652	tmp6 = _mm_aesenc_si128(tmp6, KEY[12]);
wolfSSL	15:117db924cf7c	7653	tmp7 = _mm_aesenc_si128(tmp7, KEY[12]);
wolfSSL	15:117db924cf7c	7654	tmp8 = _mm_aesenc_si128(tmp8, KEY[12]);
wolfSSL	15:117db924cf7c	7655	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
wolfSSL	15:117db924cf7c	7656	tmp2 = _mm_aesenc_si128(tmp2, KEY[13]);
wolfSSL	15:117db924cf7c	7657	tmp3 = _mm_aesenc_si128(tmp3, KEY[13]);
wolfSSL	15:117db924cf7c	7658	tmp4 = _mm_aesenc_si128(tmp4, KEY[13]);
wolfSSL	15:117db924cf7c	7659	tmp5 = _mm_aesenc_si128(tmp5, KEY[13]);
wolfSSL	15:117db924cf7c	7660	tmp6 = _mm_aesenc_si128(tmp6, KEY[13]);
wolfSSL	15:117db924cf7c	7661	tmp7 = _mm_aesenc_si128(tmp7, KEY[13]);
wolfSSL	15:117db924cf7c	7662	tmp8 = _mm_aesenc_si128(tmp8, KEY[13]);
wolfSSL	15:117db924cf7c	7663	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7664	}
wolfSSL	15:117db924cf7c	7665	}
wolfSSL	15:117db924cf7c	7666	AES_ENC_LAST_8();
wolfSSL	15:117db924cf7c	7667	}
wolfSSL	15:117db924cf7c	7668	}
wolfSSL	15:117db924cf7c	7669
wolfSSL	15:117db924cf7c	7670	#endif /* AES_GCM_AESNI_NO_UNROLL */
wolfSSL	15:117db924cf7c	7671
wolfSSL	15:117db924cf7c	7672	for (k = i*8; k < nbytes/16; k++) {
wolfSSL	15:117db924cf7c	7673	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7674	ctr1 = _mm_add_epi32(ctr1, ONE);
wolfSSL	15:117db924cf7c	7675	tmp1 = _mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7676	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
wolfSSL	15:117db924cf7c	7677	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
wolfSSL	15:117db924cf7c	7678	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
wolfSSL	15:117db924cf7c	7679	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
wolfSSL	15:117db924cf7c	7680	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
wolfSSL	15:117db924cf7c	7681	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
wolfSSL	15:117db924cf7c	7682	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
wolfSSL	15:117db924cf7c	7683	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
wolfSSL	15:117db924cf7c	7684	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
wolfSSL	15:117db924cf7c	7685	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7686	XV = _mm_loadu_si128(&((__m128i*)in)[k]);
wolfSSL	15:117db924cf7c	7687	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7688	XV = _mm_xor_si128(XV, X);
wolfSSL	15:117db924cf7c	7689	X = gfmul_shifted(XV, H);
wolfSSL	15:117db924cf7c	7690	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7691	if (nr > 10) {
wolfSSL	15:117db924cf7c	7692	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7693	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
wolfSSL	15:117db924cf7c	7694	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7695	if (nr > 12) {
wolfSSL	15:117db924cf7c	7696	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7697	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
wolfSSL	15:117db924cf7c	7698	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7699	}
wolfSSL	15:117db924cf7c	7700	}
wolfSSL	15:117db924cf7c	7701	tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7702	tmp2 = _mm_loadu_si128(&((__m128i*)in)[k]);
wolfSSL	15:117db924cf7c	7703	tmp1 = _mm_xor_si128(tmp1, tmp2);
wolfSSL	15:117db924cf7c	7704	_mm_storeu_si128(&((__m128i*)out)[k], tmp1);
wolfSSL	15:117db924cf7c	7705	}
wolfSSL	15:117db924cf7c	7706
wolfSSL	15:117db924cf7c	7707	/* If one partial block remains */
wolfSSL	15:117db924cf7c	7708	if (nbytes % 16) {
wolfSSL	15:117db924cf7c	7709	tmp1 = _mm_shuffle_epi8(ctr1, BSWAP_EPI64);
wolfSSL	15:117db924cf7c	7710	tmp1 = _mm_xor_si128(tmp1, KEY[0]);
wolfSSL	15:117db924cf7c	7711	tmp1 = _mm_aesenc_si128(tmp1, KEY[1]);
wolfSSL	15:117db924cf7c	7712	tmp1 = _mm_aesenc_si128(tmp1, KEY[2]);
wolfSSL	15:117db924cf7c	7713	tmp1 = _mm_aesenc_si128(tmp1, KEY[3]);
wolfSSL	15:117db924cf7c	7714	tmp1 = _mm_aesenc_si128(tmp1, KEY[4]);
wolfSSL	15:117db924cf7c	7715	tmp1 = _mm_aesenc_si128(tmp1, KEY[5]);
wolfSSL	15:117db924cf7c	7716	tmp1 = _mm_aesenc_si128(tmp1, KEY[6]);
wolfSSL	15:117db924cf7c	7717	tmp1 = _mm_aesenc_si128(tmp1, KEY[7]);
wolfSSL	15:117db924cf7c	7718	tmp1 = _mm_aesenc_si128(tmp1, KEY[8]);
wolfSSL	15:117db924cf7c	7719	tmp1 = _mm_aesenc_si128(tmp1, KEY[9]);
wolfSSL	15:117db924cf7c	7720	lastKey = KEY[10];
wolfSSL	15:117db924cf7c	7721	if (nr > 10) {
wolfSSL	15:117db924cf7c	7722	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7723	tmp1 = _mm_aesenc_si128(tmp1, KEY[11]);
wolfSSL	15:117db924cf7c	7724	lastKey = KEY[12];
wolfSSL	15:117db924cf7c	7725	if (nr > 12) {
wolfSSL	15:117db924cf7c	7726	tmp1 = _mm_aesenc_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7727	tmp1 = _mm_aesenc_si128(tmp1, KEY[13]);
wolfSSL	15:117db924cf7c	7728	lastKey = KEY[14];
wolfSSL	15:117db924cf7c	7729	}
wolfSSL	15:117db924cf7c	7730	}
wolfSSL	15:117db924cf7c	7731	tmp1 = _mm_aesenclast_si128(tmp1, lastKey);
wolfSSL	15:117db924cf7c	7732	last_block = _mm_setzero_si128();
wolfSSL	15:117db924cf7c	7733	for (j=0; j < nbytes%16; j++)
wolfSSL	15:117db924cf7c	7734	((unsigned char)&last_block)[j] = in[k16+j];
wolfSSL	15:117db924cf7c	7735	XV = last_block;
wolfSSL	15:117db924cf7c	7736	tmp1 = _mm_xor_si128(tmp1, last_block);
wolfSSL	15:117db924cf7c	7737	last_block = tmp1;
wolfSSL	15:117db924cf7c	7738	for (j=0; j < nbytes%16; j++)
wolfSSL	15:117db924cf7c	7739	out[k16+j] = ((unsigned char)&last_block)[j];
wolfSSL	15:117db924cf7c	7740	XV = _mm_shuffle_epi8(XV, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7741	XV = _mm_xor_si128(XV, X);
wolfSSL	15:117db924cf7c	7742	X = gfmul_shifted(XV, H);
wolfSSL	15:117db924cf7c	7743	}
wolfSSL	15:117db924cf7c	7744
wolfSSL	15:117db924cf7c	7745	tmp1 = _mm_insert_epi64(tmp1, nbytes*8, 0);
wolfSSL	15:117db924cf7c	7746	tmp1 = _mm_insert_epi64(tmp1, abytes*8, 1);
wolfSSL	15:117db924cf7c	7747	/* 128 x 128 Carryless Multiply */
wolfSSL	15:117db924cf7c	7748	X = _mm_xor_si128(X, tmp1);
wolfSSL	15:117db924cf7c	7749	X = gfmul_shifted(X, H);
wolfSSL	15:117db924cf7c	7750	X = _mm_shuffle_epi8(X, BSWAP_MASK);
wolfSSL	15:117db924cf7c	7751	T = _mm_xor_si128(X, T);
wolfSSL	15:117db924cf7c	7752
wolfSSL	15:117db924cf7c	7753	/* if (0xffff !=
wolfSSL	15:117db924cf7c	7754	_mm_movemask_epi8(_mm_cmpeq_epi8(T, _mm_loadu_si128((__m128i)tag)))) /
wolfSSL	15:117db924cf7c	7755	if (XMEMCMP(tag, &T, tbytes) != 0)
wolfSSL	15:117db924cf7c	7756	res = 0; / in case the authentication failed */
wolfSSL	15:117db924cf7c	7757	else
wolfSSL	15:117db924cf7c	7758	res = 1; / when successful returns 1 */
wolfSSL	15:117db924cf7c	7759	}
wolfSSL	15:117db924cf7c	7760
wolfSSL	15:117db924cf7c	7761	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	7762	#endif /* _MSC_VER */
wolfSSL	15:117db924cf7c	7763	#endif /* WOLFSSL_AESNI */
wolfSSL	15:117db924cf7c	7764
wolfSSL	15:117db924cf7c	7765
wolfSSL	15:117db924cf7c	7766	#if defined(GCM_SMALL)
wolfSSL	15:117db924cf7c	7767	static void GMULT(byte* X, byte* Y)
wolfSSL	15:117db924cf7c	7768	{
wolfSSL	15:117db924cf7c	7769	byte Z[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	7770	byte V[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	7771	int i, j;
wolfSSL	15:117db924cf7c	7772
wolfSSL	15:117db924cf7c	7773	XMEMSET(Z, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7774	XMEMCPY(V, X, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7775	for (i = 0; i < AES_BLOCK_SIZE; i++)
wolfSSL	15:117db924cf7c	7776	{
wolfSSL	15:117db924cf7c	7777	byte y = Y[i];
wolfSSL	15:117db924cf7c	7778	for (j = 0; j < 8; j++)
wolfSSL	15:117db924cf7c	7779	{
wolfSSL	15:117db924cf7c	7780	if (y & 0x80) {
wolfSSL	15:117db924cf7c	7781	xorbuf(Z, V, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7782	}
wolfSSL	15:117db924cf7c	7783
wolfSSL	15:117db924cf7c	7784	RIGHTSHIFTX(V);
wolfSSL	15:117db924cf7c	7785	y = y << 1;
wolfSSL	15:117db924cf7c	7786	}
wolfSSL	15:117db924cf7c	7787	}
wolfSSL	15:117db924cf7c	7788	XMEMCPY(X, Z, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7789	}
wolfSSL	15:117db924cf7c	7790
wolfSSL	15:117db924cf7c	7791
wolfSSL	15:117db924cf7c	7792	void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
wolfSSL	15:117db924cf7c	7793	word32 cSz, byte* s, word32 sSz)
wolfSSL	15:117db924cf7c	7794	{
wolfSSL	15:117db924cf7c	7795	byte x[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	7796	byte scratch[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	7797	word32 blocks, partial;
wolfSSL	15:117db924cf7c	7798	byte* h = aes->H;
wolfSSL	15:117db924cf7c	7799
wolfSSL	15:117db924cf7c	7800	XMEMSET(x, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7801
wolfSSL	15:117db924cf7c	7802	/* Hash in A, the Additional Authentication Data */
wolfSSL	15:117db924cf7c	7803	if (aSz != 0 && a != NULL) {
wolfSSL	15:117db924cf7c	7804	blocks = aSz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7805	partial = aSz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7806	while (blocks--) {
wolfSSL	15:117db924cf7c	7807	xorbuf(x, a, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7808	GMULT(x, h);
wolfSSL	15:117db924cf7c	7809	a += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7810	}
wolfSSL	15:117db924cf7c	7811	if (partial != 0) {
wolfSSL	15:117db924cf7c	7812	XMEMSET(scratch, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7813	XMEMCPY(scratch, a, partial);
wolfSSL	15:117db924cf7c	7814	xorbuf(x, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7815	GMULT(x, h);
wolfSSL	15:117db924cf7c	7816	}
wolfSSL	15:117db924cf7c	7817	}
wolfSSL	15:117db924cf7c	7818
wolfSSL	15:117db924cf7c	7819	/* Hash in C, the Ciphertext */
wolfSSL	15:117db924cf7c	7820	if (cSz != 0 && c != NULL) {
wolfSSL	15:117db924cf7c	7821	blocks = cSz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7822	partial = cSz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7823	while (blocks--) {
wolfSSL	15:117db924cf7c	7824	xorbuf(x, c, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7825	GMULT(x, h);
wolfSSL	15:117db924cf7c	7826	c += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7827	}
wolfSSL	15:117db924cf7c	7828	if (partial != 0) {
wolfSSL	15:117db924cf7c	7829	XMEMSET(scratch, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7830	XMEMCPY(scratch, c, partial);
wolfSSL	15:117db924cf7c	7831	xorbuf(x, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7832	GMULT(x, h);
wolfSSL	15:117db924cf7c	7833	}
wolfSSL	15:117db924cf7c	7834	}
wolfSSL	15:117db924cf7c	7835
wolfSSL	15:117db924cf7c	7836	/* Hash in the lengths of A and C in bits */
wolfSSL	15:117db924cf7c	7837	FlattenSzInBits(&scratch[0], aSz);
wolfSSL	15:117db924cf7c	7838	FlattenSzInBits(&scratch[8], cSz);
wolfSSL	15:117db924cf7c	7839	xorbuf(x, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7840	GMULT(x, h);
wolfSSL	15:117db924cf7c	7841
wolfSSL	15:117db924cf7c	7842	/* Copy the result into s. */
wolfSSL	15:117db924cf7c	7843	XMEMCPY(s, x, sSz);
wolfSSL	15:117db924cf7c	7844	}
wolfSSL	15:117db924cf7c	7845
wolfSSL	15:117db924cf7c	7846	/* end GCM_SMALL */
wolfSSL	15:117db924cf7c	7847	#elif defined(GCM_TABLE)
wolfSSL	15:117db924cf7c	7848
wolfSSL	15:117db924cf7c	7849	static const byte R[256][2] = {
wolfSSL	15:117db924cf7c	7850	{0x00, 0x00}, {0x01, 0xc2}, {0x03, 0x84}, {0x02, 0x46},
wolfSSL	15:117db924cf7c	7851	{0x07, 0x08}, {0x06, 0xca}, {0x04, 0x8c}, {0x05, 0x4e},
wolfSSL	15:117db924cf7c	7852	{0x0e, 0x10}, {0x0f, 0xd2}, {0x0d, 0x94}, {0x0c, 0x56},
wolfSSL	15:117db924cf7c	7853	{0x09, 0x18}, {0x08, 0xda}, {0x0a, 0x9c}, {0x0b, 0x5e},
wolfSSL	15:117db924cf7c	7854	{0x1c, 0x20}, {0x1d, 0xe2}, {0x1f, 0xa4}, {0x1e, 0x66},
wolfSSL	15:117db924cf7c	7855	{0x1b, 0x28}, {0x1a, 0xea}, {0x18, 0xac}, {0x19, 0x6e},
wolfSSL	15:117db924cf7c	7856	{0x12, 0x30}, {0x13, 0xf2}, {0x11, 0xb4}, {0x10, 0x76},
wolfSSL	15:117db924cf7c	7857	{0x15, 0x38}, {0x14, 0xfa}, {0x16, 0xbc}, {0x17, 0x7e},
wolfSSL	15:117db924cf7c	7858	{0x38, 0x40}, {0x39, 0x82}, {0x3b, 0xc4}, {0x3a, 0x06},
wolfSSL	15:117db924cf7c	7859	{0x3f, 0x48}, {0x3e, 0x8a}, {0x3c, 0xcc}, {0x3d, 0x0e},
wolfSSL	15:117db924cf7c	7860	{0x36, 0x50}, {0x37, 0x92}, {0x35, 0xd4}, {0x34, 0x16},
wolfSSL	15:117db924cf7c	7861	{0x31, 0x58}, {0x30, 0x9a}, {0x32, 0xdc}, {0x33, 0x1e},
wolfSSL	15:117db924cf7c	7862	{0x24, 0x60}, {0x25, 0xa2}, {0x27, 0xe4}, {0x26, 0x26},
wolfSSL	15:117db924cf7c	7863	{0x23, 0x68}, {0x22, 0xaa}, {0x20, 0xec}, {0x21, 0x2e},
wolfSSL	15:117db924cf7c	7864	{0x2a, 0x70}, {0x2b, 0xb2}, {0x29, 0xf4}, {0x28, 0x36},
wolfSSL	15:117db924cf7c	7865	{0x2d, 0x78}, {0x2c, 0xba}, {0x2e, 0xfc}, {0x2f, 0x3e},
wolfSSL	15:117db924cf7c	7866	{0x70, 0x80}, {0x71, 0x42}, {0x73, 0x04}, {0x72, 0xc6},
wolfSSL	15:117db924cf7c	7867	{0x77, 0x88}, {0x76, 0x4a}, {0x74, 0x0c}, {0x75, 0xce},
wolfSSL	15:117db924cf7c	7868	{0x7e, 0x90}, {0x7f, 0x52}, {0x7d, 0x14}, {0x7c, 0xd6},
wolfSSL	15:117db924cf7c	7869	{0x79, 0x98}, {0x78, 0x5a}, {0x7a, 0x1c}, {0x7b, 0xde},
wolfSSL	15:117db924cf7c	7870	{0x6c, 0xa0}, {0x6d, 0x62}, {0x6f, 0x24}, {0x6e, 0xe6},
wolfSSL	15:117db924cf7c	7871	{0x6b, 0xa8}, {0x6a, 0x6a}, {0x68, 0x2c}, {0x69, 0xee},
wolfSSL	15:117db924cf7c	7872	{0x62, 0xb0}, {0x63, 0x72}, {0x61, 0x34}, {0x60, 0xf6},
wolfSSL	15:117db924cf7c	7873	{0x65, 0xb8}, {0x64, 0x7a}, {0x66, 0x3c}, {0x67, 0xfe},
wolfSSL	15:117db924cf7c	7874	{0x48, 0xc0}, {0x49, 0x02}, {0x4b, 0x44}, {0x4a, 0x86},
wolfSSL	15:117db924cf7c	7875	{0x4f, 0xc8}, {0x4e, 0x0a}, {0x4c, 0x4c}, {0x4d, 0x8e},
wolfSSL	15:117db924cf7c	7876	{0x46, 0xd0}, {0x47, 0x12}, {0x45, 0x54}, {0x44, 0x96},
wolfSSL	15:117db924cf7c	7877	{0x41, 0xd8}, {0x40, 0x1a}, {0x42, 0x5c}, {0x43, 0x9e},
wolfSSL	15:117db924cf7c	7878	{0x54, 0xe0}, {0x55, 0x22}, {0x57, 0x64}, {0x56, 0xa6},
wolfSSL	15:117db924cf7c	7879	{0x53, 0xe8}, {0x52, 0x2a}, {0x50, 0x6c}, {0x51, 0xae},
wolfSSL	15:117db924cf7c	7880	{0x5a, 0xf0}, {0x5b, 0x32}, {0x59, 0x74}, {0x58, 0xb6},
wolfSSL	15:117db924cf7c	7881	{0x5d, 0xf8}, {0x5c, 0x3a}, {0x5e, 0x7c}, {0x5f, 0xbe},
wolfSSL	15:117db924cf7c	7882	{0xe1, 0x00}, {0xe0, 0xc2}, {0xe2, 0x84}, {0xe3, 0x46},
wolfSSL	15:117db924cf7c	7883	{0xe6, 0x08}, {0xe7, 0xca}, {0xe5, 0x8c}, {0xe4, 0x4e},
wolfSSL	15:117db924cf7c	7884	{0xef, 0x10}, {0xee, 0xd2}, {0xec, 0x94}, {0xed, 0x56},
wolfSSL	15:117db924cf7c	7885	{0xe8, 0x18}, {0xe9, 0xda}, {0xeb, 0x9c}, {0xea, 0x5e},
wolfSSL	15:117db924cf7c	7886	{0xfd, 0x20}, {0xfc, 0xe2}, {0xfe, 0xa4}, {0xff, 0x66},
wolfSSL	15:117db924cf7c	7887	{0xfa, 0x28}, {0xfb, 0xea}, {0xf9, 0xac}, {0xf8, 0x6e},
wolfSSL	15:117db924cf7c	7888	{0xf3, 0x30}, {0xf2, 0xf2}, {0xf0, 0xb4}, {0xf1, 0x76},
wolfSSL	15:117db924cf7c	7889	{0xf4, 0x38}, {0xf5, 0xfa}, {0xf7, 0xbc}, {0xf6, 0x7e},
wolfSSL	15:117db924cf7c	7890	{0xd9, 0x40}, {0xd8, 0x82}, {0xda, 0xc4}, {0xdb, 0x06},
wolfSSL	15:117db924cf7c	7891	{0xde, 0x48}, {0xdf, 0x8a}, {0xdd, 0xcc}, {0xdc, 0x0e},
wolfSSL	15:117db924cf7c	7892	{0xd7, 0x50}, {0xd6, 0x92}, {0xd4, 0xd4}, {0xd5, 0x16},
wolfSSL	15:117db924cf7c	7893	{0xd0, 0x58}, {0xd1, 0x9a}, {0xd3, 0xdc}, {0xd2, 0x1e},
wolfSSL	15:117db924cf7c	7894	{0xc5, 0x60}, {0xc4, 0xa2}, {0xc6, 0xe4}, {0xc7, 0x26},
wolfSSL	15:117db924cf7c	7895	{0xc2, 0x68}, {0xc3, 0xaa}, {0xc1, 0xec}, {0xc0, 0x2e},
wolfSSL	15:117db924cf7c	7896	{0xcb, 0x70}, {0xca, 0xb2}, {0xc8, 0xf4}, {0xc9, 0x36},
wolfSSL	15:117db924cf7c	7897	{0xcc, 0x78}, {0xcd, 0xba}, {0xcf, 0xfc}, {0xce, 0x3e},
wolfSSL	15:117db924cf7c	7898	{0x91, 0x80}, {0x90, 0x42}, {0x92, 0x04}, {0x93, 0xc6},
wolfSSL	15:117db924cf7c	7899	{0x96, 0x88}, {0x97, 0x4a}, {0x95, 0x0c}, {0x94, 0xce},
wolfSSL	15:117db924cf7c	7900	{0x9f, 0x90}, {0x9e, 0x52}, {0x9c, 0x14}, {0x9d, 0xd6},
wolfSSL	15:117db924cf7c	7901	{0x98, 0x98}, {0x99, 0x5a}, {0x9b, 0x1c}, {0x9a, 0xde},
wolfSSL	15:117db924cf7c	7902	{0x8d, 0xa0}, {0x8c, 0x62}, {0x8e, 0x24}, {0x8f, 0xe6},
wolfSSL	15:117db924cf7c	7903	{0x8a, 0xa8}, {0x8b, 0x6a}, {0x89, 0x2c}, {0x88, 0xee},
wolfSSL	15:117db924cf7c	7904	{0x83, 0xb0}, {0x82, 0x72}, {0x80, 0x34}, {0x81, 0xf6},
wolfSSL	15:117db924cf7c	7905	{0x84, 0xb8}, {0x85, 0x7a}, {0x87, 0x3c}, {0x86, 0xfe},
wolfSSL	15:117db924cf7c	7906	{0xa9, 0xc0}, {0xa8, 0x02}, {0xaa, 0x44}, {0xab, 0x86},
wolfSSL	15:117db924cf7c	7907	{0xae, 0xc8}, {0xaf, 0x0a}, {0xad, 0x4c}, {0xac, 0x8e},
wolfSSL	15:117db924cf7c	7908	{0xa7, 0xd0}, {0xa6, 0x12}, {0xa4, 0x54}, {0xa5, 0x96},
wolfSSL	15:117db924cf7c	7909	{0xa0, 0xd8}, {0xa1, 0x1a}, {0xa3, 0x5c}, {0xa2, 0x9e},
wolfSSL	15:117db924cf7c	7910	{0xb5, 0xe0}, {0xb4, 0x22}, {0xb6, 0x64}, {0xb7, 0xa6},
wolfSSL	15:117db924cf7c	7911	{0xb2, 0xe8}, {0xb3, 0x2a}, {0xb1, 0x6c}, {0xb0, 0xae},
wolfSSL	15:117db924cf7c	7912	{0xbb, 0xf0}, {0xba, 0x32}, {0xb8, 0x74}, {0xb9, 0xb6},
wolfSSL	15:117db924cf7c	7913	{0xbc, 0xf8}, {0xbd, 0x3a}, {0xbf, 0x7c}, {0xbe, 0xbe} };
wolfSSL	15:117db924cf7c	7914
wolfSSL	15:117db924cf7c	7915
wolfSSL	15:117db924cf7c	7916	static void GMULT(byte *x, byte m[256][AES_BLOCK_SIZE])
wolfSSL	15:117db924cf7c	7917	{
wolfSSL	15:117db924cf7c	7918	int i, j;
wolfSSL	15:117db924cf7c	7919	byte Z[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	7920	byte a;
wolfSSL	15:117db924cf7c	7921
wolfSSL	15:117db924cf7c	7922	XMEMSET(Z, 0, sizeof(Z));
wolfSSL	15:117db924cf7c	7923
wolfSSL	15:117db924cf7c	7924	for (i = 15; i > 0; i--) {
wolfSSL	15:117db924cf7c	7925	xorbuf(Z, m[x[i]], AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7926	a = Z[15];
wolfSSL	15:117db924cf7c	7927
wolfSSL	15:117db924cf7c	7928	for (j = 15; j > 0; j--) {
wolfSSL	15:117db924cf7c	7929	Z[j] = Z[j-1];
wolfSSL	15:117db924cf7c	7930	}
wolfSSL	15:117db924cf7c	7931
wolfSSL	15:117db924cf7c	7932	Z[0] = R[a][0];
wolfSSL	15:117db924cf7c	7933	Z[1] ^= R[a][1];
wolfSSL	15:117db924cf7c	7934	}
wolfSSL	15:117db924cf7c	7935	xorbuf(Z, m[x[0]], AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7936
wolfSSL	15:117db924cf7c	7937	XMEMCPY(x, Z, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7938	}
wolfSSL	15:117db924cf7c	7939
wolfSSL	15:117db924cf7c	7940
wolfSSL	15:117db924cf7c	7941	void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
wolfSSL	15:117db924cf7c	7942	word32 cSz, byte* s, word32 sSz)
wolfSSL	15:117db924cf7c	7943	{
wolfSSL	15:117db924cf7c	7944	byte x[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	7945	byte scratch[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	7946	word32 blocks, partial;
wolfSSL	15:117db924cf7c	7947
wolfSSL	15:117db924cf7c	7948	XMEMSET(x, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7949
wolfSSL	15:117db924cf7c	7950	/* Hash in A, the Additional Authentication Data */
wolfSSL	15:117db924cf7c	7951	if (aSz != 0 && a != NULL) {
wolfSSL	15:117db924cf7c	7952	blocks = aSz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7953	partial = aSz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7954	while (blocks--) {
wolfSSL	15:117db924cf7c	7955	xorbuf(x, a, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7956	GMULT(x, aes->M0);
wolfSSL	15:117db924cf7c	7957	a += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7958	}
wolfSSL	15:117db924cf7c	7959	if (partial != 0) {
wolfSSL	15:117db924cf7c	7960	XMEMSET(scratch, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7961	XMEMCPY(scratch, a, partial);
wolfSSL	15:117db924cf7c	7962	xorbuf(x, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7963	GMULT(x, aes->M0);
wolfSSL	15:117db924cf7c	7964	}
wolfSSL	15:117db924cf7c	7965	}
wolfSSL	15:117db924cf7c	7966
wolfSSL	15:117db924cf7c	7967	/* Hash in C, the Ciphertext */
wolfSSL	15:117db924cf7c	7968	if (cSz != 0 && c != NULL) {
wolfSSL	15:117db924cf7c	7969	blocks = cSz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7970	partial = cSz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7971	while (blocks--) {
wolfSSL	15:117db924cf7c	7972	xorbuf(x, c, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7973	GMULT(x, aes->M0);
wolfSSL	15:117db924cf7c	7974	c += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	7975	}
wolfSSL	15:117db924cf7c	7976	if (partial != 0) {
wolfSSL	15:117db924cf7c	7977	XMEMSET(scratch, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7978	XMEMCPY(scratch, c, partial);
wolfSSL	15:117db924cf7c	7979	xorbuf(x, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7980	GMULT(x, aes->M0);
wolfSSL	15:117db924cf7c	7981	}
wolfSSL	15:117db924cf7c	7982	}
wolfSSL	15:117db924cf7c	7983
wolfSSL	15:117db924cf7c	7984	/* Hash in the lengths of A and C in bits */
wolfSSL	15:117db924cf7c	7985	FlattenSzInBits(&scratch[0], aSz);
wolfSSL	15:117db924cf7c	7986	FlattenSzInBits(&scratch[8], cSz);
wolfSSL	15:117db924cf7c	7987	xorbuf(x, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	7988	GMULT(x, aes->M0);
wolfSSL	15:117db924cf7c	7989
wolfSSL	15:117db924cf7c	7990	/* Copy the result into s. */
wolfSSL	15:117db924cf7c	7991	XMEMCPY(s, x, sSz);
wolfSSL	15:117db924cf7c	7992	}
wolfSSL	15:117db924cf7c	7993
wolfSSL	15:117db924cf7c	7994	/* end GCM_TABLE */
wolfSSL	15:117db924cf7c	7995	#elif defined(WORD64_AVAILABLE) && !defined(GCM_WORD32)
wolfSSL	15:117db924cf7c	7996
wolfSSL	15:117db924cf7c	7997	#if !defined(FREESCALE_LTC_AES_GCM)
wolfSSL	15:117db924cf7c	7998	static void GMULT(word64* X, word64* Y)
wolfSSL	15:117db924cf7c	7999	{
wolfSSL	15:117db924cf7c	8000	word64 Z[2] = {0,0};
wolfSSL	15:117db924cf7c	8001	word64 V[2];
wolfSSL	15:117db924cf7c	8002	int i, j;
wolfSSL	15:117db924cf7c	8003	V[0] = X[0]; V[1] = X[1];
wolfSSL	15:117db924cf7c	8004
wolfSSL	15:117db924cf7c	8005	for (i = 0; i < 2; i++)
wolfSSL	15:117db924cf7c	8006	{
wolfSSL	15:117db924cf7c	8007	word64 y = Y[i];
wolfSSL	15:117db924cf7c	8008	for (j = 0; j < 64; j++)
wolfSSL	15:117db924cf7c	8009	{
wolfSSL	15:117db924cf7c	8010	if (y & 0x8000000000000000ULL) {
wolfSSL	15:117db924cf7c	8011	Z[0] ^= V[0];
wolfSSL	15:117db924cf7c	8012	Z[1] ^= V[1];
wolfSSL	15:117db924cf7c	8013	}
wolfSSL	15:117db924cf7c	8014
wolfSSL	15:117db924cf7c	8015	if (V[1] & 0x0000000000000001) {
wolfSSL	15:117db924cf7c	8016	V[1] >>= 1;
wolfSSL	15:117db924cf7c	8017	V[1] \|= ((V[0] & 0x0000000000000001) ?
wolfSSL	15:117db924cf7c	8018	0x8000000000000000ULL : 0);
wolfSSL	15:117db924cf7c	8019	V[0] >>= 1;
wolfSSL	15:117db924cf7c	8020	V[0] ^= 0xE100000000000000ULL;
wolfSSL	15:117db924cf7c	8021	}
wolfSSL	15:117db924cf7c	8022	else {
wolfSSL	15:117db924cf7c	8023	V[1] >>= 1;
wolfSSL	15:117db924cf7c	8024	V[1] \|= ((V[0] & 0x0000000000000001) ?
wolfSSL	15:117db924cf7c	8025	0x8000000000000000ULL : 0);
wolfSSL	15:117db924cf7c	8026	V[0] >>= 1;
wolfSSL	15:117db924cf7c	8027	}
wolfSSL	15:117db924cf7c	8028	y <<= 1;
wolfSSL	15:117db924cf7c	8029	}
wolfSSL	15:117db924cf7c	8030	}
wolfSSL	15:117db924cf7c	8031	X[0] = Z[0];
wolfSSL	15:117db924cf7c	8032	X[1] = Z[1];
wolfSSL	15:117db924cf7c	8033	}
wolfSSL	15:117db924cf7c	8034
wolfSSL	15:117db924cf7c	8035
wolfSSL	15:117db924cf7c	8036	void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
wolfSSL	15:117db924cf7c	8037	word32 cSz, byte* s, word32 sSz)
wolfSSL	15:117db924cf7c	8038	{
wolfSSL	15:117db924cf7c	8039	word64 x[2] = {0,0};
wolfSSL	15:117db924cf7c	8040	word32 blocks, partial;
wolfSSL	15:117db924cf7c	8041	word64 bigH[2];
wolfSSL	15:117db924cf7c	8042
wolfSSL	15:117db924cf7c	8043	XMEMCPY(bigH, aes->H, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8044	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8045	ByteReverseWords64(bigH, bigH, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8046	#endif
wolfSSL	15:117db924cf7c	8047
wolfSSL	15:117db924cf7c	8048	/* Hash in A, the Additional Authentication Data */
wolfSSL	15:117db924cf7c	8049	if (aSz != 0 && a != NULL) {
wolfSSL	15:117db924cf7c	8050	word64 bigA[2];
wolfSSL	15:117db924cf7c	8051	blocks = aSz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8052	partial = aSz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8053	while (blocks--) {
wolfSSL	15:117db924cf7c	8054	XMEMCPY(bigA, a, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8055	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8056	ByteReverseWords64(bigA, bigA, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8057	#endif
wolfSSL	15:117db924cf7c	8058	x[0] ^= bigA[0];
wolfSSL	15:117db924cf7c	8059	x[1] ^= bigA[1];
wolfSSL	15:117db924cf7c	8060	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8061	a += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8062	}
wolfSSL	15:117db924cf7c	8063	if (partial != 0) {
wolfSSL	15:117db924cf7c	8064	XMEMSET(bigA, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8065	XMEMCPY(bigA, a, partial);
wolfSSL	15:117db924cf7c	8066	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8067	ByteReverseWords64(bigA, bigA, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8068	#endif
wolfSSL	15:117db924cf7c	8069	x[0] ^= bigA[0];
wolfSSL	15:117db924cf7c	8070	x[1] ^= bigA[1];
wolfSSL	15:117db924cf7c	8071	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8072	}
wolfSSL	15:117db924cf7c	8073	}
wolfSSL	15:117db924cf7c	8074
wolfSSL	15:117db924cf7c	8075	/* Hash in C, the Ciphertext */
wolfSSL	15:117db924cf7c	8076	if (cSz != 0 && c != NULL) {
wolfSSL	15:117db924cf7c	8077	word64 bigC[2];
wolfSSL	15:117db924cf7c	8078	blocks = cSz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8079	partial = cSz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8080	while (blocks--) {
wolfSSL	15:117db924cf7c	8081	XMEMCPY(bigC, c, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8082	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8083	ByteReverseWords64(bigC, bigC, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8084	#endif
wolfSSL	15:117db924cf7c	8085	x[0] ^= bigC[0];
wolfSSL	15:117db924cf7c	8086	x[1] ^= bigC[1];
wolfSSL	15:117db924cf7c	8087	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8088	c += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8089	}
wolfSSL	15:117db924cf7c	8090	if (partial != 0) {
wolfSSL	15:117db924cf7c	8091	XMEMSET(bigC, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8092	XMEMCPY(bigC, c, partial);
wolfSSL	15:117db924cf7c	8093	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8094	ByteReverseWords64(bigC, bigC, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8095	#endif
wolfSSL	15:117db924cf7c	8096	x[0] ^= bigC[0];
wolfSSL	15:117db924cf7c	8097	x[1] ^= bigC[1];
wolfSSL	15:117db924cf7c	8098	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8099	}
wolfSSL	15:117db924cf7c	8100	}
wolfSSL	15:117db924cf7c	8101
wolfSSL	15:117db924cf7c	8102	/* Hash in the lengths in bits of A and C */
wolfSSL	15:117db924cf7c	8103	{
wolfSSL	15:117db924cf7c	8104	word64 len[2];
wolfSSL	15:117db924cf7c	8105	len[0] = aSz; len[1] = cSz;
wolfSSL	15:117db924cf7c	8106
wolfSSL	15:117db924cf7c	8107	/* Lengths are in bytes. Convert to bits. */
wolfSSL	15:117db924cf7c	8108	len[0] *= 8;
wolfSSL	15:117db924cf7c	8109	len[1] *= 8;
wolfSSL	15:117db924cf7c	8110
wolfSSL	15:117db924cf7c	8111	x[0] ^= len[0];
wolfSSL	15:117db924cf7c	8112	x[1] ^= len[1];
wolfSSL	15:117db924cf7c	8113	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8114	}
wolfSSL	15:117db924cf7c	8115	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8116	ByteReverseWords64(x, x, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8117	#endif
wolfSSL	15:117db924cf7c	8118	XMEMCPY(s, x, sSz);
wolfSSL	15:117db924cf7c	8119	}
wolfSSL	15:117db924cf7c	8120	#endif /* !FREESCALE_LTC_AES_GCM */
wolfSSL	15:117db924cf7c	8121
wolfSSL	15:117db924cf7c	8122	/* end defined(WORD64_AVAILABLE) && !defined(GCM_WORD32) */
wolfSSL	15:117db924cf7c	8123	#else /* GCM_WORD32 */
wolfSSL	15:117db924cf7c	8124
wolfSSL	15:117db924cf7c	8125	static void GMULT(word32* X, word32* Y)
wolfSSL	15:117db924cf7c	8126	{
wolfSSL	15:117db924cf7c	8127	word32 Z[4] = {0,0,0,0};
wolfSSL	15:117db924cf7c	8128	word32 V[4];
wolfSSL	15:117db924cf7c	8129	int i, j;
wolfSSL	15:117db924cf7c	8130
wolfSSL	15:117db924cf7c	8131	V[0] = X[0]; V[1] = X[1]; V[2] = X[2]; V[3] = X[3];
wolfSSL	15:117db924cf7c	8132
wolfSSL	15:117db924cf7c	8133	for (i = 0; i < 4; i++)
wolfSSL	15:117db924cf7c	8134	{
wolfSSL	15:117db924cf7c	8135	word32 y = Y[i];
wolfSSL	15:117db924cf7c	8136	for (j = 0; j < 32; j++)
wolfSSL	15:117db924cf7c	8137	{
wolfSSL	15:117db924cf7c	8138	if (y & 0x80000000) {
wolfSSL	15:117db924cf7c	8139	Z[0] ^= V[0];
wolfSSL	15:117db924cf7c	8140	Z[1] ^= V[1];
wolfSSL	15:117db924cf7c	8141	Z[2] ^= V[2];
wolfSSL	15:117db924cf7c	8142	Z[3] ^= V[3];
wolfSSL	15:117db924cf7c	8143	}
wolfSSL	15:117db924cf7c	8144
wolfSSL	15:117db924cf7c	8145	if (V[3] & 0x00000001) {
wolfSSL	15:117db924cf7c	8146	V[3] >>= 1;
wolfSSL	15:117db924cf7c	8147	V[3] \|= ((V[2] & 0x00000001) ? 0x80000000 : 0);
wolfSSL	15:117db924cf7c	8148	V[2] >>= 1;
wolfSSL	15:117db924cf7c	8149	V[2] \|= ((V[1] & 0x00000001) ? 0x80000000 : 0);
wolfSSL	15:117db924cf7c	8150	V[1] >>= 1;
wolfSSL	15:117db924cf7c	8151	V[1] \|= ((V[0] & 0x00000001) ? 0x80000000 : 0);
wolfSSL	15:117db924cf7c	8152	V[0] >>= 1;
wolfSSL	15:117db924cf7c	8153	V[0] ^= 0xE1000000;
wolfSSL	15:117db924cf7c	8154	} else {
wolfSSL	15:117db924cf7c	8155	V[3] >>= 1;
wolfSSL	15:117db924cf7c	8156	V[3] \|= ((V[2] & 0x00000001) ? 0x80000000 : 0);
wolfSSL	15:117db924cf7c	8157	V[2] >>= 1;
wolfSSL	15:117db924cf7c	8158	V[2] \|= ((V[1] & 0x00000001) ? 0x80000000 : 0);
wolfSSL	15:117db924cf7c	8159	V[1] >>= 1;
wolfSSL	15:117db924cf7c	8160	V[1] \|= ((V[0] & 0x00000001) ? 0x80000000 : 0);
wolfSSL	15:117db924cf7c	8161	V[0] >>= 1;
wolfSSL	15:117db924cf7c	8162	}
wolfSSL	15:117db924cf7c	8163	y <<= 1;
wolfSSL	15:117db924cf7c	8164	}
wolfSSL	15:117db924cf7c	8165	}
wolfSSL	15:117db924cf7c	8166	X[0] = Z[0];
wolfSSL	15:117db924cf7c	8167	X[1] = Z[1];
wolfSSL	15:117db924cf7c	8168	X[2] = Z[2];
wolfSSL	15:117db924cf7c	8169	X[3] = Z[3];
wolfSSL	15:117db924cf7c	8170	}
wolfSSL	15:117db924cf7c	8171
wolfSSL	15:117db924cf7c	8172
wolfSSL	15:117db924cf7c	8173	void GHASH(Aes* aes, const byte* a, word32 aSz, const byte* c,
wolfSSL	15:117db924cf7c	8174	word32 cSz, byte* s, word32 sSz)
wolfSSL	15:117db924cf7c	8175	{
wolfSSL	15:117db924cf7c	8176	word32 x[4] = {0,0,0,0};
wolfSSL	15:117db924cf7c	8177	word32 blocks, partial;
wolfSSL	15:117db924cf7c	8178	word32 bigH[4];
wolfSSL	15:117db924cf7c	8179
wolfSSL	15:117db924cf7c	8180	XMEMCPY(bigH, aes->H, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8181	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8182	ByteReverseWords(bigH, bigH, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8183	#endif
wolfSSL	15:117db924cf7c	8184
wolfSSL	15:117db924cf7c	8185	/* Hash in A, the Additional Authentication Data */
wolfSSL	15:117db924cf7c	8186	if (aSz != 0 && a != NULL) {
wolfSSL	15:117db924cf7c	8187	word32 bigA[4];
wolfSSL	15:117db924cf7c	8188	blocks = aSz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8189	partial = aSz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8190	while (blocks--) {
wolfSSL	15:117db924cf7c	8191	XMEMCPY(bigA, a, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8192	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8193	ByteReverseWords(bigA, bigA, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8194	#endif
wolfSSL	15:117db924cf7c	8195	x[0] ^= bigA[0];
wolfSSL	15:117db924cf7c	8196	x[1] ^= bigA[1];
wolfSSL	15:117db924cf7c	8197	x[2] ^= bigA[2];
wolfSSL	15:117db924cf7c	8198	x[3] ^= bigA[3];
wolfSSL	15:117db924cf7c	8199	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8200	a += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8201	}
wolfSSL	15:117db924cf7c	8202	if (partial != 0) {
wolfSSL	15:117db924cf7c	8203	XMEMSET(bigA, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8204	XMEMCPY(bigA, a, partial);
wolfSSL	15:117db924cf7c	8205	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8206	ByteReverseWords(bigA, bigA, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8207	#endif
wolfSSL	15:117db924cf7c	8208	x[0] ^= bigA[0];
wolfSSL	15:117db924cf7c	8209	x[1] ^= bigA[1];
wolfSSL	15:117db924cf7c	8210	x[2] ^= bigA[2];
wolfSSL	15:117db924cf7c	8211	x[3] ^= bigA[3];
wolfSSL	15:117db924cf7c	8212	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8213	}
wolfSSL	15:117db924cf7c	8214	}
wolfSSL	15:117db924cf7c	8215
wolfSSL	15:117db924cf7c	8216	/* Hash in C, the Ciphertext */
wolfSSL	15:117db924cf7c	8217	if (cSz != 0 && c != NULL) {
wolfSSL	15:117db924cf7c	8218	word32 bigC[4];
wolfSSL	15:117db924cf7c	8219	blocks = cSz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8220	partial = cSz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8221	while (blocks--) {
wolfSSL	15:117db924cf7c	8222	XMEMCPY(bigC, c, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8223	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8224	ByteReverseWords(bigC, bigC, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8225	#endif
wolfSSL	15:117db924cf7c	8226	x[0] ^= bigC[0];
wolfSSL	15:117db924cf7c	8227	x[1] ^= bigC[1];
wolfSSL	15:117db924cf7c	8228	x[2] ^= bigC[2];
wolfSSL	15:117db924cf7c	8229	x[3] ^= bigC[3];
wolfSSL	15:117db924cf7c	8230	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8231	c += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8232	}
wolfSSL	15:117db924cf7c	8233	if (partial != 0) {
wolfSSL	15:117db924cf7c	8234	XMEMSET(bigC, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8235	XMEMCPY(bigC, c, partial);
wolfSSL	15:117db924cf7c	8236	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8237	ByteReverseWords(bigC, bigC, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8238	#endif
wolfSSL	15:117db924cf7c	8239	x[0] ^= bigC[0];
wolfSSL	15:117db924cf7c	8240	x[1] ^= bigC[1];
wolfSSL	15:117db924cf7c	8241	x[2] ^= bigC[2];
wolfSSL	15:117db924cf7c	8242	x[3] ^= bigC[3];
wolfSSL	15:117db924cf7c	8243	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8244	}
wolfSSL	15:117db924cf7c	8245	}
wolfSSL	15:117db924cf7c	8246
wolfSSL	15:117db924cf7c	8247	/* Hash in the lengths in bits of A and C */
wolfSSL	15:117db924cf7c	8248	{
wolfSSL	15:117db924cf7c	8249	word32 len[4];
wolfSSL	15:117db924cf7c	8250
wolfSSL	15:117db924cf7c	8251	/* Lengths are in bytes. Convert to bits. */
wolfSSL	15:117db924cf7c	8252	len[0] = (aSz >> (8*sizeof(aSz) - 3));
wolfSSL	15:117db924cf7c	8253	len[1] = aSz << 3;
wolfSSL	15:117db924cf7c	8254	len[2] = (cSz >> (8*sizeof(cSz) - 3));
wolfSSL	15:117db924cf7c	8255	len[3] = cSz << 3;
wolfSSL	15:117db924cf7c	8256
wolfSSL	15:117db924cf7c	8257	x[0] ^= len[0];
wolfSSL	15:117db924cf7c	8258	x[1] ^= len[1];
wolfSSL	15:117db924cf7c	8259	x[2] ^= len[2];
wolfSSL	15:117db924cf7c	8260	x[3] ^= len[3];
wolfSSL	15:117db924cf7c	8261	GMULT(x, bigH);
wolfSSL	15:117db924cf7c	8262	}
wolfSSL	15:117db924cf7c	8263	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	8264	ByteReverseWords(x, x, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8265	#endif
wolfSSL	15:117db924cf7c	8266	XMEMCPY(s, x, sSz);
wolfSSL	15:117db924cf7c	8267	}
wolfSSL	15:117db924cf7c	8268
wolfSSL	15:117db924cf7c	8269	#endif /* end GCM_WORD32 */
wolfSSL	15:117db924cf7c	8270
wolfSSL	15:117db924cf7c	8271
wolfSSL	15:117db924cf7c	8272	#if !defined(WOLFSSL_XILINX_CRYPT)
wolfSSL	15:117db924cf7c	8273	#ifdef FREESCALE_LTC_AES_GCM
wolfSSL	15:117db924cf7c	8274	int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8275	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8276	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8277	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	8278	{
wolfSSL	15:117db924cf7c	8279	status_t status;
wolfSSL	15:117db924cf7c	8280	word32 keySize;
wolfSSL	15:117db924cf7c	8281
wolfSSL	15:117db924cf7c	8282	/* argument checks */
wolfSSL	15:117db924cf7c	8283	if (aes == NULL \|\| authTagSz > AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	8284	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8285	}
wolfSSL	15:117db924cf7c	8286
wolfSSL	15:117db924cf7c	8287	if (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
wolfSSL	15:117db924cf7c	8288	WOLFSSL_MSG("GcmEncrypt authTagSz too small error");
wolfSSL	15:117db924cf7c	8289	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8290	}
wolfSSL	15:117db924cf7c	8291
wolfSSL	15:117db924cf7c	8292	status = wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	8293	if (status)
wolfSSL	15:117db924cf7c	8294	return status;
wolfSSL	15:117db924cf7c	8295
wolfSSL	15:117db924cf7c	8296	status = LTC_AES_EncryptTagGcm(LTC_BASE, in, out, sz, iv, ivSz,
wolfSSL	15:117db924cf7c	8297	authIn, authInSz, (byte*)aes->key, keySize, authTag, authTagSz);
wolfSSL	15:117db924cf7c	8298
wolfSSL	15:117db924cf7c	8299	return (status == kStatus_Success) ? 0 : AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	8300	}
wolfSSL	15:117db924cf7c	8301	#else
wolfSSL	15:117db924cf7c	8302	#if defined(STM32_CRYPTO) && (defined(WOLFSSL_STM32F4) \|\| \
wolfSSL	15:117db924cf7c	8303	defined(WOLFSSL_STM32F7) \|\| \
wolfSSL	15:117db924cf7c	8304	defined(WOLFSSL_STM32L4))
wolfSSL	15:117db924cf7c	8305
wolfSSL	15:117db924cf7c	8306	static WC_INLINE int wc_AesGcmEncrypt_STM32(Aes* aes, byte* out, const byte* in,
wolfSSL	15:117db924cf7c	8307	word32 sz, const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8308	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8309	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	8310	{
wolfSSL	15:117db924cf7c	8311	int ret;
wolfSSL	15:117db924cf7c	8312	word32 keySize;
wolfSSL	15:117db924cf7c	8313	byte initialCounter[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8314	#ifdef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	8315	CRYP_HandleTypeDef hcryp;
wolfSSL	15:117db924cf7c	8316	#else
wolfSSL	15:117db924cf7c	8317	byte keyCopy[AES_BLOCK_SIZE * 2];
wolfSSL	15:117db924cf7c	8318	#endif /* WOLFSSL_STM32_CUBEMX */
wolfSSL	15:117db924cf7c	8319	int status = 0;
wolfSSL	15:117db924cf7c	8320	byte* authInPadded = NULL;
wolfSSL	15:117db924cf7c	8321	byte tag[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8322	int authPadSz;
wolfSSL	15:117db924cf7c	8323
wolfSSL	15:117db924cf7c	8324	ret = wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	8325	if (ret != 0)
wolfSSL	15:117db924cf7c	8326	return ret;
wolfSSL	15:117db924cf7c	8327
wolfSSL	15:117db924cf7c	8328	XMEMSET(initialCounter, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8329	XMEMCPY(initialCounter, iv, ivSz);
wolfSSL	15:117db924cf7c	8330	initialCounter[AES_BLOCK_SIZE - 1] = STM32_GCM_IV_START;
wolfSSL	15:117db924cf7c	8331
wolfSSL	15:117db924cf7c	8332	/* pad authIn if it is not a block multiple */
wolfSSL	15:117db924cf7c	8333	if ((authInSz % AES_BLOCK_SIZE) != 0) {
wolfSSL	15:117db924cf7c	8334	authPadSz = ((authInSz / AES_BLOCK_SIZE) + 1) * AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8335	/* Need to pad the AAD to a full block with zeros. */
wolfSSL	15:117db924cf7c	8336	authInPadded = XMALLOC(authPadSz, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	8337	if (authInPadded == NULL) {
wolfSSL	15:117db924cf7c	8338	return MEMORY_E;
wolfSSL	15:117db924cf7c	8339	}
wolfSSL	15:117db924cf7c	8340	XMEMSET(authInPadded, 0, authPadSz);
wolfSSL	15:117db924cf7c	8341	XMEMCPY(authInPadded, authIn, authInSz);
wolfSSL	15:117db924cf7c	8342	} else {
wolfSSL	15:117db924cf7c	8343	authPadSz = authInSz;
wolfSSL	15:117db924cf7c	8344	authInPadded = (byte*)authIn;
wolfSSL	15:117db924cf7c	8345	}
wolfSSL	15:117db924cf7c	8346
wolfSSL	15:117db924cf7c	8347
wolfSSL	15:117db924cf7c	8348	#ifdef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	8349	XMEMSET(&hcryp, 0, sizeof(CRYP_HandleTypeDef));
wolfSSL	15:117db924cf7c	8350	switch (keySize) {
wolfSSL	15:117db924cf7c	8351	case 16: /* 128-bit key */
wolfSSL	15:117db924cf7c	8352	hcryp.Init.KeySize = CRYP_KEYSIZE_128B;
wolfSSL	15:117db924cf7c	8353	break;
wolfSSL	15:117db924cf7c	8354	#ifdef CRYP_KEYSIZE_192B
wolfSSL	15:117db924cf7c	8355	case 24: /* 192-bit key */
wolfSSL	15:117db924cf7c	8356	hcryp.Init.KeySize = CRYP_KEYSIZE_192B;
wolfSSL	15:117db924cf7c	8357	break;
wolfSSL	15:117db924cf7c	8358	#endif
wolfSSL	15:117db924cf7c	8359	case 32: /* 256-bit key */
wolfSSL	15:117db924cf7c	8360	hcryp.Init.KeySize = CRYP_KEYSIZE_256B;
wolfSSL	15:117db924cf7c	8361	break;
wolfSSL	15:117db924cf7c	8362	default:
wolfSSL	15:117db924cf7c	8363	break;
wolfSSL	15:117db924cf7c	8364	}
wolfSSL	15:117db924cf7c	8365	hcryp.Instance = CRYP;
wolfSSL	15:117db924cf7c	8366	hcryp.Init.DataType = CRYP_DATATYPE_8B;
wolfSSL	15:117db924cf7c	8367	hcryp.Init.pKey = (byte*)aes->key;
wolfSSL	15:117db924cf7c	8368	hcryp.Init.pInitVect = initialCounter;
wolfSSL	15:117db924cf7c	8369	hcryp.Init.Header = authInPadded;
wolfSSL	15:117db924cf7c	8370	hcryp.Init.HeaderSize = authInSz;
wolfSSL	15:117db924cf7c	8371
wolfSSL	15:117db924cf7c	8372	#ifdef WOLFSSL_STM32L4
wolfSSL	15:117db924cf7c	8373	/* Set the CRYP parameters */
wolfSSL	15:117db924cf7c	8374	hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_GCM_GMAC;
wolfSSL	15:117db924cf7c	8375	hcryp.Init.OperatingMode = CRYP_ALGOMODE_ENCRYPT;
wolfSSL	15:117db924cf7c	8376	hcryp.Init.GCMCMACPhase = CRYP_INIT_PHASE;
wolfSSL	15:117db924cf7c	8377	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	8378
wolfSSL	15:117db924cf7c	8379	/* GCM init phase */
wolfSSL	15:117db924cf7c	8380	status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8381	if (status == HAL_OK) {
wolfSSL	15:117db924cf7c	8382	/* GCM header phase */
wolfSSL	15:117db924cf7c	8383	hcryp.Init.GCMCMACPhase = CRYP_HEADER_PHASE;
wolfSSL	15:117db924cf7c	8384	status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8385	if (status == HAL_OK) {
wolfSSL	15:117db924cf7c	8386	/* GCM payload phase */
wolfSSL	15:117db924cf7c	8387	hcryp.Init.GCMCMACPhase = CRYP_PAYLOAD_PHASE;
wolfSSL	15:117db924cf7c	8388	status = HAL_CRYPEx_AES_Auth(&hcryp, (byte*)in, sz, out, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8389	if (status == HAL_OK) {
wolfSSL	15:117db924cf7c	8390	/* GCM final phase */
wolfSSL	15:117db924cf7c	8391	hcryp.Init.GCMCMACPhase = CRYP_FINAL_PHASE;
wolfSSL	15:117db924cf7c	8392	status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, sz, tag, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8393	}
wolfSSL	15:117db924cf7c	8394	}
wolfSSL	15:117db924cf7c	8395	}
wolfSSL	15:117db924cf7c	8396	#else
wolfSSL	15:117db924cf7c	8397	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	8398
wolfSSL	15:117db924cf7c	8399	status = HAL_CRYPEx_AESGCM_Encrypt(&hcryp, (byte*)in, sz,
wolfSSL	15:117db924cf7c	8400	out, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8401	/* Compute the authTag */
wolfSSL	15:117db924cf7c	8402	if (status == HAL_OK) {
wolfSSL	15:117db924cf7c	8403	status = HAL_CRYPEx_AESGCM_Finish(&hcryp, sz, tag, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8404	}
wolfSSL	15:117db924cf7c	8405	#endif
wolfSSL	15:117db924cf7c	8406
wolfSSL	15:117db924cf7c	8407	if (status != HAL_OK)
wolfSSL	15:117db924cf7c	8408	ret = AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	8409	HAL_CRYP_DeInit(&hcryp);
wolfSSL	15:117db924cf7c	8410	#else
wolfSSL	15:117db924cf7c	8411	ByteReverseWords((word32)keyCopy, (word32)aes->key, keySize);
wolfSSL	15:117db924cf7c	8412	status = CRYP_AES_GCM(MODE_ENCRYPT, (uint8_t*)initialCounter,
wolfSSL	15:117db924cf7c	8413	(uint8_t)keyCopy, keySize 8,
wolfSSL	15:117db924cf7c	8414	(uint8_t*)in, sz,
wolfSSL	15:117db924cf7c	8415	(uint8_t*)authInPadded,authInSz,
wolfSSL	15:117db924cf7c	8416	(uint8_t*)out, tag);
wolfSSL	15:117db924cf7c	8417	if (status != SUCCESS)
wolfSSL	15:117db924cf7c	8418	ret = AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	8419	#endif /* WOLFSSL_STM32_CUBEMX */
wolfSSL	15:117db924cf7c	8420
wolfSSL	15:117db924cf7c	8421	/* authTag may be shorter than AES_BLOCK_SZ, store separately */
wolfSSL	15:117db924cf7c	8422	if (ret == 0)
wolfSSL	15:117db924cf7c	8423	XMEMCPY(authTag, tag, authTagSz);
wolfSSL	15:117db924cf7c	8424
wolfSSL	15:117db924cf7c	8425	/* We only allocate extra memory if authInPadded is not a multiple of AES_BLOCK_SZ */
wolfSSL	15:117db924cf7c	8426	if (authInPadded != NULL && authInSz != authPadSz) {
wolfSSL	15:117db924cf7c	8427	XFREE(authInPadded, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	8428	}
wolfSSL	15:117db924cf7c	8429
wolfSSL	15:117db924cf7c	8430	return ret;
wolfSSL	15:117db924cf7c	8431	}
wolfSSL	15:117db924cf7c	8432	#endif /* STM32_CRYPTO */
wolfSSL	15:117db924cf7c	8433
wolfSSL	15:117db924cf7c	8434	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	8435	int AES_GCM_encrypt_C(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8436	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8437	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8438	const byte* authIn, word32 authInSz);
wolfSSL	15:117db924cf7c	8439	#else
wolfSSL	15:117db924cf7c	8440	static
wolfSSL	15:117db924cf7c	8441	#endif
wolfSSL	15:117db924cf7c	8442	int AES_GCM_encrypt_C(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8443	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8444	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8445	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	8446	{
wolfSSL	15:117db924cf7c	8447	int ret = 0;
wolfSSL	15:117db924cf7c	8448	word32 blocks = sz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8449	word32 partial = sz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8450	const byte* p = in;
wolfSSL	15:117db924cf7c	8451	byte* c = out;
wolfSSL	15:117db924cf7c	8452	byte counter[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8453	byte initialCounter[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8454	byte *ctr;
wolfSSL	15:117db924cf7c	8455	byte scratch[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8456
wolfSSL	15:117db924cf7c	8457	ctr = counter;
wolfSSL	15:117db924cf7c	8458	XMEMSET(initialCounter, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8459	if (ivSz == GCM_NONCE_MID_SZ) {
wolfSSL	15:117db924cf7c	8460	XMEMCPY(initialCounter, iv, ivSz);
wolfSSL	15:117db924cf7c	8461	initialCounter[AES_BLOCK_SIZE - 1] = 1;
wolfSSL	15:117db924cf7c	8462	}
wolfSSL	15:117db924cf7c	8463	else {
wolfSSL	15:117db924cf7c	8464	GHASH(aes, NULL, 0, iv, ivSz, initialCounter, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8465	}
wolfSSL	15:117db924cf7c	8466	XMEMCPY(ctr, initialCounter, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8467
wolfSSL	15:117db924cf7c	8468	#ifdef WOLFSSL_PIC32MZ_CRYPT
wolfSSL	15:117db924cf7c	8469	if (blocks) {
wolfSSL	15:117db924cf7c	8470	/* use intitial IV for PIC32 HW, but don't use it below */
wolfSSL	15:117db924cf7c	8471	XMEMCPY(aes->reg, ctr, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8472
wolfSSL	15:117db924cf7c	8473	ret = wc_Pic32AesCrypt(
wolfSSL	15:117db924cf7c	8474	aes->key, aes->keylen, aes->reg, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	8475	out, in, (blocks * AES_BLOCK_SIZE),
wolfSSL	15:117db924cf7c	8476	PIC32_ENCRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_AES_GCM);
wolfSSL	15:117db924cf7c	8477	if (ret != 0)
wolfSSL	15:117db924cf7c	8478	return ret;
wolfSSL	15:117db924cf7c	8479	}
wolfSSL	15:117db924cf7c	8480	/* process remainder using partial handling */
wolfSSL	15:117db924cf7c	8481	#endif
wolfSSL	15:117db924cf7c	8482
wolfSSL	15:117db924cf7c	8483	#if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT)
wolfSSL	15:117db924cf7c	8484	/* some hardware acceleration can gain performance from doing AES encryption
wolfSSL	15:117db924cf7c	8485	* of the whole buffer at once */
wolfSSL	15:117db924cf7c	8486	if (c != p) { /* can not handle inline encryption */
wolfSSL	15:117db924cf7c	8487	while (blocks--) {
wolfSSL	15:117db924cf7c	8488	IncrementGcmCounter(ctr);
wolfSSL	15:117db924cf7c	8489	XMEMCPY(c, ctr, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8490	c += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8491	}
wolfSSL	15:117db924cf7c	8492
wolfSSL	15:117db924cf7c	8493	/* reset number of blocks and then do encryption */
wolfSSL	15:117db924cf7c	8494	blocks = sz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8495	wc_AesEcbEncrypt(aes, out, out, AES_BLOCK_SIZE * blocks);
wolfSSL	15:117db924cf7c	8496	xorbuf(out, p, AES_BLOCK_SIZE * blocks);
wolfSSL	15:117db924cf7c	8497	p += AES_BLOCK_SIZE * blocks;
wolfSSL	15:117db924cf7c	8498	}
wolfSSL	15:117db924cf7c	8499	else
wolfSSL	15:117db924cf7c	8500	#endif /* HAVE_AES_ECB */
wolfSSL	15:117db924cf7c	8501
wolfSSL	15:117db924cf7c	8502	while (blocks--) {
wolfSSL	15:117db924cf7c	8503	IncrementGcmCounter(ctr);
wolfSSL	15:117db924cf7c	8504	#ifndef WOLFSSL_PIC32MZ_CRYPT
wolfSSL	15:117db924cf7c	8505	wc_AesEncrypt(aes, ctr, scratch);
wolfSSL	15:117db924cf7c	8506	xorbuf(scratch, p, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8507	XMEMCPY(c, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8508	#endif
wolfSSL	15:117db924cf7c	8509	p += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8510	c += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8511	}
wolfSSL	15:117db924cf7c	8512
wolfSSL	15:117db924cf7c	8513	if (partial != 0) {
wolfSSL	15:117db924cf7c	8514	IncrementGcmCounter(ctr);
wolfSSL	15:117db924cf7c	8515	wc_AesEncrypt(aes, ctr, scratch);
wolfSSL	15:117db924cf7c	8516	xorbuf(scratch, p, partial);
wolfSSL	15:117db924cf7c	8517	XMEMCPY(c, scratch, partial);
wolfSSL	15:117db924cf7c	8518	}
wolfSSL	15:117db924cf7c	8519
wolfSSL	15:117db924cf7c	8520	GHASH(aes, authIn, authInSz, out, sz, authTag, authTagSz);
wolfSSL	15:117db924cf7c	8521	wc_AesEncrypt(aes, initialCounter, scratch);
wolfSSL	15:117db924cf7c	8522	xorbuf(authTag, scratch, authTagSz);
wolfSSL	15:117db924cf7c	8523
wolfSSL	15:117db924cf7c	8524	return ret;
wolfSSL	15:117db924cf7c	8525	}
wolfSSL	15:117db924cf7c	8526
wolfSSL	15:117db924cf7c	8527	int wc_AesGcmEncrypt(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8528	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8529	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8530	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	8531	{
wolfSSL	15:117db924cf7c	8532	/* argument checks */
wolfSSL	15:117db924cf7c	8533	if (aes == NULL \|\| authTagSz > AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	8534	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8535	}
wolfSSL	15:117db924cf7c	8536
wolfSSL	15:117db924cf7c	8537	if (authTagSz < WOLFSSL_MIN_AUTH_TAG_SZ) {
wolfSSL	15:117db924cf7c	8538	WOLFSSL_MSG("GcmEncrypt authTagSz too small error");
wolfSSL	15:117db924cf7c	8539	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8540	}
wolfSSL	15:117db924cf7c	8541
wolfSSL	15:117db924cf7c	8542	#if defined(STM32_CRYPTO) && (defined(WOLFSSL_STM32F4) \|\| \
wolfSSL	15:117db924cf7c	8543	defined(WOLFSSL_STM32F7) \|\| \
wolfSSL	15:117db924cf7c	8544	defined(WOLFSSL_STM32L4))
wolfSSL	15:117db924cf7c	8545
wolfSSL	15:117db924cf7c	8546	/* additional argument checks - STM32 HW only supports 12 byte IV */
wolfSSL	15:117db924cf7c	8547	if (ivSz != GCM_NONCE_MID_SZ) {
wolfSSL	15:117db924cf7c	8548	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8549	}
wolfSSL	15:117db924cf7c	8550
wolfSSL	15:117db924cf7c	8551	/* STM32 HW AES-GCM requires / assumes inputs are a multiple of block size.
wolfSSL	15:117db924cf7c	8552	* We can avoid this by zero padding (authIn) AAD, but zero-padded plaintext
wolfSSL	15:117db924cf7c	8553	* will be encrypted and output incorrectly, causing a bad authTag.
wolfSSL	15:117db924cf7c	8554	* We will use HW accelerated AES-GCM if plain%AES_BLOCK_SZ==0.
wolfSSL	15:117db924cf7c	8555	* Otherwise, we will use accelerated AES_CTR for encrypt, and then
wolfSSL	15:117db924cf7c	8556	* perform GHASH in software.
wolfSSL	15:117db924cf7c	8557	* See NIST SP 800-38D */
wolfSSL	15:117db924cf7c	8558
wolfSSL	15:117db924cf7c	8559	/* Plain text is a multiple of block size, so use HW-Accelerated AES_GCM */
wolfSSL	15:117db924cf7c	8560	if (sz % AES_BLOCK_SIZE == 0) {
wolfSSL	15:117db924cf7c	8561	return wc_AesGcmEncrypt_STM32(aes, out, in, sz, iv, ivSz,
wolfSSL	15:117db924cf7c	8562	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	8563	}
wolfSSL	15:117db924cf7c	8564	#endif
wolfSSL	15:117db924cf7c	8565
wolfSSL	15:117db924cf7c	8566	#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
wolfSSL	15:117db924cf7c	8567	/* if async and byte count above threshold */
wolfSSL	15:117db924cf7c	8568	/* only 12-byte IV is supported in HW */
wolfSSL	15:117db924cf7c	8569	if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
wolfSSL	15:117db924cf7c	8570	sz >= WC_ASYNC_THRESH_AES_GCM && ivSz == GCM_NONCE_MID_SZ) {
wolfSSL	15:117db924cf7c	8571	#if defined(HAVE_CAVIUM)
wolfSSL	15:117db924cf7c	8572	#ifdef HAVE_CAVIUM_V
wolfSSL	15:117db924cf7c	8573	if (authInSz == 20) { /* Nitrox V GCM is only working with 20 byte AAD */
wolfSSL	15:117db924cf7c	8574	return NitroxAesGcmEncrypt(aes, out, in, sz,
wolfSSL	15:117db924cf7c	8575	(const byte*)aes->asyncKey, aes->keylen, iv, ivSz,
wolfSSL	15:117db924cf7c	8576	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	8577	}
wolfSSL	15:117db924cf7c	8578	#endif
wolfSSL	15:117db924cf7c	8579	#elif defined(HAVE_INTEL_QA)
wolfSSL	15:117db924cf7c	8580	return IntelQaSymAesGcmEncrypt(&aes->asyncDev, out, in, sz,
wolfSSL	15:117db924cf7c	8581	(const byte*)aes->asyncKey, aes->keylen, iv, ivSz,
wolfSSL	15:117db924cf7c	8582	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	8583	#else /* WOLFSSL_ASYNC_CRYPT_TEST */
wolfSSL	15:117db924cf7c	8584	if (wc_AsyncTestInit(&aes->asyncDev, ASYNC_TEST_AES_GCM_ENCRYPT)) {
wolfSSL	15:117db924cf7c	8585	WC_ASYNC_TEST* testDev = &aes->asyncDev.test;
wolfSSL	15:117db924cf7c	8586	testDev->aes.aes = aes;
wolfSSL	15:117db924cf7c	8587	testDev->aes.out = out;
wolfSSL	15:117db924cf7c	8588	testDev->aes.in = in;
wolfSSL	15:117db924cf7c	8589	testDev->aes.sz = sz;
wolfSSL	15:117db924cf7c	8590	testDev->aes.iv = iv;
wolfSSL	15:117db924cf7c	8591	testDev->aes.ivSz = ivSz;
wolfSSL	15:117db924cf7c	8592	testDev->aes.authTag = authTag;
wolfSSL	15:117db924cf7c	8593	testDev->aes.authTagSz = authTagSz;
wolfSSL	15:117db924cf7c	8594	testDev->aes.authIn = authIn;
wolfSSL	15:117db924cf7c	8595	testDev->aes.authInSz = authInSz;
wolfSSL	15:117db924cf7c	8596	return WC_PENDING_E;
wolfSSL	15:117db924cf7c	8597	}
wolfSSL	15:117db924cf7c	8598	#endif
wolfSSL	15:117db924cf7c	8599	}
wolfSSL	15:117db924cf7c	8600	#endif /* WOLFSSL_ASYNC_CRYPT */
wolfSSL	15:117db924cf7c	8601
wolfSSL	15:117db924cf7c	8602	/* Software AES-GCM */
wolfSSL	15:117db924cf7c	8603
wolfSSL	15:117db924cf7c	8604	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	8605	#ifdef HAVE_INTEL_AVX2
wolfSSL	15:117db924cf7c	8606	if (IS_INTEL_AVX2(intel_flags)) {
wolfSSL	15:117db924cf7c	8607	AES_GCM_encrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
wolfSSL	15:117db924cf7c	8608	authTagSz, (const byte*)aes->key, aes->rounds);
wolfSSL	15:117db924cf7c	8609	return 0;
wolfSSL	15:117db924cf7c	8610	}
wolfSSL	15:117db924cf7c	8611	else
wolfSSL	15:117db924cf7c	8612	#endif
wolfSSL	15:117db924cf7c	8613	#ifdef HAVE_INTEL_AVX1
wolfSSL	15:117db924cf7c	8614	if (IS_INTEL_AVX1(intel_flags)) {
wolfSSL	15:117db924cf7c	8615	AES_GCM_encrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
wolfSSL	15:117db924cf7c	8616	authTagSz, (const byte*)aes->key, aes->rounds);
wolfSSL	15:117db924cf7c	8617	return 0;
wolfSSL	15:117db924cf7c	8618	}
wolfSSL	15:117db924cf7c	8619	else
wolfSSL	15:117db924cf7c	8620	#endif
wolfSSL	15:117db924cf7c	8621	if (haveAESNI) {
wolfSSL	15:117db924cf7c	8622	AES_GCM_encrypt(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
wolfSSL	15:117db924cf7c	8623	authTagSz, (const byte*)aes->key, aes->rounds);
wolfSSL	15:117db924cf7c	8624	return 0;
wolfSSL	15:117db924cf7c	8625	}
wolfSSL	15:117db924cf7c	8626	else
wolfSSL	15:117db924cf7c	8627	#endif
wolfSSL	15:117db924cf7c	8628	{
wolfSSL	15:117db924cf7c	8629	return AES_GCM_encrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
wolfSSL	15:117db924cf7c	8630	authIn, authInSz);
wolfSSL	15:117db924cf7c	8631	}
wolfSSL	15:117db924cf7c	8632	}
wolfSSL	15:117db924cf7c	8633	#endif
wolfSSL	15:117db924cf7c	8634
wolfSSL	15:117db924cf7c	8635
wolfSSL	15:117db924cf7c	8636	#if defined(HAVE_AES_DECRYPT) \|\| defined(HAVE_AESGCM_DECRYPT)
wolfSSL	15:117db924cf7c	8637	#ifdef FREESCALE_LTC_AES_GCM
wolfSSL	15:117db924cf7c	8638	int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8639	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8640	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8641	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	8642	{
wolfSSL	15:117db924cf7c	8643	int ret;
wolfSSL	15:117db924cf7c	8644	word32 keySize;
wolfSSL	15:117db924cf7c	8645	status_t status;
wolfSSL	15:117db924cf7c	8646
wolfSSL	15:117db924cf7c	8647	/* argument checks */
wolfSSL	15:117db924cf7c	8648	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| iv == NULL \|\|
wolfSSL	15:117db924cf7c	8649	authTag == NULL \|\| authTagSz > AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	8650	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8651	}
wolfSSL	15:117db924cf7c	8652
wolfSSL	15:117db924cf7c	8653	ret = wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	8654	if (ret != 0) {
wolfSSL	15:117db924cf7c	8655	return ret;
wolfSSL	15:117db924cf7c	8656	}
wolfSSL	15:117db924cf7c	8657
wolfSSL	15:117db924cf7c	8658	status = LTC_AES_DecryptTagGcm(LTC_BASE, in, out, sz, iv, ivSz,
wolfSSL	15:117db924cf7c	8659	authIn, authInSz, (byte*)aes->key, keySize, authTag, authTagSz);
wolfSSL	15:117db924cf7c	8660
wolfSSL	15:117db924cf7c	8661	return (status == kStatus_Success) ? 0 : AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	8662	}
wolfSSL	15:117db924cf7c	8663	#elif defined(STM32_CRYPTO) && (defined(WOLFSSL_STM32F4) \|\| \
wolfSSL	15:117db924cf7c	8664	defined(WOLFSSL_STM32F7) \|\| \
wolfSSL	15:117db924cf7c	8665	defined(WOLFSSL_STM32L4))
wolfSSL	15:117db924cf7c	8666	int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8667	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8668	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8669	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	8670	{
wolfSSL	15:117db924cf7c	8671	int ret;
wolfSSL	15:117db924cf7c	8672	word32 keySize;
wolfSSL	15:117db924cf7c	8673	#ifdef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	8674	CRYP_HandleTypeDef hcryp;
wolfSSL	15:117db924cf7c	8675	#else
wolfSSL	15:117db924cf7c	8676	byte keyCopy[AES_BLOCK_SIZE * 2];
wolfSSL	15:117db924cf7c	8677	#endif /* WOLFSSL_STM32_CUBEMX */
wolfSSL	15:117db924cf7c	8678	int status;
wolfSSL	15:117db924cf7c	8679	int inPadSz, authPadSz;
wolfSSL	15:117db924cf7c	8680	byte tag[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8681	byte *inPadded = NULL;
wolfSSL	15:117db924cf7c	8682	byte *authInPadded = NULL;
wolfSSL	15:117db924cf7c	8683	byte initialCounter[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8684
wolfSSL	15:117db924cf7c	8685	/* argument checks */
wolfSSL	15:117db924cf7c	8686	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| iv == NULL \|\|
wolfSSL	15:117db924cf7c	8687	authTag == NULL \|\| authTagSz > AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	8688	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8689	}
wolfSSL	15:117db924cf7c	8690
wolfSSL	15:117db924cf7c	8691	ret = wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	8692	if (ret != 0) {
wolfSSL	15:117db924cf7c	8693	return ret;
wolfSSL	15:117db924cf7c	8694	}
wolfSSL	15:117db924cf7c	8695
wolfSSL	15:117db924cf7c	8696	/* additional argument checks - STM32 HW only supports 12 byte IV */
wolfSSL	15:117db924cf7c	8697	if (ivSz != GCM_NONCE_MID_SZ) {
wolfSSL	15:117db924cf7c	8698	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8699	}
wolfSSL	15:117db924cf7c	8700
wolfSSL	15:117db924cf7c	8701	XMEMSET(initialCounter, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8702	XMEMCPY(initialCounter, iv, ivSz);
wolfSSL	15:117db924cf7c	8703	initialCounter[AES_BLOCK_SIZE - 1] = STM32_GCM_IV_START;
wolfSSL	15:117db924cf7c	8704
wolfSSL	15:117db924cf7c	8705	/* Need to pad the AAD and input cipher text to a full block size since
wolfSSL	15:117db924cf7c	8706	* CRYP_AES_GCM will assume these are a multiple of AES_BLOCK_SIZE.
wolfSSL	15:117db924cf7c	8707	* It is okay to pad with zeros because GCM does this before GHASH already.
wolfSSL	15:117db924cf7c	8708	* See NIST SP 800-38D */
wolfSSL	15:117db924cf7c	8709
wolfSSL	15:117db924cf7c	8710	if ((sz % AES_BLOCK_SIZE) > 0) {
wolfSSL	15:117db924cf7c	8711	inPadSz = ((sz / AES_BLOCK_SIZE) + 1) * AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8712	inPadded = XMALLOC(inPadSz, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	8713	if (inPadded == NULL) {
wolfSSL	15:117db924cf7c	8714	return MEMORY_E;
wolfSSL	15:117db924cf7c	8715	}
wolfSSL	15:117db924cf7c	8716	XMEMSET(inPadded, 0, inPadSz);
wolfSSL	15:117db924cf7c	8717	XMEMCPY(inPadded, in, sz);
wolfSSL	15:117db924cf7c	8718	} else {
wolfSSL	15:117db924cf7c	8719	inPadSz = sz;
wolfSSL	15:117db924cf7c	8720	inPadded = (byte*)in;
wolfSSL	15:117db924cf7c	8721	}
wolfSSL	15:117db924cf7c	8722
wolfSSL	15:117db924cf7c	8723	if ((authInSz % AES_BLOCK_SIZE) > 0) {
wolfSSL	15:117db924cf7c	8724	authPadSz = ((authInSz / AES_BLOCK_SIZE) + 1) * AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8725	authInPadded = XMALLOC(authPadSz, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	8726	if (authInPadded == NULL) {
wolfSSL	15:117db924cf7c	8727	if (inPadded != NULL && inPadSz != sz)
wolfSSL	15:117db924cf7c	8728	XFREE(inPadded , aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	8729	return MEMORY_E;
wolfSSL	15:117db924cf7c	8730	}
wolfSSL	15:117db924cf7c	8731	XMEMSET(authInPadded, 0, authPadSz);
wolfSSL	15:117db924cf7c	8732	XMEMCPY(authInPadded, authIn, authInSz);
wolfSSL	15:117db924cf7c	8733	} else {
wolfSSL	15:117db924cf7c	8734	authPadSz = authInSz;
wolfSSL	15:117db924cf7c	8735	authInPadded = (byte*)authIn;
wolfSSL	15:117db924cf7c	8736	}
wolfSSL	15:117db924cf7c	8737
wolfSSL	15:117db924cf7c	8738	#ifdef WOLFSSL_STM32_CUBEMX
wolfSSL	15:117db924cf7c	8739	XMEMSET(&hcryp, 0, sizeof(CRYP_HandleTypeDef));
wolfSSL	15:117db924cf7c	8740	switch(keySize) {
wolfSSL	15:117db924cf7c	8741	case 16: /* 128-bit key */
wolfSSL	15:117db924cf7c	8742	hcryp.Init.KeySize = CRYP_KEYSIZE_128B;
wolfSSL	15:117db924cf7c	8743	break;
wolfSSL	15:117db924cf7c	8744	#ifdef CRYP_KEYSIZE_192B
wolfSSL	15:117db924cf7c	8745	case 24: /* 192-bit key */
wolfSSL	15:117db924cf7c	8746	hcryp.Init.KeySize = CRYP_KEYSIZE_192B;
wolfSSL	15:117db924cf7c	8747	break;
wolfSSL	15:117db924cf7c	8748	#endif
wolfSSL	15:117db924cf7c	8749	case 32: /* 256-bit key */
wolfSSL	15:117db924cf7c	8750	hcryp.Init.KeySize = CRYP_KEYSIZE_256B;
wolfSSL	15:117db924cf7c	8751	break;
wolfSSL	15:117db924cf7c	8752	default:
wolfSSL	15:117db924cf7c	8753	break;
wolfSSL	15:117db924cf7c	8754	}
wolfSSL	15:117db924cf7c	8755	hcryp.Instance = CRYP;
wolfSSL	15:117db924cf7c	8756	hcryp.Init.DataType = CRYP_DATATYPE_8B;
wolfSSL	15:117db924cf7c	8757	hcryp.Init.pKey = (byte*)aes->key;
wolfSSL	15:117db924cf7c	8758	hcryp.Init.pInitVect = initialCounter;
wolfSSL	15:117db924cf7c	8759	hcryp.Init.Header = authInPadded;
wolfSSL	15:117db924cf7c	8760	hcryp.Init.HeaderSize = authInSz;
wolfSSL	15:117db924cf7c	8761
wolfSSL	15:117db924cf7c	8762	#ifdef WOLFSSL_STM32L4
wolfSSL	15:117db924cf7c	8763	/* Set the CRYP parameters */
wolfSSL	15:117db924cf7c	8764	hcryp.Init.ChainingMode = CRYP_CHAINMODE_AES_GCM_GMAC;
wolfSSL	15:117db924cf7c	8765	hcryp.Init.OperatingMode = CRYP_ALGOMODE_DECRYPT;
wolfSSL	15:117db924cf7c	8766	hcryp.Init.GCMCMACPhase = CRYP_INIT_PHASE;
wolfSSL	15:117db924cf7c	8767	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	8768
wolfSSL	15:117db924cf7c	8769	/* GCM init phase */
wolfSSL	15:117db924cf7c	8770	status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8771	if (status == HAL_OK) {
wolfSSL	15:117db924cf7c	8772	/* GCM header phase */
wolfSSL	15:117db924cf7c	8773	hcryp.Init.GCMCMACPhase = CRYP_HEADER_PHASE;
wolfSSL	15:117db924cf7c	8774	status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, 0, NULL, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8775	if (status == HAL_OK) {
wolfSSL	15:117db924cf7c	8776	/* GCM payload phase */
wolfSSL	15:117db924cf7c	8777	hcryp.Init.GCMCMACPhase = CRYP_PAYLOAD_PHASE;
wolfSSL	15:117db924cf7c	8778	status = HAL_CRYPEx_AES_Auth(&hcryp, (byte*)inPadded, sz, inPadded,
wolfSSL	15:117db924cf7c	8779	STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8780	if (status == HAL_OK) {
wolfSSL	15:117db924cf7c	8781	/* GCM final phase */
wolfSSL	15:117db924cf7c	8782	hcryp.Init.GCMCMACPhase = CRYP_FINAL_PHASE;
wolfSSL	15:117db924cf7c	8783	status = HAL_CRYPEx_AES_Auth(&hcryp, NULL, sz, tag,
wolfSSL	15:117db924cf7c	8784	STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8785	}
wolfSSL	15:117db924cf7c	8786	}
wolfSSL	15:117db924cf7c	8787	}
wolfSSL	15:117db924cf7c	8788	#else
wolfSSL	15:117db924cf7c	8789	HAL_CRYP_Init(&hcryp);
wolfSSL	15:117db924cf7c	8790	/* Use inPadded for output buffer instead of
wolfSSL	15:117db924cf7c	8791	* out so that we don't overflow our size. */
wolfSSL	15:117db924cf7c	8792	status = HAL_CRYPEx_AESGCM_Decrypt(&hcryp, (byte*)inPadded,
wolfSSL	15:117db924cf7c	8793	sz, inPadded, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8794	/* Compute the authTag */
wolfSSL	15:117db924cf7c	8795	if (status == HAL_OK) {
wolfSSL	15:117db924cf7c	8796	status = HAL_CRYPEx_AESGCM_Finish(&hcryp, sz, tag, STM32_HAL_TIMEOUT);
wolfSSL	15:117db924cf7c	8797	}
wolfSSL	15:117db924cf7c	8798	#endif
wolfSSL	15:117db924cf7c	8799
wolfSSL	15:117db924cf7c	8800	if (status != HAL_OK)
wolfSSL	15:117db924cf7c	8801	ret = AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	8802
wolfSSL	15:117db924cf7c	8803	HAL_CRYP_DeInit(&hcryp);
wolfSSL	15:117db924cf7c	8804	#else
wolfSSL	15:117db924cf7c	8805	ByteReverseWords((word32)keyCopy, (word32)aes->key, keySize);
wolfSSL	15:117db924cf7c	8806
wolfSSL	15:117db924cf7c	8807	/* Input size and auth size need to be the actual sizes, even though
wolfSSL	15:117db924cf7c	8808	* they are not block aligned, because this length (in bits) is used
wolfSSL	15:117db924cf7c	8809	* in the final GHASH. Use inPadded for output buffer instead of
wolfSSL	15:117db924cf7c	8810	* out so that we don't overflow our size. */
wolfSSL	15:117db924cf7c	8811	status = CRYP_AES_GCM(MODE_DECRYPT, (uint8_t*)initialCounter,
wolfSSL	15:117db924cf7c	8812	(uint8_t)keyCopy, keySize 8,
wolfSSL	15:117db924cf7c	8813	(uint8_t*)inPadded, sz,
wolfSSL	15:117db924cf7c	8814	(uint8_t*)authInPadded,authInSz,
wolfSSL	15:117db924cf7c	8815	(uint8_t*)inPadded, tag);
wolfSSL	15:117db924cf7c	8816	if (status != SUCCESS)
wolfSSL	15:117db924cf7c	8817	ret = AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	8818	#endif /* WOLFSSL_STM32_CUBEMX */
wolfSSL	15:117db924cf7c	8819
wolfSSL	15:117db924cf7c	8820	if (ret == 0 && ConstantCompare(authTag, tag, authTagSz) == 0) {
wolfSSL	15:117db924cf7c	8821	/* Only keep the decrypted data if authTag success. */
wolfSSL	15:117db924cf7c	8822	XMEMCPY(out, inPadded, sz);
wolfSSL	15:117db924cf7c	8823	ret = 0; /* success */
wolfSSL	15:117db924cf7c	8824	}
wolfSSL	15:117db924cf7c	8825
wolfSSL	15:117db924cf7c	8826	/* only allocate padding buffers if the inputs are not a multiple of block sz */
wolfSSL	15:117db924cf7c	8827	if (inPadded != NULL && inPadSz != sz)
wolfSSL	15:117db924cf7c	8828	XFREE(inPadded , aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	8829	if (authInPadded != NULL && authPadSz != authInSz)
wolfSSL	15:117db924cf7c	8830	XFREE(authInPadded, aes->heap, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	15:117db924cf7c	8831
wolfSSL	15:117db924cf7c	8832	return ret;
wolfSSL	15:117db924cf7c	8833	}
wolfSSL	15:117db924cf7c	8834	#else
wolfSSL	15:117db924cf7c	8835	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	8836	int AES_GCM_decrypt_C(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8837	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8838	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8839	const byte* authIn, word32 authInSz);
wolfSSL	15:117db924cf7c	8840	#else
wolfSSL	15:117db924cf7c	8841	static
wolfSSL	15:117db924cf7c	8842	#endif
wolfSSL	15:117db924cf7c	8843	int AES_GCM_decrypt_C(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8844	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8845	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8846	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	8847	{
wolfSSL	15:117db924cf7c	8848	int ret = 0;
wolfSSL	15:117db924cf7c	8849	word32 blocks = sz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8850	word32 partial = sz % AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8851	const byte* c = in;
wolfSSL	15:117db924cf7c	8852	byte* p = out;
wolfSSL	15:117db924cf7c	8853	byte counter[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8854	byte initialCounter[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8855	byte *ctr;
wolfSSL	15:117db924cf7c	8856	byte scratch[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8857	byte Tprime[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8858	byte EKY0[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	8859	ctr = counter;
wolfSSL	15:117db924cf7c	8860
wolfSSL	15:117db924cf7c	8861	XMEMSET(initialCounter, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8862	if (ivSz == GCM_NONCE_MID_SZ) {
wolfSSL	15:117db924cf7c	8863	XMEMCPY(initialCounter, iv, ivSz);
wolfSSL	15:117db924cf7c	8864	initialCounter[AES_BLOCK_SIZE - 1] = 1;
wolfSSL	15:117db924cf7c	8865	}
wolfSSL	15:117db924cf7c	8866	else {
wolfSSL	15:117db924cf7c	8867	GHASH(aes, NULL, 0, iv, ivSz, initialCounter, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8868	}
wolfSSL	15:117db924cf7c	8869	XMEMCPY(ctr, initialCounter, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8870
wolfSSL	15:117db924cf7c	8871	/* Calc the authTag again using the received auth data and the cipher text */
wolfSSL	15:117db924cf7c	8872	GHASH(aes, authIn, authInSz, in, sz, Tprime, sizeof(Tprime));
wolfSSL	15:117db924cf7c	8873	wc_AesEncrypt(aes, ctr, EKY0);
wolfSSL	15:117db924cf7c	8874	xorbuf(Tprime, EKY0, sizeof(Tprime));
wolfSSL	15:117db924cf7c	8875
wolfSSL	15:117db924cf7c	8876	if (ConstantCompare(authTag, Tprime, authTagSz) != 0) {
wolfSSL	15:117db924cf7c	8877	return AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	8878	}
wolfSSL	15:117db924cf7c	8879
wolfSSL	15:117db924cf7c	8880	#ifdef WOLFSSL_PIC32MZ_CRYPT
wolfSSL	15:117db924cf7c	8881	if (blocks) {
wolfSSL	15:117db924cf7c	8882	/* use intitial IV for PIC32 HW, but don't use it below */
wolfSSL	15:117db924cf7c	8883	XMEMCPY(aes->reg, ctr, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8884
wolfSSL	15:117db924cf7c	8885	ret = wc_Pic32AesCrypt(
wolfSSL	15:117db924cf7c	8886	aes->key, aes->keylen, aes->reg, AES_BLOCK_SIZE,
wolfSSL	15:117db924cf7c	8887	out, in, (blocks * AES_BLOCK_SIZE),
wolfSSL	15:117db924cf7c	8888	PIC32_DECRYPTION, PIC32_ALGO_AES, PIC32_CRYPTOALGO_AES_GCM);
wolfSSL	15:117db924cf7c	8889	if (ret != 0)
wolfSSL	15:117db924cf7c	8890	return ret;
wolfSSL	15:117db924cf7c	8891	}
wolfSSL	15:117db924cf7c	8892	/* process remainder using partial handling */
wolfSSL	15:117db924cf7c	8893	#endif
wolfSSL	15:117db924cf7c	8894
wolfSSL	15:117db924cf7c	8895	#if defined(HAVE_AES_ECB) && !defined(WOLFSSL_PIC32MZ_CRYPT)
wolfSSL	15:117db924cf7c	8896	/* some hardware acceleration can gain performance from doing AES encryption
wolfSSL	15:117db924cf7c	8897	* of the whole buffer at once */
wolfSSL	15:117db924cf7c	8898	if (c != p) { /* can not handle inline decryption */
wolfSSL	15:117db924cf7c	8899	while (blocks--) {
wolfSSL	15:117db924cf7c	8900	IncrementGcmCounter(ctr);
wolfSSL	15:117db924cf7c	8901	XMEMCPY(p, ctr, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8902	p += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8903	}
wolfSSL	15:117db924cf7c	8904
wolfSSL	15:117db924cf7c	8905	/* reset number of blocks and then do encryption */
wolfSSL	15:117db924cf7c	8906	blocks = sz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8907	wc_AesEcbEncrypt(aes, out, out, AES_BLOCK_SIZE * blocks);
wolfSSL	15:117db924cf7c	8908	xorbuf(out, c, AES_BLOCK_SIZE * blocks);
wolfSSL	15:117db924cf7c	8909	c += AES_BLOCK_SIZE * blocks;
wolfSSL	15:117db924cf7c	8910	}
wolfSSL	15:117db924cf7c	8911	else
wolfSSL	15:117db924cf7c	8912	#endif /* HAVE_AES_ECB */
wolfSSL	15:117db924cf7c	8913	while (blocks--) {
wolfSSL	15:117db924cf7c	8914	IncrementGcmCounter(ctr);
wolfSSL	15:117db924cf7c	8915	#ifndef WOLFSSL_PIC32MZ_CRYPT
wolfSSL	15:117db924cf7c	8916	wc_AesEncrypt(aes, ctr, scratch);
wolfSSL	15:117db924cf7c	8917	xorbuf(scratch, c, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8918	XMEMCPY(p, scratch, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	8919	#endif
wolfSSL	15:117db924cf7c	8920	p += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8921	c += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	8922	}
wolfSSL	15:117db924cf7c	8923
wolfSSL	15:117db924cf7c	8924	if (partial != 0) {
wolfSSL	15:117db924cf7c	8925	IncrementGcmCounter(ctr);
wolfSSL	15:117db924cf7c	8926	wc_AesEncrypt(aes, ctr, scratch);
wolfSSL	15:117db924cf7c	8927	xorbuf(scratch, c, partial);
wolfSSL	15:117db924cf7c	8928	XMEMCPY(p, scratch, partial);
wolfSSL	15:117db924cf7c	8929	}
wolfSSL	15:117db924cf7c	8930
wolfSSL	15:117db924cf7c	8931	return ret;
wolfSSL	15:117db924cf7c	8932	}
wolfSSL	15:117db924cf7c	8933
wolfSSL	15:117db924cf7c	8934	int wc_AesGcmDecrypt(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	8935	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	8936	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	8937	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	8938	{
wolfSSL	15:117db924cf7c	8939	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	8940	int res;
wolfSSL	15:117db924cf7c	8941	#endif
wolfSSL	15:117db924cf7c	8942
wolfSSL	15:117db924cf7c	8943	/* argument checks */
wolfSSL	15:117db924cf7c	8944	/* If the sz is non-zero, both in and out must be set. If sz is 0,
wolfSSL	15:117db924cf7c	8945	* in and out are don't cares, as this is is the GMAC case. */
wolfSSL	15:117db924cf7c	8946	if (aes == NULL \|\| iv == NULL \|\| (sz != 0 && (in == NULL \|\| out == NULL)) \|\|
wolfSSL	15:117db924cf7c	8947	authTag == NULL \|\| authTagSz > AES_BLOCK_SIZE \|\| authTagSz == 0) {
wolfSSL	15:117db924cf7c	8948
wolfSSL	15:117db924cf7c	8949	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	8950	}
wolfSSL	15:117db924cf7c	8951
wolfSSL	15:117db924cf7c	8952	#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
wolfSSL	15:117db924cf7c	8953	/* if async and byte count above threshold */
wolfSSL	15:117db924cf7c	8954	/* only 12-byte IV is supported in HW */
wolfSSL	15:117db924cf7c	8955	if (aes->asyncDev.marker == WOLFSSL_ASYNC_MARKER_AES &&
wolfSSL	15:117db924cf7c	8956	sz >= WC_ASYNC_THRESH_AES_GCM && ivSz == GCM_NONCE_MID_SZ) {
wolfSSL	15:117db924cf7c	8957	#if defined(HAVE_CAVIUM)
wolfSSL	15:117db924cf7c	8958	#ifdef HAVE_CAVIUM_V
wolfSSL	15:117db924cf7c	8959	if (authInSz == 20) { /* Nitrox V GCM is only working with 20 byte AAD */
wolfSSL	15:117db924cf7c	8960	return NitroxAesGcmDecrypt(aes, out, in, sz,
wolfSSL	15:117db924cf7c	8961	(const byte*)aes->asyncKey, aes->keylen, iv, ivSz,
wolfSSL	15:117db924cf7c	8962	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	8963	}
wolfSSL	15:117db924cf7c	8964	#endif
wolfSSL	15:117db924cf7c	8965	#elif defined(HAVE_INTEL_QA)
wolfSSL	15:117db924cf7c	8966	return IntelQaSymAesGcmDecrypt(&aes->asyncDev, out, in, sz,
wolfSSL	15:117db924cf7c	8967	(const byte*)aes->asyncKey, aes->keylen, iv, ivSz,
wolfSSL	15:117db924cf7c	8968	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	8969	#else /* WOLFSSL_ASYNC_CRYPT_TEST */
wolfSSL	15:117db924cf7c	8970	if (wc_AsyncTestInit(&aes->asyncDev, ASYNC_TEST_AES_GCM_DECRYPT)) {
wolfSSL	15:117db924cf7c	8971	WC_ASYNC_TEST* testDev = &aes->asyncDev.test;
wolfSSL	15:117db924cf7c	8972	testDev->aes.aes = aes;
wolfSSL	15:117db924cf7c	8973	testDev->aes.out = out;
wolfSSL	15:117db924cf7c	8974	testDev->aes.in = in;
wolfSSL	15:117db924cf7c	8975	testDev->aes.sz = sz;
wolfSSL	15:117db924cf7c	8976	testDev->aes.iv = iv;
wolfSSL	15:117db924cf7c	8977	testDev->aes.ivSz = ivSz;
wolfSSL	15:117db924cf7c	8978	testDev->aes.authTag = (byte*)authTag;
wolfSSL	15:117db924cf7c	8979	testDev->aes.authTagSz = authTagSz;
wolfSSL	15:117db924cf7c	8980	testDev->aes.authIn = authIn;
wolfSSL	15:117db924cf7c	8981	testDev->aes.authInSz = authInSz;
wolfSSL	15:117db924cf7c	8982	return WC_PENDING_E;
wolfSSL	15:117db924cf7c	8983	}
wolfSSL	15:117db924cf7c	8984	#endif
wolfSSL	15:117db924cf7c	8985	}
wolfSSL	15:117db924cf7c	8986	#endif /* WOLFSSL_ASYNC_CRYPT */
wolfSSL	15:117db924cf7c	8987
wolfSSL	15:117db924cf7c	8988	/* software AES GCM */
wolfSSL	15:117db924cf7c	8989
wolfSSL	15:117db924cf7c	8990	#ifdef WOLFSSL_AESNI
wolfSSL	15:117db924cf7c	8991	#ifdef HAVE_INTEL_AVX2
wolfSSL	15:117db924cf7c	8992	if (IS_INTEL_AVX2(intel_flags)) {
wolfSSL	15:117db924cf7c	8993	AES_GCM_decrypt_avx2(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
wolfSSL	15:117db924cf7c	8994	authTagSz, (byte*)aes->key, aes->rounds, &res);
wolfSSL	15:117db924cf7c	8995	if (res == 0)
wolfSSL	15:117db924cf7c	8996	return AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	8997	return 0;
wolfSSL	15:117db924cf7c	8998	}
wolfSSL	15:117db924cf7c	8999	else
wolfSSL	15:117db924cf7c	9000	#endif
wolfSSL	15:117db924cf7c	9001	#ifdef HAVE_INTEL_AVX1
wolfSSL	15:117db924cf7c	9002	if (IS_INTEL_AVX1(intel_flags)) {
wolfSSL	15:117db924cf7c	9003	AES_GCM_decrypt_avx1(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
wolfSSL	15:117db924cf7c	9004	authTagSz, (byte*)aes->key, aes->rounds, &res);
wolfSSL	15:117db924cf7c	9005	if (res == 0)
wolfSSL	15:117db924cf7c	9006	return AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	9007	return 0;
wolfSSL	15:117db924cf7c	9008	}
wolfSSL	15:117db924cf7c	9009	else
wolfSSL	15:117db924cf7c	9010	#endif
wolfSSL	15:117db924cf7c	9011	if (haveAESNI) {
wolfSSL	15:117db924cf7c	9012	AES_GCM_decrypt(in, out, authIn, iv, authTag, sz, authInSz, ivSz,
wolfSSL	15:117db924cf7c	9013	authTagSz, (byte*)aes->key, aes->rounds, &res);
wolfSSL	15:117db924cf7c	9014	if (res == 0)
wolfSSL	15:117db924cf7c	9015	return AES_GCM_AUTH_E;
wolfSSL	15:117db924cf7c	9016	return 0;
wolfSSL	15:117db924cf7c	9017	}
wolfSSL	15:117db924cf7c	9018	else
wolfSSL	15:117db924cf7c	9019	#endif
wolfSSL	15:117db924cf7c	9020	{
wolfSSL	15:117db924cf7c	9021	return AES_GCM_decrypt_C(aes, out, in, sz, iv, ivSz, authTag, authTagSz,
wolfSSL	15:117db924cf7c	9022	authIn, authInSz);
wolfSSL	15:117db924cf7c	9023	}
wolfSSL	15:117db924cf7c	9024	}
wolfSSL	15:117db924cf7c	9025	#endif
wolfSSL	15:117db924cf7c	9026	#endif /* HAVE_AES_DECRYPT \|\| HAVE_AESGCM_DECRYPT */
wolfSSL	15:117db924cf7c	9027	#endif /* (WOLFSSL_XILINX_CRYPT) */
wolfSSL	15:117db924cf7c	9028	#endif /* end of block for AESGCM implementation selection */
wolfSSL	15:117db924cf7c	9029
wolfSSL	15:117db924cf7c	9030
wolfSSL	15:117db924cf7c	9031	/* Common to all, abstract functions that build off of lower level AESGCM
wolfSSL	15:117db924cf7c	9032	* functions */
wolfSSL	15:117db924cf7c	9033	#ifndef WC_NO_RNG
wolfSSL	15:117db924cf7c	9034
wolfSSL	15:117db924cf7c	9035	int wc_AesGcmSetExtIV(Aes* aes, const byte* iv, word32 ivSz)
wolfSSL	15:117db924cf7c	9036	{
wolfSSL	15:117db924cf7c	9037	int ret = 0;
wolfSSL	15:117db924cf7c	9038
wolfSSL	15:117db924cf7c	9039	if (aes == NULL \|\| iv == NULL \|\|
wolfSSL	15:117db924cf7c	9040	(ivSz != GCM_NONCE_MIN_SZ && ivSz != GCM_NONCE_MID_SZ &&
wolfSSL	15:117db924cf7c	9041	ivSz != GCM_NONCE_MAX_SZ)) {
wolfSSL	15:117db924cf7c	9042
wolfSSL	15:117db924cf7c	9043	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9044	}
wolfSSL	15:117db924cf7c	9045
wolfSSL	15:117db924cf7c	9046	if (ret == 0) {
wolfSSL	15:117db924cf7c	9047	XMEMCPY((byte*)aes->reg, iv, ivSz);
wolfSSL	15:117db924cf7c	9048
wolfSSL	15:117db924cf7c	9049	/* If the IV is 96, allow for a 2^64 invocation counter.
wolfSSL	15:117db924cf7c	9050	* For any other size for the nonce, limit the invocation
wolfSSL	15:117db924cf7c	9051	* counter to 32-bits. (SP 800-38D 8.3) */
wolfSSL	15:117db924cf7c	9052	aes->invokeCtr[0] = 0;
wolfSSL	15:117db924cf7c	9053	aes->invokeCtr[1] = (ivSz == GCM_NONCE_MID_SZ) ? 0 : 0xFFFFFFFF;
wolfSSL	15:117db924cf7c	9054	aes->nonceSz = ivSz;
wolfSSL	15:117db924cf7c	9055	}
wolfSSL	15:117db924cf7c	9056
wolfSSL	15:117db924cf7c	9057	return ret;
wolfSSL	15:117db924cf7c	9058	}
wolfSSL	15:117db924cf7c	9059
wolfSSL	15:117db924cf7c	9060
wolfSSL	15:117db924cf7c	9061	int wc_AesGcmSetIV(Aes* aes, word32 ivSz,
wolfSSL	15:117db924cf7c	9062	const byte* ivFixed, word32 ivFixedSz,
wolfSSL	15:117db924cf7c	9063	WC_RNG* rng)
wolfSSL	15:117db924cf7c	9064	{
wolfSSL	15:117db924cf7c	9065	int ret = 0;
wolfSSL	15:117db924cf7c	9066
wolfSSL	15:117db924cf7c	9067	if (aes == NULL \|\| rng == NULL \|\|
wolfSSL	15:117db924cf7c	9068	(ivSz != GCM_NONCE_MIN_SZ && ivSz != GCM_NONCE_MID_SZ &&
wolfSSL	15:117db924cf7c	9069	ivSz != GCM_NONCE_MAX_SZ) \|\|
wolfSSL	15:117db924cf7c	9070	(ivFixed == NULL && ivFixedSz != 0) \|\|
wolfSSL	15:117db924cf7c	9071	(ivFixed != NULL && ivFixedSz != AES_IV_FIXED_SZ)) {
wolfSSL	15:117db924cf7c	9072
wolfSSL	15:117db924cf7c	9073	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9074	}
wolfSSL	15:117db924cf7c	9075
wolfSSL	15:117db924cf7c	9076	if (ret == 0) {
wolfSSL	15:117db924cf7c	9077	byte* iv = (byte*)aes->reg;
wolfSSL	15:117db924cf7c	9078
wolfSSL	15:117db924cf7c	9079	if (ivFixedSz)
wolfSSL	15:117db924cf7c	9080	XMEMCPY(iv, ivFixed, ivFixedSz);
wolfSSL	15:117db924cf7c	9081
wolfSSL	15:117db924cf7c	9082	ret = wc_RNG_GenerateBlock(rng, iv + ivFixedSz, ivSz - ivFixedSz);
wolfSSL	15:117db924cf7c	9083	}
wolfSSL	15:117db924cf7c	9084
wolfSSL	15:117db924cf7c	9085	if (ret == 0) {
wolfSSL	15:117db924cf7c	9086	/* If the IV is 96, allow for a 2^64 invocation counter.
wolfSSL	15:117db924cf7c	9087	* For any other size for the nonce, limit the invocation
wolfSSL	15:117db924cf7c	9088	* counter to 32-bits. (SP 800-38D 8.3) */
wolfSSL	15:117db924cf7c	9089	aes->invokeCtr[0] = 0;
wolfSSL	15:117db924cf7c	9090	aes->invokeCtr[1] = (ivSz == GCM_NONCE_MID_SZ) ? 0 : 0xFFFFFFFF;
wolfSSL	15:117db924cf7c	9091	aes->nonceSz = ivSz;
wolfSSL	15:117db924cf7c	9092	}
wolfSSL	15:117db924cf7c	9093
wolfSSL	15:117db924cf7c	9094	return ret;
wolfSSL	15:117db924cf7c	9095	}
wolfSSL	15:117db924cf7c	9096
wolfSSL	15:117db924cf7c	9097
wolfSSL	15:117db924cf7c	9098	int wc_AesGcmEncrypt_ex(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	9099	byte* ivOut, word32 ivOutSz,
wolfSSL	15:117db924cf7c	9100	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	9101	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	9102	{
wolfSSL	15:117db924cf7c	9103	int ret = 0;
wolfSSL	15:117db924cf7c	9104
wolfSSL	15:117db924cf7c	9105	if (aes == NULL \|\| (sz != 0 && (in == NULL \|\| out == NULL)) \|\|
wolfSSL	15:117db924cf7c	9106	ivOut == NULL \|\| ivOutSz != aes->nonceSz \|\|
wolfSSL	15:117db924cf7c	9107	(authIn == NULL && authInSz != 0)) {
wolfSSL	15:117db924cf7c	9108
wolfSSL	15:117db924cf7c	9109	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9110	}
wolfSSL	15:117db924cf7c	9111
wolfSSL	15:117db924cf7c	9112	if (ret == 0) {
wolfSSL	15:117db924cf7c	9113	aes->invokeCtr[0]++;
wolfSSL	15:117db924cf7c	9114	if (aes->invokeCtr[0] == 0) {
wolfSSL	15:117db924cf7c	9115	aes->invokeCtr[1]++;
wolfSSL	15:117db924cf7c	9116	if (aes->invokeCtr[1] == 0)
wolfSSL	15:117db924cf7c	9117	ret = AES_GCM_OVERFLOW_E;
wolfSSL	15:117db924cf7c	9118	}
wolfSSL	15:117db924cf7c	9119	}
wolfSSL	15:117db924cf7c	9120
wolfSSL	15:117db924cf7c	9121	if (ret == 0) {
wolfSSL	15:117db924cf7c	9122	XMEMCPY(ivOut, aes->reg, ivOutSz);
wolfSSL	15:117db924cf7c	9123	ret = wc_AesGcmEncrypt(aes, out, in, sz,
wolfSSL	15:117db924cf7c	9124	(byte*)aes->reg, ivOutSz,
wolfSSL	15:117db924cf7c	9125	authTag, authTagSz,
wolfSSL	15:117db924cf7c	9126	authIn, authInSz);
wolfSSL	15:117db924cf7c	9127	IncCtr((byte*)aes->reg, ivOutSz);
wolfSSL	15:117db924cf7c	9128	}
wolfSSL	15:117db924cf7c	9129
wolfSSL	15:117db924cf7c	9130	return ret;
wolfSSL	15:117db924cf7c	9131	}
wolfSSL	15:117db924cf7c	9132
wolfSSL	15:117db924cf7c	9133	int wc_Gmac(const byte* key, word32 keySz, byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	9134	const byte* authIn, word32 authInSz,
wolfSSL	15:117db924cf7c	9135	byte* authTag, word32 authTagSz, WC_RNG* rng)
wolfSSL	15:117db924cf7c	9136	{
wolfSSL	15:117db924cf7c	9137	Aes aes;
wolfSSL	15:117db924cf7c	9138	int ret = 0;
wolfSSL	15:117db924cf7c	9139
wolfSSL	15:117db924cf7c	9140	if (key == NULL \|\| iv == NULL \|\| (authIn == NULL && authInSz != 0) \|\|
wolfSSL	15:117db924cf7c	9141	authTag == NULL \|\| authTagSz == 0 \|\| rng == NULL) {
wolfSSL	15:117db924cf7c	9142
wolfSSL	15:117db924cf7c	9143	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9144	}
wolfSSL	15:117db924cf7c	9145
wolfSSL	15:117db924cf7c	9146	if (ret == 0)
wolfSSL	15:117db924cf7c	9147	ret = wc_AesGcmSetKey(&aes, key, keySz);
wolfSSL	15:117db924cf7c	9148	if (ret == 0)
wolfSSL	15:117db924cf7c	9149	ret = wc_AesGcmSetIV(&aes, ivSz, NULL, 0, rng);
wolfSSL	15:117db924cf7c	9150	if (ret == 0)
wolfSSL	15:117db924cf7c	9151	ret = wc_AesGcmEncrypt_ex(&aes, NULL, NULL, 0, iv, ivSz,
wolfSSL	15:117db924cf7c	9152	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	9153	ForceZero(&aes, sizeof(aes));
wolfSSL	15:117db924cf7c	9154
wolfSSL	15:117db924cf7c	9155	return ret;
wolfSSL	15:117db924cf7c	9156	}
wolfSSL	15:117db924cf7c	9157
wolfSSL	15:117db924cf7c	9158	int wc_GmacVerify(const byte* key, word32 keySz,
wolfSSL	15:117db924cf7c	9159	const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	9160	const byte* authIn, word32 authInSz,
wolfSSL	15:117db924cf7c	9161	const byte* authTag, word32 authTagSz)
wolfSSL	15:117db924cf7c	9162	{
wolfSSL	15:117db924cf7c	9163	Aes aes;
wolfSSL	15:117db924cf7c	9164	int ret = 0;
wolfSSL	15:117db924cf7c	9165
wolfSSL	15:117db924cf7c	9166	if (key == NULL \|\| iv == NULL \|\| (authIn == NULL && authInSz != 0) \|\|
wolfSSL	15:117db924cf7c	9167	authTag == NULL \|\| authTagSz == 0 \|\| authTagSz > AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	9168
wolfSSL	15:117db924cf7c	9169	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9170	}
wolfSSL	15:117db924cf7c	9171
wolfSSL	15:117db924cf7c	9172	if (ret == 0)
wolfSSL	15:117db924cf7c	9173	ret = wc_AesGcmSetKey(&aes, key, keySz);
wolfSSL	15:117db924cf7c	9174	if (ret == 0)
wolfSSL	15:117db924cf7c	9175	ret = wc_AesGcmDecrypt(&aes, NULL, NULL, 0, iv, ivSz,
wolfSSL	15:117db924cf7c	9176	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	9177	ForceZero(&aes, sizeof(aes));
wolfSSL	15:117db924cf7c	9178
wolfSSL	15:117db924cf7c	9179	return ret;
wolfSSL	15:117db924cf7c	9180	}
wolfSSL	15:117db924cf7c	9181
wolfSSL	15:117db924cf7c	9182	#endif /* WC_NO_RNG */
wolfSSL	15:117db924cf7c	9183
wolfSSL	15:117db924cf7c	9184
wolfSSL	15:117db924cf7c	9185	WOLFSSL_API int wc_GmacSetKey(Gmac* gmac, const byte* key, word32 len)
wolfSSL	15:117db924cf7c	9186	{
wolfSSL	15:117db924cf7c	9187	if (gmac == NULL \|\| key == NULL) {
wolfSSL	15:117db924cf7c	9188	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9189	}
wolfSSL	15:117db924cf7c	9190	return wc_AesGcmSetKey(&gmac->aes, key, len);
wolfSSL	15:117db924cf7c	9191	}
wolfSSL	15:117db924cf7c	9192
wolfSSL	15:117db924cf7c	9193
wolfSSL	15:117db924cf7c	9194	WOLFSSL_API int wc_GmacUpdate(Gmac* gmac, const byte* iv, word32 ivSz,
wolfSSL	15:117db924cf7c	9195	const byte* authIn, word32 authInSz,
wolfSSL	15:117db924cf7c	9196	byte* authTag, word32 authTagSz)
wolfSSL	15:117db924cf7c	9197	{
wolfSSL	15:117db924cf7c	9198	return wc_AesGcmEncrypt(&gmac->aes, NULL, NULL, 0, iv, ivSz,
wolfSSL	15:117db924cf7c	9199	authTag, authTagSz, authIn, authInSz);
wolfSSL	15:117db924cf7c	9200	}
wolfSSL	15:117db924cf7c	9201
wolfSSL	15:117db924cf7c	9202	#endif /* HAVE_AESGCM */
wolfSSL	15:117db924cf7c	9203
wolfSSL	15:117db924cf7c	9204
wolfSSL	15:117db924cf7c	9205	#ifdef HAVE_AESCCM
wolfSSL	15:117db924cf7c	9206
wolfSSL	15:117db924cf7c	9207	int wc_AesCcmSetKey(Aes* aes, const byte* key, word32 keySz)
wolfSSL	15:117db924cf7c	9208	{
wolfSSL	15:117db924cf7c	9209	if (!((keySz == 16) \|\| (keySz == 24) \|\| (keySz == 32)))
wolfSSL	15:117db924cf7c	9210	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9211
wolfSSL	15:117db924cf7c	9212	return wc_AesSetKey(aes, key, keySz, NULL, AES_ENCRYPTION);
wolfSSL	15:117db924cf7c	9213	}
wolfSSL	15:117db924cf7c	9214
wolfSSL	15:117db924cf7c	9215	#ifdef WOLFSSL_ARMASM
wolfSSL	15:117db924cf7c	9216	/* implementation located in wolfcrypt/src/port/arm/armv8-aes.c */
wolfSSL	15:117db924cf7c	9217
wolfSSL	15:117db924cf7c	9218	#elif defined(HAVE_COLDFIRE_SEC)
wolfSSL	15:117db924cf7c	9219	#error "Coldfire SEC doesn't currently support AES-CCM mode"
wolfSSL	15:117db924cf7c	9220
wolfSSL	15:117db924cf7c	9221	#elif defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES)
wolfSSL	15:117db924cf7c	9222	/* implemented in wolfcrypt/src/port/caam_aes.c */
wolfSSL	15:117db924cf7c	9223
wolfSSL	15:117db924cf7c	9224	#elif defined(FREESCALE_LTC)
wolfSSL	15:117db924cf7c	9225
wolfSSL	15:117db924cf7c	9226	/* return 0 on success */
wolfSSL	15:117db924cf7c	9227	int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
wolfSSL	15:117db924cf7c	9228	const byte* nonce, word32 nonceSz,
wolfSSL	15:117db924cf7c	9229	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	9230	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	9231	{
wolfSSL	15:117db924cf7c	9232	byte *key;
wolfSSL	15:117db924cf7c	9233	uint32_t keySize;
wolfSSL	15:117db924cf7c	9234	status_t status;
wolfSSL	15:117db924cf7c	9235
wolfSSL	15:117db924cf7c	9236	/* sanity check on arguments */
wolfSSL	15:117db924cf7c	9237	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| nonce == NULL
wolfSSL	15:117db924cf7c	9238	\|\| authTag == NULL \|\| nonceSz < 7 \|\| nonceSz > 13)
wolfSSL	15:117db924cf7c	9239	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9240
wolfSSL	15:117db924cf7c	9241	key = (byte*)aes->key;
wolfSSL	15:117db924cf7c	9242
wolfSSL	15:117db924cf7c	9243	status = wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	9244	if (status != 0) {
wolfSSL	15:117db924cf7c	9245	return status;
wolfSSL	15:117db924cf7c	9246	}
wolfSSL	15:117db924cf7c	9247
wolfSSL	15:117db924cf7c	9248	status = LTC_AES_EncryptTagCcm(LTC_BASE, in, out, inSz,
wolfSSL	15:117db924cf7c	9249	nonce, nonceSz, authIn, authInSz, key, keySize, authTag, authTagSz);
wolfSSL	15:117db924cf7c	9250
wolfSSL	15:117db924cf7c	9251	return (kStatus_Success == status) ? 0 : BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9252	}
wolfSSL	15:117db924cf7c	9253
wolfSSL	15:117db924cf7c	9254	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	9255	int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
wolfSSL	15:117db924cf7c	9256	const byte* nonce, word32 nonceSz,
wolfSSL	15:117db924cf7c	9257	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	9258	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	9259	{
wolfSSL	15:117db924cf7c	9260	byte *key;
wolfSSL	15:117db924cf7c	9261	uint32_t keySize;
wolfSSL	15:117db924cf7c	9262	status_t status;
wolfSSL	15:117db924cf7c	9263
wolfSSL	15:117db924cf7c	9264	/* sanity check on arguments */
wolfSSL	15:117db924cf7c	9265	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| nonce == NULL
wolfSSL	15:117db924cf7c	9266	\|\| authTag == NULL \|\| nonceSz < 7 \|\| nonceSz > 13)
wolfSSL	15:117db924cf7c	9267	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9268
wolfSSL	15:117db924cf7c	9269	key = (byte*)aes->key;
wolfSSL	15:117db924cf7c	9270
wolfSSL	15:117db924cf7c	9271	status = wc_AesGetKeySize(aes, &keySize);
wolfSSL	15:117db924cf7c	9272	if (status != 0) {
wolfSSL	15:117db924cf7c	9273	return status;
wolfSSL	15:117db924cf7c	9274	}
wolfSSL	15:117db924cf7c	9275
wolfSSL	15:117db924cf7c	9276	status = LTC_AES_DecryptTagCcm(LTC_BASE, in, out, inSz,
wolfSSL	15:117db924cf7c	9277	nonce, nonceSz, authIn, authInSz, key, keySize, authTag, authTagSz);
wolfSSL	15:117db924cf7c	9278
wolfSSL	15:117db924cf7c	9279	if (status == kStatus_Success) {
wolfSSL	15:117db924cf7c	9280	return 0;
wolfSSL	15:117db924cf7c	9281	}
wolfSSL	15:117db924cf7c	9282	else {
wolfSSL	15:117db924cf7c	9283	XMEMSET(out, 0, inSz);
wolfSSL	15:117db924cf7c	9284	return AES_CCM_AUTH_E;
wolfSSL	15:117db924cf7c	9285	}
wolfSSL	15:117db924cf7c	9286	}
wolfSSL	15:117db924cf7c	9287	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	9288
wolfSSL	15:117db924cf7c	9289
wolfSSL	15:117db924cf7c	9290	/* software AES CCM */
wolfSSL	15:117db924cf7c	9291	#else
wolfSSL	15:117db924cf7c	9292
wolfSSL	15:117db924cf7c	9293	static void roll_x(Aes* aes, const byte* in, word32 inSz, byte* out)
wolfSSL	15:117db924cf7c	9294	{
wolfSSL	15:117db924cf7c	9295	/* process the bulk of the data */
wolfSSL	15:117db924cf7c	9296	while (inSz >= AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	9297	xorbuf(out, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9298	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9299	inSz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9300
wolfSSL	15:117db924cf7c	9301	wc_AesEncrypt(aes, out, out);
wolfSSL	15:117db924cf7c	9302	}
wolfSSL	15:117db924cf7c	9303
wolfSSL	15:117db924cf7c	9304	/* process remainder of the data */
wolfSSL	15:117db924cf7c	9305	if (inSz > 0) {
wolfSSL	15:117db924cf7c	9306	xorbuf(out, in, inSz);
wolfSSL	15:117db924cf7c	9307	wc_AesEncrypt(aes, out, out);
wolfSSL	15:117db924cf7c	9308	}
wolfSSL	15:117db924cf7c	9309	}
wolfSSL	15:117db924cf7c	9310
wolfSSL	15:117db924cf7c	9311	static void roll_auth(Aes* aes, const byte* in, word32 inSz, byte* out)
wolfSSL	15:117db924cf7c	9312	{
wolfSSL	15:117db924cf7c	9313	word32 authLenSz;
wolfSSL	15:117db924cf7c	9314	word32 remainder;
wolfSSL	15:117db924cf7c	9315
wolfSSL	15:117db924cf7c	9316	/* encode the length in */
wolfSSL	15:117db924cf7c	9317	if (inSz <= 0xFEFF) {
wolfSSL	15:117db924cf7c	9318	authLenSz = 2;
wolfSSL	15:117db924cf7c	9319	out[0] ^= ((inSz & 0xFF00) >> 8);
wolfSSL	15:117db924cf7c	9320	out[1] ^= (inSz & 0x00FF);
wolfSSL	15:117db924cf7c	9321	}
wolfSSL	15:117db924cf7c	9322	else if (inSz <= 0xFFFFFFFF) {
wolfSSL	15:117db924cf7c	9323	authLenSz = 6;
wolfSSL	15:117db924cf7c	9324	out[0] ^= 0xFF; out[1] ^= 0xFE;
wolfSSL	15:117db924cf7c	9325	out[2] ^= ((inSz & 0xFF000000) >> 24);
wolfSSL	15:117db924cf7c	9326	out[3] ^= ((inSz & 0x00FF0000) >> 16);
wolfSSL	15:117db924cf7c	9327	out[4] ^= ((inSz & 0x0000FF00) >> 8);
wolfSSL	15:117db924cf7c	9328	out[5] ^= (inSz & 0x000000FF);
wolfSSL	15:117db924cf7c	9329	}
wolfSSL	15:117db924cf7c	9330	/* Note, the protocol handles auth data up to 2^64, but we are
wolfSSL	15:117db924cf7c	9331	* using 32-bit sizes right now, so the bigger data isn't handled
wolfSSL	15:117db924cf7c	9332	* else if (inSz <= 0xFFFFFFFFFFFFFFFF) {} */
wolfSSL	15:117db924cf7c	9333	else
wolfSSL	15:117db924cf7c	9334	return;
wolfSSL	15:117db924cf7c	9335
wolfSSL	15:117db924cf7c	9336	/* start fill out the rest of the first block */
wolfSSL	15:117db924cf7c	9337	remainder = AES_BLOCK_SIZE - authLenSz;
wolfSSL	15:117db924cf7c	9338	if (inSz >= remainder) {
wolfSSL	15:117db924cf7c	9339	/* plenty of bulk data to fill the remainder of this block */
wolfSSL	15:117db924cf7c	9340	xorbuf(out + authLenSz, in, remainder);
wolfSSL	15:117db924cf7c	9341	inSz -= remainder;
wolfSSL	15:117db924cf7c	9342	in += remainder;
wolfSSL	15:117db924cf7c	9343	}
wolfSSL	15:117db924cf7c	9344	else {
wolfSSL	15:117db924cf7c	9345	/* not enough bulk data, copy what is available, and pad zero */
wolfSSL	15:117db924cf7c	9346	xorbuf(out + authLenSz, in, inSz);
wolfSSL	15:117db924cf7c	9347	inSz = 0;
wolfSSL	15:117db924cf7c	9348	}
wolfSSL	15:117db924cf7c	9349	wc_AesEncrypt(aes, out, out);
wolfSSL	15:117db924cf7c	9350
wolfSSL	15:117db924cf7c	9351	if (inSz > 0)
wolfSSL	15:117db924cf7c	9352	roll_x(aes, in, inSz, out);
wolfSSL	15:117db924cf7c	9353	}
wolfSSL	15:117db924cf7c	9354
wolfSSL	15:117db924cf7c	9355
wolfSSL	15:117db924cf7c	9356	static WC_INLINE void AesCcmCtrInc(byte* B, word32 lenSz)
wolfSSL	15:117db924cf7c	9357	{
wolfSSL	15:117db924cf7c	9358	word32 i;
wolfSSL	15:117db924cf7c	9359
wolfSSL	15:117db924cf7c	9360	for (i = 0; i < lenSz; i++) {
wolfSSL	15:117db924cf7c	9361	if (++B[AES_BLOCK_SIZE - 1 - i] != 0) return;
wolfSSL	15:117db924cf7c	9362	}
wolfSSL	15:117db924cf7c	9363	}
wolfSSL	15:117db924cf7c	9364
wolfSSL	15:117db924cf7c	9365	/* return 0 on success */
wolfSSL	15:117db924cf7c	9366	int wc_AesCcmEncrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
wolfSSL	15:117db924cf7c	9367	const byte* nonce, word32 nonceSz,
wolfSSL	15:117db924cf7c	9368	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	9369	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	9370	{
wolfSSL	15:117db924cf7c	9371	byte A[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	9372	byte B[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	9373	byte lenSz;
wolfSSL	15:117db924cf7c	9374	word32 i;
wolfSSL	15:117db924cf7c	9375	byte mask = 0xFF;
wolfSSL	15:117db924cf7c	9376	const word32 wordSz = (word32)sizeof(word32);
wolfSSL	15:117db924cf7c	9377
wolfSSL	15:117db924cf7c	9378	/* sanity check on arguments */
wolfSSL	15:117db924cf7c	9379	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| nonce == NULL
wolfSSL	15:117db924cf7c	9380	\|\| authTag == NULL \|\| nonceSz < 7 \|\| nonceSz > 13 \|\|
wolfSSL	15:117db924cf7c	9381	authTagSz > AES_BLOCK_SIZE)
wolfSSL	15:117db924cf7c	9382	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9383
wolfSSL	15:117db924cf7c	9384	XMEMCPY(B+1, nonce, nonceSz);
wolfSSL	15:117db924cf7c	9385	lenSz = AES_BLOCK_SIZE - 1 - (byte)nonceSz;
wolfSSL	15:117db924cf7c	9386	B[0] = (authInSz > 0 ? 64 : 0)
wolfSSL	15:117db924cf7c	9387	+ (8 * (((byte)authTagSz - 2) / 2))
wolfSSL	15:117db924cf7c	9388	+ (lenSz - 1);
wolfSSL	15:117db924cf7c	9389	for (i = 0; i < lenSz; i++) {
wolfSSL	15:117db924cf7c	9390	if (mask && i >= wordSz)
wolfSSL	15:117db924cf7c	9391	mask = 0x00;
wolfSSL	15:117db924cf7c	9392	B[AES_BLOCK_SIZE - 1 - i] = (inSz >> ((8 * i) & mask)) & mask;
wolfSSL	15:117db924cf7c	9393	}
wolfSSL	15:117db924cf7c	9394
wolfSSL	15:117db924cf7c	9395	wc_AesEncrypt(aes, B, A);
wolfSSL	15:117db924cf7c	9396
wolfSSL	15:117db924cf7c	9397	if (authInSz > 0)
wolfSSL	15:117db924cf7c	9398	roll_auth(aes, authIn, authInSz, A);
wolfSSL	15:117db924cf7c	9399	if (inSz > 0)
wolfSSL	15:117db924cf7c	9400	roll_x(aes, in, inSz, A);
wolfSSL	15:117db924cf7c	9401	XMEMCPY(authTag, A, authTagSz);
wolfSSL	15:117db924cf7c	9402
wolfSSL	15:117db924cf7c	9403	B[0] = lenSz - 1;
wolfSSL	15:117db924cf7c	9404	for (i = 0; i < lenSz; i++)
wolfSSL	15:117db924cf7c	9405	B[AES_BLOCK_SIZE - 1 - i] = 0;
wolfSSL	15:117db924cf7c	9406	wc_AesEncrypt(aes, B, A);
wolfSSL	15:117db924cf7c	9407	xorbuf(authTag, A, authTagSz);
wolfSSL	15:117db924cf7c	9408
wolfSSL	15:117db924cf7c	9409	B[15] = 1;
wolfSSL	15:117db924cf7c	9410	while (inSz >= AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	9411	wc_AesEncrypt(aes, B, A);
wolfSSL	15:117db924cf7c	9412	xorbuf(A, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9413	XMEMCPY(out, A, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9414
wolfSSL	15:117db924cf7c	9415	AesCcmCtrInc(B, lenSz);
wolfSSL	15:117db924cf7c	9416	inSz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9417	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9418	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9419	}
wolfSSL	15:117db924cf7c	9420	if (inSz > 0) {
wolfSSL	15:117db924cf7c	9421	wc_AesEncrypt(aes, B, A);
wolfSSL	15:117db924cf7c	9422	xorbuf(A, in, inSz);
wolfSSL	15:117db924cf7c	9423	XMEMCPY(out, A, inSz);
wolfSSL	15:117db924cf7c	9424	}
wolfSSL	15:117db924cf7c	9425
wolfSSL	15:117db924cf7c	9426	ForceZero(A, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9427	ForceZero(B, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9428
wolfSSL	15:117db924cf7c	9429	return 0;
wolfSSL	15:117db924cf7c	9430	}
wolfSSL	15:117db924cf7c	9431
wolfSSL	15:117db924cf7c	9432	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	9433	int wc_AesCcmDecrypt(Aes* aes, byte* out, const byte* in, word32 inSz,
wolfSSL	15:117db924cf7c	9434	const byte* nonce, word32 nonceSz,
wolfSSL	15:117db924cf7c	9435	const byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	9436	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	9437	{
wolfSSL	15:117db924cf7c	9438	byte A[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	9439	byte B[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	9440	byte* o;
wolfSSL	15:117db924cf7c	9441	byte lenSz;
wolfSSL	15:117db924cf7c	9442	word32 i, oSz;
wolfSSL	15:117db924cf7c	9443	int result = 0;
wolfSSL	15:117db924cf7c	9444	byte mask = 0xFF;
wolfSSL	15:117db924cf7c	9445	const word32 wordSz = (word32)sizeof(word32);
wolfSSL	15:117db924cf7c	9446
wolfSSL	15:117db924cf7c	9447	/* sanity check on arguments */
wolfSSL	15:117db924cf7c	9448	if (aes == NULL \|\| out == NULL \|\| in == NULL \|\| nonce == NULL
wolfSSL	15:117db924cf7c	9449	\|\| authTag == NULL \|\| nonceSz < 7 \|\| nonceSz > 13 \|\|
wolfSSL	15:117db924cf7c	9450	authTagSz > AES_BLOCK_SIZE)
wolfSSL	15:117db924cf7c	9451	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9452
wolfSSL	15:117db924cf7c	9453	o = out;
wolfSSL	15:117db924cf7c	9454	oSz = inSz;
wolfSSL	15:117db924cf7c	9455	XMEMCPY(B+1, nonce, nonceSz);
wolfSSL	15:117db924cf7c	9456	lenSz = AES_BLOCK_SIZE - 1 - (byte)nonceSz;
wolfSSL	15:117db924cf7c	9457
wolfSSL	15:117db924cf7c	9458	B[0] = lenSz - 1;
wolfSSL	15:117db924cf7c	9459	for (i = 0; i < lenSz; i++)
wolfSSL	15:117db924cf7c	9460	B[AES_BLOCK_SIZE - 1 - i] = 0;
wolfSSL	15:117db924cf7c	9461	B[15] = 1;
wolfSSL	15:117db924cf7c	9462
wolfSSL	15:117db924cf7c	9463	while (oSz >= AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	9464	wc_AesEncrypt(aes, B, A);
wolfSSL	15:117db924cf7c	9465	xorbuf(A, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9466	XMEMCPY(o, A, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9467
wolfSSL	15:117db924cf7c	9468	AesCcmCtrInc(B, lenSz);
wolfSSL	15:117db924cf7c	9469	oSz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9470	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9471	o += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9472	}
wolfSSL	15:117db924cf7c	9473	if (inSz > 0) {
wolfSSL	15:117db924cf7c	9474	wc_AesEncrypt(aes, B, A);
wolfSSL	15:117db924cf7c	9475	xorbuf(A, in, oSz);
wolfSSL	15:117db924cf7c	9476	XMEMCPY(o, A, oSz);
wolfSSL	15:117db924cf7c	9477	}
wolfSSL	15:117db924cf7c	9478
wolfSSL	15:117db924cf7c	9479	for (i = 0; i < lenSz; i++)
wolfSSL	15:117db924cf7c	9480	B[AES_BLOCK_SIZE - 1 - i] = 0;
wolfSSL	15:117db924cf7c	9481	wc_AesEncrypt(aes, B, A);
wolfSSL	15:117db924cf7c	9482
wolfSSL	15:117db924cf7c	9483	o = out;
wolfSSL	15:117db924cf7c	9484	oSz = inSz;
wolfSSL	15:117db924cf7c	9485
wolfSSL	15:117db924cf7c	9486	B[0] = (authInSz > 0 ? 64 : 0)
wolfSSL	15:117db924cf7c	9487	+ (8 * (((byte)authTagSz - 2) / 2))
wolfSSL	15:117db924cf7c	9488	+ (lenSz - 1);
wolfSSL	15:117db924cf7c	9489	for (i = 0; i < lenSz; i++) {
wolfSSL	15:117db924cf7c	9490	if (mask && i >= wordSz)
wolfSSL	15:117db924cf7c	9491	mask = 0x00;
wolfSSL	15:117db924cf7c	9492	B[AES_BLOCK_SIZE - 1 - i] = (inSz >> ((8 * i) & mask)) & mask;
wolfSSL	15:117db924cf7c	9493	}
wolfSSL	15:117db924cf7c	9494
wolfSSL	15:117db924cf7c	9495	wc_AesEncrypt(aes, B, A);
wolfSSL	15:117db924cf7c	9496
wolfSSL	15:117db924cf7c	9497	if (authInSz > 0)
wolfSSL	15:117db924cf7c	9498	roll_auth(aes, authIn, authInSz, A);
wolfSSL	15:117db924cf7c	9499	if (inSz > 0)
wolfSSL	15:117db924cf7c	9500	roll_x(aes, o, oSz, A);
wolfSSL	15:117db924cf7c	9501
wolfSSL	15:117db924cf7c	9502	B[0] = lenSz - 1;
wolfSSL	15:117db924cf7c	9503	for (i = 0; i < lenSz; i++)
wolfSSL	15:117db924cf7c	9504	B[AES_BLOCK_SIZE - 1 - i] = 0;
wolfSSL	15:117db924cf7c	9505	wc_AesEncrypt(aes, B, B);
wolfSSL	15:117db924cf7c	9506	xorbuf(A, B, authTagSz);
wolfSSL	15:117db924cf7c	9507
wolfSSL	15:117db924cf7c	9508	if (ConstantCompare(A, authTag, authTagSz) != 0) {
wolfSSL	15:117db924cf7c	9509	/* If the authTag check fails, don't keep the decrypted data.
wolfSSL	15:117db924cf7c	9510	* Unfortunately, you need the decrypted data to calculate the
wolfSSL	15:117db924cf7c	9511	* check value. */
wolfSSL	15:117db924cf7c	9512	XMEMSET(out, 0, inSz);
wolfSSL	15:117db924cf7c	9513	result = AES_CCM_AUTH_E;
wolfSSL	15:117db924cf7c	9514	}
wolfSSL	15:117db924cf7c	9515
wolfSSL	15:117db924cf7c	9516	ForceZero(A, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9517	ForceZero(B, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9518	o = NULL;
wolfSSL	15:117db924cf7c	9519
wolfSSL	15:117db924cf7c	9520	return result;
wolfSSL	15:117db924cf7c	9521	}
wolfSSL	15:117db924cf7c	9522
wolfSSL	15:117db924cf7c	9523	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	9524	#endif /* software AES CCM */
wolfSSL	15:117db924cf7c	9525
wolfSSL	15:117db924cf7c	9526	/* abstract functions that call lower level AESCCM functions */
wolfSSL	15:117db924cf7c	9527	#ifndef WC_NO_RNG
wolfSSL	15:117db924cf7c	9528
wolfSSL	15:117db924cf7c	9529	int wc_AesCcmSetNonce(Aes* aes, const byte* nonce, word32 nonceSz)
wolfSSL	15:117db924cf7c	9530	{
wolfSSL	15:117db924cf7c	9531	int ret = 0;
wolfSSL	15:117db924cf7c	9532
wolfSSL	15:117db924cf7c	9533	if (aes == NULL \|\| nonce == NULL \|\|
wolfSSL	15:117db924cf7c	9534	nonceSz < CCM_NONCE_MIN_SZ \|\| nonceSz > CCM_NONCE_MAX_SZ) {
wolfSSL	15:117db924cf7c	9535
wolfSSL	15:117db924cf7c	9536	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9537	}
wolfSSL	15:117db924cf7c	9538
wolfSSL	15:117db924cf7c	9539	if (ret == 0) {
wolfSSL	15:117db924cf7c	9540	XMEMCPY(aes->reg, nonce, nonceSz);
wolfSSL	15:117db924cf7c	9541	aes->nonceSz = nonceSz;
wolfSSL	15:117db924cf7c	9542
wolfSSL	15:117db924cf7c	9543	/* Invocation counter should be 2^61 */
wolfSSL	15:117db924cf7c	9544	aes->invokeCtr[0] = 0;
wolfSSL	15:117db924cf7c	9545	aes->invokeCtr[1] = 0xE0000000;
wolfSSL	15:117db924cf7c	9546	}
wolfSSL	15:117db924cf7c	9547
wolfSSL	15:117db924cf7c	9548	return ret;
wolfSSL	15:117db924cf7c	9549	}
wolfSSL	15:117db924cf7c	9550
wolfSSL	15:117db924cf7c	9551
wolfSSL	15:117db924cf7c	9552	int wc_AesCcmEncrypt_ex(Aes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	9553	byte* ivOut, word32 ivOutSz,
wolfSSL	15:117db924cf7c	9554	byte* authTag, word32 authTagSz,
wolfSSL	15:117db924cf7c	9555	const byte* authIn, word32 authInSz)
wolfSSL	15:117db924cf7c	9556	{
wolfSSL	15:117db924cf7c	9557	int ret = 0;
wolfSSL	15:117db924cf7c	9558
wolfSSL	15:117db924cf7c	9559	if (aes == NULL \|\| out == NULL \|\|
wolfSSL	15:117db924cf7c	9560	(in == NULL && sz != 0) \|\|
wolfSSL	15:117db924cf7c	9561	ivOut == NULL \|\|
wolfSSL	15:117db924cf7c	9562	(authIn == NULL && authInSz != 0) \|\|
wolfSSL	15:117db924cf7c	9563	(ivOutSz != aes->nonceSz)) {
wolfSSL	15:117db924cf7c	9564
wolfSSL	15:117db924cf7c	9565	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9566	}
wolfSSL	15:117db924cf7c	9567
wolfSSL	15:117db924cf7c	9568	if (ret == 0) {
wolfSSL	15:117db924cf7c	9569	aes->invokeCtr[0]++;
wolfSSL	15:117db924cf7c	9570	if (aes->invokeCtr[0] == 0) {
wolfSSL	15:117db924cf7c	9571	aes->invokeCtr[1]++;
wolfSSL	15:117db924cf7c	9572	if (aes->invokeCtr[1] == 0)
wolfSSL	15:117db924cf7c	9573	ret = AES_CCM_OVERFLOW_E;
wolfSSL	15:117db924cf7c	9574	}
wolfSSL	15:117db924cf7c	9575	}
wolfSSL	15:117db924cf7c	9576
wolfSSL	15:117db924cf7c	9577	if (ret == 0) {
wolfSSL	15:117db924cf7c	9578	ret = wc_AesCcmEncrypt(aes, out, in, sz,
wolfSSL	15:117db924cf7c	9579	(byte*)aes->reg, aes->nonceSz,
wolfSSL	15:117db924cf7c	9580	authTag, authTagSz,
wolfSSL	15:117db924cf7c	9581	authIn, authInSz);
wolfSSL	15:117db924cf7c	9582	XMEMCPY(ivOut, aes->reg, aes->nonceSz);
wolfSSL	15:117db924cf7c	9583	IncCtr((byte*)aes->reg, aes->nonceSz);
wolfSSL	15:117db924cf7c	9584	}
wolfSSL	15:117db924cf7c	9585
wolfSSL	15:117db924cf7c	9586	return ret;
wolfSSL	15:117db924cf7c	9587	}
wolfSSL	15:117db924cf7c	9588
wolfSSL	15:117db924cf7c	9589	#endif /* WC_NO_RNG */
wolfSSL	15:117db924cf7c	9590
wolfSSL	15:117db924cf7c	9591	#endif /* HAVE_AESCCM */
wolfSSL	15:117db924cf7c	9592
wolfSSL	15:117db924cf7c	9593
wolfSSL	15:117db924cf7c	9594	/* Initialize Aes for use with async hardware */
wolfSSL	15:117db924cf7c	9595	int wc_AesInit(Aes* aes, void* heap, int devId)
wolfSSL	15:117db924cf7c	9596	{
wolfSSL	15:117db924cf7c	9597	int ret = 0;
wolfSSL	15:117db924cf7c	9598
wolfSSL	15:117db924cf7c	9599	if (aes == NULL)
wolfSSL	15:117db924cf7c	9600	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9601
wolfSSL	15:117db924cf7c	9602	aes->heap = heap;
wolfSSL	15:117db924cf7c	9603
wolfSSL	15:117db924cf7c	9604	#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
wolfSSL	15:117db924cf7c	9605	ret = wolfAsync_DevCtxInit(&aes->asyncDev, WOLFSSL_ASYNC_MARKER_AES,
wolfSSL	15:117db924cf7c	9606	aes->heap, devId);
wolfSSL	15:117db924cf7c	9607	#else
wolfSSL	15:117db924cf7c	9608	(void)devId;
wolfSSL	15:117db924cf7c	9609	#endif /* WOLFSSL_ASYNC_CRYPT */
wolfSSL	15:117db924cf7c	9610
wolfSSL	15:117db924cf7c	9611	return ret;
wolfSSL	15:117db924cf7c	9612	}
wolfSSL	15:117db924cf7c	9613
wolfSSL	15:117db924cf7c	9614	/* Free Aes from use with async hardware */
wolfSSL	15:117db924cf7c	9615	void wc_AesFree(Aes* aes)
wolfSSL	15:117db924cf7c	9616	{
wolfSSL	15:117db924cf7c	9617	if (aes == NULL)
wolfSSL	15:117db924cf7c	9618	return;
wolfSSL	15:117db924cf7c	9619
wolfSSL	15:117db924cf7c	9620	#if defined(WOLFSSL_ASYNC_CRYPT) && defined(WC_ASYNC_ENABLE_AES)
wolfSSL	15:117db924cf7c	9621	wolfAsync_DevCtxFree(&aes->asyncDev, WOLFSSL_ASYNC_MARKER_AES);
wolfSSL	15:117db924cf7c	9622	#endif /* WOLFSSL_ASYNC_CRYPT */
wolfSSL	15:117db924cf7c	9623	}
wolfSSL	15:117db924cf7c	9624
wolfSSL	15:117db924cf7c	9625
wolfSSL	15:117db924cf7c	9626	int wc_AesGetKeySize(Aes* aes, word32* keySize)
wolfSSL	15:117db924cf7c	9627	{
wolfSSL	15:117db924cf7c	9628	int ret = 0;
wolfSSL	15:117db924cf7c	9629
wolfSSL	15:117db924cf7c	9630	if (aes == NULL \|\| keySize == NULL) {
wolfSSL	15:117db924cf7c	9631	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9632	}
wolfSSL	15:117db924cf7c	9633
wolfSSL	15:117db924cf7c	9634	switch (aes->rounds) {
wolfSSL	15:117db924cf7c	9635	#ifdef WOLFSSL_AES_128
wolfSSL	15:117db924cf7c	9636	case 10:
wolfSSL	15:117db924cf7c	9637	*keySize = 16;
wolfSSL	15:117db924cf7c	9638	break;
wolfSSL	15:117db924cf7c	9639	#endif
wolfSSL	15:117db924cf7c	9640	#ifdef WOLFSSL_AES_192
wolfSSL	15:117db924cf7c	9641	case 12:
wolfSSL	15:117db924cf7c	9642	*keySize = 24;
wolfSSL	15:117db924cf7c	9643	break;
wolfSSL	15:117db924cf7c	9644	#endif
wolfSSL	15:117db924cf7c	9645	#ifdef WOLFSSL_AES_256
wolfSSL	15:117db924cf7c	9646	case 14:
wolfSSL	15:117db924cf7c	9647	*keySize = 32;
wolfSSL	15:117db924cf7c	9648	break;
wolfSSL	15:117db924cf7c	9649	#endif
wolfSSL	15:117db924cf7c	9650	default:
wolfSSL	15:117db924cf7c	9651	*keySize = 0;
wolfSSL	15:117db924cf7c	9652	ret = BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9653	}
wolfSSL	15:117db924cf7c	9654
wolfSSL	15:117db924cf7c	9655	return ret;
wolfSSL	15:117db924cf7c	9656	}
wolfSSL	15:117db924cf7c	9657
wolfSSL	15:117db924cf7c	9658	#endif /* !WOLFSSL_TI_CRYPT */
wolfSSL	15:117db924cf7c	9659
wolfSSL	15:117db924cf7c	9660	#ifdef HAVE_AES_ECB
wolfSSL	15:117db924cf7c	9661	#if defined(WOLFSSL_IMX6_CAAM) && !defined(NO_IMX6_CAAM_AES)
wolfSSL	15:117db924cf7c	9662	/* implemented in wolfcrypt/src/port/caam/caam_aes.c */
wolfSSL	15:117db924cf7c	9663	#else
wolfSSL	15:117db924cf7c	9664
wolfSSL	15:117db924cf7c	9665	/* software implementation */
wolfSSL	15:117db924cf7c	9666	int wc_AesEcbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	9667	{
wolfSSL	15:117db924cf7c	9668	word32 blocks = sz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9669
wolfSSL	15:117db924cf7c	9670	if ((in == NULL) \|\| (out == NULL) \|\| (aes == NULL))
wolfSSL	15:117db924cf7c	9671	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9672	while (blocks>0) {
wolfSSL	15:117db924cf7c	9673	wc_AesEncryptDirect(aes, out, in);
wolfSSL	15:117db924cf7c	9674	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9675	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9676	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9677	blocks--;
wolfSSL	15:117db924cf7c	9678	}
wolfSSL	15:117db924cf7c	9679	return 0;
wolfSSL	15:117db924cf7c	9680	}
wolfSSL	15:117db924cf7c	9681
wolfSSL	15:117db924cf7c	9682
wolfSSL	15:117db924cf7c	9683	int wc_AesEcbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	9684	{
wolfSSL	15:117db924cf7c	9685	word32 blocks = sz / AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9686
wolfSSL	15:117db924cf7c	9687	if ((in == NULL) \|\| (out == NULL) \|\| (aes == NULL))
wolfSSL	15:117db924cf7c	9688	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9689	while (blocks>0) {
wolfSSL	15:117db924cf7c	9690	wc_AesDecryptDirect(aes, out, in);
wolfSSL	15:117db924cf7c	9691	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9692	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9693	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9694	blocks--;
wolfSSL	15:117db924cf7c	9695	}
wolfSSL	15:117db924cf7c	9696	return 0;
wolfSSL	15:117db924cf7c	9697	}
wolfSSL	15:117db924cf7c	9698	#endif
wolfSSL	15:117db924cf7c	9699	#endif /* HAVE_AES_ECB */
wolfSSL	15:117db924cf7c	9700
wolfSSL	15:117db924cf7c	9701	#ifdef WOLFSSL_AES_CFB
wolfSSL	15:117db924cf7c	9702	/* CFB 128
wolfSSL	15:117db924cf7c	9703	*
wolfSSL	15:117db924cf7c	9704	* aes structure holding key to use for encryption
wolfSSL	15:117db924cf7c	9705	* out buffer to hold result of encryption (must be at least as large as input
wolfSSL	15:117db924cf7c	9706	* buffer)
wolfSSL	15:117db924cf7c	9707	* in buffer to encrypt
wolfSSL	15:117db924cf7c	9708	* sz size of input buffer
wolfSSL	15:117db924cf7c	9709	*
wolfSSL	15:117db924cf7c	9710	* returns 0 on success and negative error values on failure
wolfSSL	15:117db924cf7c	9711	*/
wolfSSL	15:117db924cf7c	9712	int wc_AesCfbEncrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	9713	{
wolfSSL	15:117db924cf7c	9714	byte* tmp = NULL;
wolfSSL	15:117db924cf7c	9715	byte* reg = NULL;
wolfSSL	15:117db924cf7c	9716
wolfSSL	15:117db924cf7c	9717	WOLFSSL_ENTER("wc_AesCfbEncrypt");
wolfSSL	15:117db924cf7c	9718
wolfSSL	15:117db924cf7c	9719	if (aes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	9720	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9721	}
wolfSSL	15:117db924cf7c	9722
wolfSSL	15:117db924cf7c	9723	if (aes->left && sz) {
wolfSSL	15:117db924cf7c	9724	reg = (byte*)aes->reg + AES_BLOCK_SIZE - aes->left;
wolfSSL	15:117db924cf7c	9725	}
wolfSSL	15:117db924cf7c	9726
wolfSSL	15:117db924cf7c	9727	/* consume any unused bytes left in aes->tmp */
wolfSSL	15:117db924cf7c	9728	tmp = (byte*)aes->tmp + AES_BLOCK_SIZE - aes->left;
wolfSSL	15:117db924cf7c	9729	while (aes->left && sz) {
wolfSSL	15:117db924cf7c	9730	(out++) = (reg++) = (in++) ^ (tmp++);
wolfSSL	15:117db924cf7c	9731	aes->left--;
wolfSSL	15:117db924cf7c	9732	sz--;
wolfSSL	15:117db924cf7c	9733	}
wolfSSL	15:117db924cf7c	9734
wolfSSL	15:117db924cf7c	9735	while (sz >= AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	9736	wc_AesEncryptDirect(aes, out, (byte*)aes->reg);
wolfSSL	15:117db924cf7c	9737	xorbuf(out, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9738	XMEMCPY(aes->reg, out, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9739	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9740	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9741	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9742	aes->left = 0;
wolfSSL	15:117db924cf7c	9743	}
wolfSSL	15:117db924cf7c	9744
wolfSSL	15:117db924cf7c	9745	/* encrypt left over data */
wolfSSL	15:117db924cf7c	9746	if (sz) {
wolfSSL	15:117db924cf7c	9747	wc_AesEncryptDirect(aes, (byte)aes->tmp, (byte)aes->reg);
wolfSSL	15:117db924cf7c	9748	aes->left = AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9749	tmp = (byte*)aes->tmp;
wolfSSL	15:117db924cf7c	9750	reg = (byte*)aes->reg;
wolfSSL	15:117db924cf7c	9751
wolfSSL	15:117db924cf7c	9752	while (sz--) {
wolfSSL	15:117db924cf7c	9753	(out++) = (reg++) = (in++) ^ (tmp++);
wolfSSL	15:117db924cf7c	9754	aes->left--;
wolfSSL	15:117db924cf7c	9755	}
wolfSSL	15:117db924cf7c	9756	}
wolfSSL	15:117db924cf7c	9757
wolfSSL	15:117db924cf7c	9758	return 0;
wolfSSL	15:117db924cf7c	9759	}
wolfSSL	15:117db924cf7c	9760
wolfSSL	15:117db924cf7c	9761
wolfSSL	15:117db924cf7c	9762	#ifdef HAVE_AES_DECRYPT
wolfSSL	15:117db924cf7c	9763	/* CFB 128
wolfSSL	15:117db924cf7c	9764	*
wolfSSL	15:117db924cf7c	9765	* aes structure holding key to use for decryption
wolfSSL	15:117db924cf7c	9766	* out buffer to hold result of decryption (must be at least as large as input
wolfSSL	15:117db924cf7c	9767	* buffer)
wolfSSL	15:117db924cf7c	9768	* in buffer to decrypt
wolfSSL	15:117db924cf7c	9769	* sz size of input buffer
wolfSSL	15:117db924cf7c	9770	*
wolfSSL	15:117db924cf7c	9771	* returns 0 on success and negative error values on failure
wolfSSL	15:117db924cf7c	9772	*/
wolfSSL	15:117db924cf7c	9773	int wc_AesCfbDecrypt(Aes* aes, byte* out, const byte* in, word32 sz)
wolfSSL	15:117db924cf7c	9774	{
wolfSSL	15:117db924cf7c	9775	byte* tmp;
wolfSSL	15:117db924cf7c	9776
wolfSSL	15:117db924cf7c	9777	WOLFSSL_ENTER("wc_AesCfbDecrypt");
wolfSSL	15:117db924cf7c	9778
wolfSSL	15:117db924cf7c	9779	if (aes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	9780	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9781	}
wolfSSL	15:117db924cf7c	9782
wolfSSL	15:117db924cf7c	9783	/* check if more input needs copied over to aes->reg */
wolfSSL	15:117db924cf7c	9784	if (aes->left && sz) {
wolfSSL	15:117db924cf7c	9785	int size = min(aes->left, sz);
wolfSSL	15:117db924cf7c	9786	XMEMCPY((byte*)aes->reg + AES_BLOCK_SIZE - aes->left, in, size);
wolfSSL	15:117db924cf7c	9787	}
wolfSSL	15:117db924cf7c	9788
wolfSSL	15:117db924cf7c	9789	/* consume any unused bytes left in aes->tmp */
wolfSSL	15:117db924cf7c	9790	tmp = (byte*)aes->tmp + AES_BLOCK_SIZE - aes->left;
wolfSSL	15:117db924cf7c	9791	while (aes->left && sz) {
wolfSSL	15:117db924cf7c	9792	(out++) = (in++) ^ *(tmp++);
wolfSSL	15:117db924cf7c	9793	aes->left--;
wolfSSL	15:117db924cf7c	9794	sz--;
wolfSSL	15:117db924cf7c	9795	}
wolfSSL	15:117db924cf7c	9796
wolfSSL	15:117db924cf7c	9797	while (sz > AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	9798	wc_AesEncryptDirect(aes, out, (byte*)aes->reg);
wolfSSL	15:117db924cf7c	9799	xorbuf(out, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9800	XMEMCPY(aes->reg, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9801	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9802	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9803	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9804	aes->left = 0;
wolfSSL	15:117db924cf7c	9805	}
wolfSSL	15:117db924cf7c	9806
wolfSSL	15:117db924cf7c	9807	/* decrypt left over data */
wolfSSL	15:117db924cf7c	9808	if (sz) {
wolfSSL	15:117db924cf7c	9809	wc_AesEncryptDirect(aes, (byte)aes->tmp, (byte)aes->reg);
wolfSSL	15:117db924cf7c	9810	XMEMCPY(aes->reg, in, sz);
wolfSSL	15:117db924cf7c	9811	aes->left = AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9812	tmp = (byte*)aes->tmp;
wolfSSL	15:117db924cf7c	9813
wolfSSL	15:117db924cf7c	9814	while (sz--) {
wolfSSL	15:117db924cf7c	9815	(out++) = (in++) ^ *(tmp++);
wolfSSL	15:117db924cf7c	9816	aes->left--;
wolfSSL	15:117db924cf7c	9817	}
wolfSSL	15:117db924cf7c	9818	}
wolfSSL	15:117db924cf7c	9819
wolfSSL	15:117db924cf7c	9820	return 0;
wolfSSL	15:117db924cf7c	9821	}
wolfSSL	15:117db924cf7c	9822	#endif /* HAVE_AES_DECRYPT */
wolfSSL	15:117db924cf7c	9823	#endif /* WOLFSSL_AES_CFB */
wolfSSL	15:117db924cf7c	9824
wolfSSL	15:117db924cf7c	9825
wolfSSL	15:117db924cf7c	9826	#ifdef HAVE_AES_KEYWRAP
wolfSSL	15:117db924cf7c	9827
wolfSSL	15:117db924cf7c	9828	/* Initialize key wrap counter with value */
wolfSSL	15:117db924cf7c	9829	static WC_INLINE void InitKeyWrapCounter(byte* inOutCtr, word32 value)
wolfSSL	15:117db924cf7c	9830	{
wolfSSL	15:117db924cf7c	9831	int i;
wolfSSL	15:117db924cf7c	9832	word32 bytes;
wolfSSL	15:117db924cf7c	9833
wolfSSL	15:117db924cf7c	9834	bytes = sizeof(word32);
wolfSSL	15:117db924cf7c	9835	for (i = 0; i < (int)sizeof(word32); i++) {
wolfSSL	15:117db924cf7c	9836	inOutCtr[i+sizeof(word32)] = (value >> ((bytes - 1) * 8)) & 0xFF;
wolfSSL	15:117db924cf7c	9837	bytes--;
wolfSSL	15:117db924cf7c	9838	}
wolfSSL	15:117db924cf7c	9839	}
wolfSSL	15:117db924cf7c	9840
wolfSSL	15:117db924cf7c	9841	/* Increment key wrap counter */
wolfSSL	15:117db924cf7c	9842	static WC_INLINE void IncrementKeyWrapCounter(byte* inOutCtr)
wolfSSL	15:117db924cf7c	9843	{
wolfSSL	15:117db924cf7c	9844	int i;
wolfSSL	15:117db924cf7c	9845
wolfSSL	15:117db924cf7c	9846	/* in network byte order so start at end and work back */
wolfSSL	15:117db924cf7c	9847	for (i = KEYWRAP_BLOCK_SIZE - 1; i >= 0; i--) {
wolfSSL	15:117db924cf7c	9848	if (++inOutCtr[i]) /* we're done unless we overflow */
wolfSSL	15:117db924cf7c	9849	return;
wolfSSL	15:117db924cf7c	9850	}
wolfSSL	15:117db924cf7c	9851	}
wolfSSL	15:117db924cf7c	9852
wolfSSL	15:117db924cf7c	9853	/* Decrement key wrap counter */
wolfSSL	15:117db924cf7c	9854	static WC_INLINE void DecrementKeyWrapCounter(byte* inOutCtr)
wolfSSL	15:117db924cf7c	9855	{
wolfSSL	15:117db924cf7c	9856	int i;
wolfSSL	15:117db924cf7c	9857
wolfSSL	15:117db924cf7c	9858	for (i = KEYWRAP_BLOCK_SIZE - 1; i >= 0; i--) {
wolfSSL	15:117db924cf7c	9859	if (--inOutCtr[i] != 0xFF) /* we're done unless we underflow */
wolfSSL	15:117db924cf7c	9860	return;
wolfSSL	15:117db924cf7c	9861	}
wolfSSL	15:117db924cf7c	9862	}
wolfSSL	15:117db924cf7c	9863
wolfSSL	15:117db924cf7c	9864	/* perform AES key wrap (RFC3394), return out sz on success, negative on err */
wolfSSL	15:117db924cf7c	9865	int wc_AesKeyWrap(const byte* key, word32 keySz, const byte* in, word32 inSz,
wolfSSL	15:117db924cf7c	9866	byte* out, word32 outSz, const byte* iv)
wolfSSL	15:117db924cf7c	9867	{
wolfSSL	15:117db924cf7c	9868	Aes aes;
wolfSSL	15:117db924cf7c	9869	byte* r;
wolfSSL	15:117db924cf7c	9870	word32 i;
wolfSSL	15:117db924cf7c	9871	int ret, j;
wolfSSL	15:117db924cf7c	9872
wolfSSL	15:117db924cf7c	9873	byte t[KEYWRAP_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	9874	byte tmp[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	9875
wolfSSL	15:117db924cf7c	9876	/* n must be at least 2, output size is n + 8 bytes */
wolfSSL	15:117db924cf7c	9877	if (key == NULL \|\| in == NULL \|\| inSz < 2 \|\|
wolfSSL	15:117db924cf7c	9878	out == NULL \|\| outSz < (inSz + KEYWRAP_BLOCK_SIZE))
wolfSSL	15:117db924cf7c	9879	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9880
wolfSSL	15:117db924cf7c	9881	/* input must be multiple of 64-bits */
wolfSSL	15:117db924cf7c	9882	if (inSz % KEYWRAP_BLOCK_SIZE != 0)
wolfSSL	15:117db924cf7c	9883	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9884
wolfSSL	15:117db924cf7c	9885	/* user IV is optional */
wolfSSL	15:117db924cf7c	9886	if (iv == NULL) {
wolfSSL	15:117db924cf7c	9887	XMEMSET(tmp, 0xA6, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9888	} else {
wolfSSL	15:117db924cf7c	9889	XMEMCPY(tmp, iv, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9890	}
wolfSSL	15:117db924cf7c	9891
wolfSSL	15:117db924cf7c	9892	r = out + 8;
wolfSSL	15:117db924cf7c	9893	XMEMCPY(r, in, inSz);
wolfSSL	15:117db924cf7c	9894	XMEMSET(t, 0, sizeof(t));
wolfSSL	15:117db924cf7c	9895
wolfSSL	15:117db924cf7c	9896	ret = wc_AesInit(&aes, NULL, INVALID_DEVID);
wolfSSL	15:117db924cf7c	9897	if (ret != 0)
wolfSSL	15:117db924cf7c	9898	return ret;
wolfSSL	15:117db924cf7c	9899
wolfSSL	15:117db924cf7c	9900	ret = wc_AesSetKey(&aes, key, keySz, NULL, AES_ENCRYPTION);
wolfSSL	15:117db924cf7c	9901	if (ret != 0)
wolfSSL	15:117db924cf7c	9902	return ret;
wolfSSL	15:117db924cf7c	9903
wolfSSL	15:117db924cf7c	9904	for (j = 0; j <= 5; j++) {
wolfSSL	15:117db924cf7c	9905	for (i = 1; i <= inSz / KEYWRAP_BLOCK_SIZE; i++) {
wolfSSL	15:117db924cf7c	9906
wolfSSL	15:117db924cf7c	9907	/* load R[i] */
wolfSSL	15:117db924cf7c	9908	XMEMCPY(tmp + KEYWRAP_BLOCK_SIZE, r, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9909
wolfSSL	15:117db924cf7c	9910	wc_AesEncryptDirect(&aes, tmp, tmp);
wolfSSL	15:117db924cf7c	9911
wolfSSL	15:117db924cf7c	9912	/* calculate new A */
wolfSSL	15:117db924cf7c	9913	IncrementKeyWrapCounter(t);
wolfSSL	15:117db924cf7c	9914	xorbuf(tmp, t, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9915
wolfSSL	15:117db924cf7c	9916	/* save R[i] */
wolfSSL	15:117db924cf7c	9917	XMEMCPY(r, tmp + KEYWRAP_BLOCK_SIZE, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9918	r += KEYWRAP_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9919	}
wolfSSL	15:117db924cf7c	9920	r = out + KEYWRAP_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9921	}
wolfSSL	15:117db924cf7c	9922
wolfSSL	15:117db924cf7c	9923	/* C[0] = A */
wolfSSL	15:117db924cf7c	9924	XMEMCPY(out, tmp, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9925
wolfSSL	15:117db924cf7c	9926	wc_AesFree(&aes);
wolfSSL	15:117db924cf7c	9927
wolfSSL	15:117db924cf7c	9928	return inSz + KEYWRAP_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9929	}
wolfSSL	15:117db924cf7c	9930
wolfSSL	15:117db924cf7c	9931	int wc_AesKeyUnWrap(const byte* key, word32 keySz, const byte* in, word32 inSz,
wolfSSL	15:117db924cf7c	9932	byte* out, word32 outSz, const byte* iv)
wolfSSL	15:117db924cf7c	9933	{
wolfSSL	15:117db924cf7c	9934	Aes aes;
wolfSSL	15:117db924cf7c	9935	byte* r;
wolfSSL	15:117db924cf7c	9936	word32 i, n;
wolfSSL	15:117db924cf7c	9937	int ret, j;
wolfSSL	15:117db924cf7c	9938
wolfSSL	15:117db924cf7c	9939	byte t[KEYWRAP_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	9940	byte tmp[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	9941
wolfSSL	15:117db924cf7c	9942	const byte* expIv;
wolfSSL	15:117db924cf7c	9943	const byte defaultIV[] = {
wolfSSL	15:117db924cf7c	9944	0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6, 0xA6
wolfSSL	15:117db924cf7c	9945	};
wolfSSL	15:117db924cf7c	9946
wolfSSL	15:117db924cf7c	9947	(void)iv;
wolfSSL	15:117db924cf7c	9948
wolfSSL	15:117db924cf7c	9949	if (key == NULL \|\| in == NULL \|\| inSz < 3 \|\|
wolfSSL	15:117db924cf7c	9950	out == NULL \|\| outSz < (inSz - KEYWRAP_BLOCK_SIZE))
wolfSSL	15:117db924cf7c	9951	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9952
wolfSSL	15:117db924cf7c	9953	/* input must be multiple of 64-bits */
wolfSSL	15:117db924cf7c	9954	if (inSz % KEYWRAP_BLOCK_SIZE != 0)
wolfSSL	15:117db924cf7c	9955	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	9956
wolfSSL	15:117db924cf7c	9957	/* user IV optional */
wolfSSL	15:117db924cf7c	9958	if (iv != NULL) {
wolfSSL	15:117db924cf7c	9959	expIv = iv;
wolfSSL	15:117db924cf7c	9960	} else {
wolfSSL	15:117db924cf7c	9961	expIv = defaultIV;
wolfSSL	15:117db924cf7c	9962	}
wolfSSL	15:117db924cf7c	9963
wolfSSL	15:117db924cf7c	9964	/* A = C[0], R[i] = C[i] */
wolfSSL	15:117db924cf7c	9965	XMEMCPY(tmp, in, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9966	XMEMCPY(out, in + KEYWRAP_BLOCK_SIZE, inSz - KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9967	XMEMSET(t, 0, sizeof(t));
wolfSSL	15:117db924cf7c	9968
wolfSSL	15:117db924cf7c	9969	ret = wc_AesInit(&aes, NULL, INVALID_DEVID);
wolfSSL	15:117db924cf7c	9970	if (ret != 0)
wolfSSL	15:117db924cf7c	9971	return ret;
wolfSSL	15:117db924cf7c	9972
wolfSSL	15:117db924cf7c	9973	ret = wc_AesSetKey(&aes, key, keySz, NULL, AES_DECRYPTION);
wolfSSL	15:117db924cf7c	9974	if (ret != 0)
wolfSSL	15:117db924cf7c	9975	return ret;
wolfSSL	15:117db924cf7c	9976
wolfSSL	15:117db924cf7c	9977	/* initialize counter to 6n */
wolfSSL	15:117db924cf7c	9978	n = (inSz - 1) / KEYWRAP_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	9979	InitKeyWrapCounter(t, 6 * n);
wolfSSL	15:117db924cf7c	9980
wolfSSL	15:117db924cf7c	9981	for (j = 5; j >= 0; j--) {
wolfSSL	15:117db924cf7c	9982	for (i = n; i >= 1; i--) {
wolfSSL	15:117db924cf7c	9983
wolfSSL	15:117db924cf7c	9984	/* calculate A */
wolfSSL	15:117db924cf7c	9985	xorbuf(tmp, t, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9986	DecrementKeyWrapCounter(t);
wolfSSL	15:117db924cf7c	9987
wolfSSL	15:117db924cf7c	9988	/* load R[i], starting at end of R */
wolfSSL	15:117db924cf7c	9989	r = out + ((i - 1) * KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9990	XMEMCPY(tmp + KEYWRAP_BLOCK_SIZE, r, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9991	wc_AesDecryptDirect(&aes, tmp, tmp);
wolfSSL	15:117db924cf7c	9992
wolfSSL	15:117db924cf7c	9993	/* save R[i] */
wolfSSL	15:117db924cf7c	9994	XMEMCPY(r, tmp + KEYWRAP_BLOCK_SIZE, KEYWRAP_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	9995	}
wolfSSL	15:117db924cf7c	9996	}
wolfSSL	15:117db924cf7c	9997
wolfSSL	15:117db924cf7c	9998	wc_AesFree(&aes);
wolfSSL	15:117db924cf7c	9999
wolfSSL	15:117db924cf7c	10000	/* verify IV */
wolfSSL	15:117db924cf7c	10001	if (XMEMCMP(tmp, expIv, KEYWRAP_BLOCK_SIZE) != 0)
wolfSSL	15:117db924cf7c	10002	return BAD_KEYWRAP_IV_E;
wolfSSL	15:117db924cf7c	10003
wolfSSL	15:117db924cf7c	10004	return inSz - KEYWRAP_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10005	}
wolfSSL	15:117db924cf7c	10006
wolfSSL	15:117db924cf7c	10007	#endif /* HAVE_AES_KEYWRAP */
wolfSSL	15:117db924cf7c	10008
wolfSSL	15:117db924cf7c	10009	#ifdef WOLFSSL_AES_XTS
wolfSSL	15:117db924cf7c	10010
wolfSSL	15:117db924cf7c	10011	/* Galios Field to use */
wolfSSL	15:117db924cf7c	10012	#define GF_XTS 0x87
wolfSSL	15:117db924cf7c	10013
wolfSSL	15:117db924cf7c	10014	/* This is to help with setting keys to correct encrypt or decrypt type.
wolfSSL	15:117db924cf7c	10015	*
wolfSSL	15:117db924cf7c	10016	* tweak AES key for tweak in XTS
wolfSSL	15:117db924cf7c	10017	* aes AES key for encrypt/decrypt process
wolfSSL	15:117db924cf7c	10018	* key buffer holding aes key \| tweak key
wolfSSL	15:117db924cf7c	10019	* len length of key buffer in bytes. Should be twice that of key size. i.e.
wolfSSL	15:117db924cf7c	10020	* 32 for a 16 byte key.
wolfSSL	15:117db924cf7c	10021	* dir direction, either AES_ENCRYPTION or AES_DECRYPTION
wolfSSL	15:117db924cf7c	10022	* heap heap hint to use for memory. Can be NULL
wolfSSL	15:117db924cf7c	10023	* devId id to use with async crypto. Can be 0
wolfSSL	15:117db924cf7c	10024	*
wolfSSL	15:117db924cf7c	10025	* Note: is up to user to call wc_AesFree on tweak and aes key when done.
wolfSSL	15:117db924cf7c	10026	*
wolfSSL	15:117db924cf7c	10027	* return 0 on success
wolfSSL	15:117db924cf7c	10028	*/
wolfSSL	15:117db924cf7c	10029	int wc_AesXtsSetKey(XtsAes* aes, const byte* key, word32 len, int dir,
wolfSSL	15:117db924cf7c	10030	void* heap, int devId)
wolfSSL	15:117db924cf7c	10031	{
wolfSSL	15:117db924cf7c	10032	word32 keySz;
wolfSSL	15:117db924cf7c	10033	int ret = 0;
wolfSSL	15:117db924cf7c	10034
wolfSSL	15:117db924cf7c	10035	if (aes == NULL \|\| key == NULL) {
wolfSSL	15:117db924cf7c	10036	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	10037	}
wolfSSL	15:117db924cf7c	10038
wolfSSL	15:117db924cf7c	10039	if ((ret = wc_AesInit(&aes->tweak, heap, devId)) != 0) {
wolfSSL	15:117db924cf7c	10040	return ret;
wolfSSL	15:117db924cf7c	10041	}
wolfSSL	15:117db924cf7c	10042	if ((ret = wc_AesInit(&aes->aes, heap, devId)) != 0) {
wolfSSL	15:117db924cf7c	10043	return ret;
wolfSSL	15:117db924cf7c	10044	}
wolfSSL	15:117db924cf7c	10045
wolfSSL	15:117db924cf7c	10046	keySz = len/2;
wolfSSL	15:117db924cf7c	10047	if (keySz != 16 && keySz != 32) {
wolfSSL	15:117db924cf7c	10048	WOLFSSL_MSG("Unsupported key size");
wolfSSL	15:117db924cf7c	10049	return WC_KEY_SIZE_E;
wolfSSL	15:117db924cf7c	10050	}
wolfSSL	15:117db924cf7c	10051
wolfSSL	15:117db924cf7c	10052	if ((ret = wc_AesSetKey(&aes->aes, key, keySz, NULL, dir)) == 0) {
wolfSSL	15:117db924cf7c	10053	ret = wc_AesSetKey(&aes->tweak, key + keySz, keySz, NULL,
wolfSSL	15:117db924cf7c	10054	AES_ENCRYPTION);
wolfSSL	15:117db924cf7c	10055	if (ret != 0) {
wolfSSL	15:117db924cf7c	10056	wc_AesFree(&aes->aes);
wolfSSL	15:117db924cf7c	10057	}
wolfSSL	15:117db924cf7c	10058	}
wolfSSL	15:117db924cf7c	10059
wolfSSL	15:117db924cf7c	10060	return ret;
wolfSSL	15:117db924cf7c	10061	}
wolfSSL	15:117db924cf7c	10062
wolfSSL	15:117db924cf7c	10063
wolfSSL	15:117db924cf7c	10064	/* This is used to free up resources used by Aes structs
wolfSSL	15:117db924cf7c	10065	*
wolfSSL	15:117db924cf7c	10066	* aes AES keys to free
wolfSSL	15:117db924cf7c	10067	*
wolfSSL	15:117db924cf7c	10068	* return 0 on success
wolfSSL	15:117db924cf7c	10069	*/
wolfSSL	15:117db924cf7c	10070	int wc_AesXtsFree(XtsAes* aes)
wolfSSL	15:117db924cf7c	10071	{
wolfSSL	15:117db924cf7c	10072	if (aes != NULL) {
wolfSSL	15:117db924cf7c	10073	wc_AesFree(&aes->aes);
wolfSSL	15:117db924cf7c	10074	wc_AesFree(&aes->tweak);
wolfSSL	15:117db924cf7c	10075	}
wolfSSL	15:117db924cf7c	10076
wolfSSL	15:117db924cf7c	10077	return 0;
wolfSSL	15:117db924cf7c	10078	}
wolfSSL	15:117db924cf7c	10079
wolfSSL	15:117db924cf7c	10080
wolfSSL	15:117db924cf7c	10081	/* Same process as wc_AesXtsEncrypt but uses a word64 type as the tweak value
wolfSSL	15:117db924cf7c	10082	* instead of a byte array. This just converts the word64 to a byte array and
wolfSSL	15:117db924cf7c	10083	* calls wc_AesXtsEncrypt.
wolfSSL	15:117db924cf7c	10084	*
wolfSSL	15:117db924cf7c	10085	* aes AES keys to use for block encrypt/decrypt
wolfSSL	15:117db924cf7c	10086	* out output buffer to hold cipher text
wolfSSL	15:117db924cf7c	10087	* in input plain text buffer to encrypt
wolfSSL	15:117db924cf7c	10088	* sz size of both out and in buffers
wolfSSL	15:117db924cf7c	10089	* sector value to use for tweak
wolfSSL	15:117db924cf7c	10090	*
wolfSSL	15:117db924cf7c	10091	* returns 0 on success
wolfSSL	15:117db924cf7c	10092	*/
wolfSSL	15:117db924cf7c	10093	int wc_AesXtsEncryptSector(XtsAes* aes, byte* out, const byte* in,
wolfSSL	15:117db924cf7c	10094	word32 sz, word64 sector)
wolfSSL	15:117db924cf7c	10095	{
wolfSSL	15:117db924cf7c	10096	byte* pt;
wolfSSL	15:117db924cf7c	10097	byte i[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10098
wolfSSL	15:117db924cf7c	10099	XMEMSET(i, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10100	#ifdef BIG_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	10101	sector = ByteReverseWord64(sector);
wolfSSL	15:117db924cf7c	10102	#endif
wolfSSL	15:117db924cf7c	10103	pt = (byte*)&sector;
wolfSSL	15:117db924cf7c	10104	XMEMCPY(i, pt, sizeof(word64));
wolfSSL	15:117db924cf7c	10105
wolfSSL	15:117db924cf7c	10106	return wc_AesXtsEncrypt(aes, out, in, sz, (const byte*)i, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10107	}
wolfSSL	15:117db924cf7c	10108
wolfSSL	15:117db924cf7c	10109
wolfSSL	15:117db924cf7c	10110	/* Same process as wc_AesXtsDecrypt but uses a word64 type as the tweak value
wolfSSL	15:117db924cf7c	10111	* instead of a byte array. This just converts the word64 to a byte array.
wolfSSL	15:117db924cf7c	10112	*
wolfSSL	15:117db924cf7c	10113	* aes AES keys to use for block encrypt/decrypt
wolfSSL	15:117db924cf7c	10114	* out output buffer to hold plain text
wolfSSL	15:117db924cf7c	10115	* in input cipher text buffer to encrypt
wolfSSL	15:117db924cf7c	10116	* sz size of both out and in buffers
wolfSSL	15:117db924cf7c	10117	* sector value to use for tweak
wolfSSL	15:117db924cf7c	10118	*
wolfSSL	15:117db924cf7c	10119	* returns 0 on success
wolfSSL	15:117db924cf7c	10120	*/
wolfSSL	15:117db924cf7c	10121	int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	10122	word64 sector)
wolfSSL	15:117db924cf7c	10123	{
wolfSSL	15:117db924cf7c	10124	byte* pt;
wolfSSL	15:117db924cf7c	10125	byte i[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10126
wolfSSL	15:117db924cf7c	10127	XMEMSET(i, 0, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10128	#ifdef BIG_ENDIAN_ORDER
wolfSSL	15:117db924cf7c	10129	sector = ByteReverseWord64(sector);
wolfSSL	15:117db924cf7c	10130	#endif
wolfSSL	15:117db924cf7c	10131	pt = (byte*)&sector;
wolfSSL	15:117db924cf7c	10132	XMEMCPY(i, pt, sizeof(word64));
wolfSSL	15:117db924cf7c	10133
wolfSSL	15:117db924cf7c	10134	return wc_AesXtsDecrypt(aes, out, in, sz, (const byte*)i, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10135	}
wolfSSL	15:117db924cf7c	10136
wolfSSL	15:117db924cf7c	10137	#ifdef HAVE_AES_ECB
wolfSSL	15:117db924cf7c	10138	/* helper function for encrypting / decrypting full buffer at once */
wolfSSL	15:117db924cf7c	10139	static int _AesXtsHelper(Aes* aes, byte* out, const byte* in, word32 sz, int dir)
wolfSSL	15:117db924cf7c	10140	{
wolfSSL	15:117db924cf7c	10141	word32 outSz = sz;
wolfSSL	15:117db924cf7c	10142	word32 totalSz = (sz / AES_BLOCK_SIZE) * AES_BLOCK_SIZE; /* total bytes */
wolfSSL	15:117db924cf7c	10143	byte* pt = out;
wolfSSL	15:117db924cf7c	10144
wolfSSL	15:117db924cf7c	10145	outSz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10146
wolfSSL	15:117db924cf7c	10147	while (outSz > 0) {
wolfSSL	15:117db924cf7c	10148	word32 j;
wolfSSL	15:117db924cf7c	10149	byte carry = 0;
wolfSSL	15:117db924cf7c	10150
wolfSSL	15:117db924cf7c	10151	/* multiply by shift left and propogate carry */
wolfSSL	15:117db924cf7c	10152	for (j = 0; j < AES_BLOCK_SIZE && outSz > 0; j++, outSz--) {
wolfSSL	15:117db924cf7c	10153	byte tmpC;
wolfSSL	15:117db924cf7c	10154
wolfSSL	15:117db924cf7c	10155	tmpC = (pt[j] >> 7) & 0x01;
wolfSSL	15:117db924cf7c	10156	pt[j+AES_BLOCK_SIZE] = ((pt[j] << 1) + carry) & 0xFF;
wolfSSL	15:117db924cf7c	10157	carry = tmpC;
wolfSSL	15:117db924cf7c	10158	}
wolfSSL	15:117db924cf7c	10159	if (carry) {
wolfSSL	15:117db924cf7c	10160	pt[AES_BLOCK_SIZE] ^= GF_XTS;
wolfSSL	15:117db924cf7c	10161	}
wolfSSL	15:117db924cf7c	10162
wolfSSL	15:117db924cf7c	10163	pt += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10164	}
wolfSSL	15:117db924cf7c	10165
wolfSSL	15:117db924cf7c	10166	xorbuf(out, in, totalSz);
wolfSSL	15:117db924cf7c	10167	if (dir == AES_ENCRYPTION) {
wolfSSL	15:117db924cf7c	10168	return wc_AesEcbEncrypt(aes, out, out, totalSz);
wolfSSL	15:117db924cf7c	10169	}
wolfSSL	15:117db924cf7c	10170	else {
wolfSSL	15:117db924cf7c	10171	return wc_AesEcbDecrypt(aes, out, out, totalSz);
wolfSSL	15:117db924cf7c	10172	}
wolfSSL	15:117db924cf7c	10173	}
wolfSSL	15:117db924cf7c	10174	#endif /* HAVE_AES_ECB */
wolfSSL	15:117db924cf7c	10175
wolfSSL	15:117db924cf7c	10176
wolfSSL	15:117db924cf7c	10177	/* AES with XTS mode. (XTS) XEX encryption with Tweak and cipher text Stealing.
wolfSSL	15:117db924cf7c	10178	*
wolfSSL	15:117db924cf7c	10179	* xaes AES keys to use for block encrypt/decrypt
wolfSSL	15:117db924cf7c	10180	* out output buffer to hold cipher text
wolfSSL	15:117db924cf7c	10181	* in input plain text buffer to encrypt
wolfSSL	15:117db924cf7c	10182	* sz size of both out and in buffers
wolfSSL	15:117db924cf7c	10183	* i value to use for tweak
wolfSSL	15:117db924cf7c	10184	* iSz size of i buffer, should always be AES_BLOCK_SIZE but having this input
wolfSSL	15:117db924cf7c	10185	* adds a sanity check on how the user calls the function.
wolfSSL	15:117db924cf7c	10186	*
wolfSSL	15:117db924cf7c	10187	* returns 0 on success
wolfSSL	15:117db924cf7c	10188	*/
wolfSSL	15:117db924cf7c	10189	int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	10190	const byte* i, word32 iSz)
wolfSSL	15:117db924cf7c	10191	{
wolfSSL	15:117db924cf7c	10192	int ret = 0;
wolfSSL	15:117db924cf7c	10193	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10194	Aes aes, tweak;
wolfSSL	15:117db924cf7c	10195
wolfSSL	15:117db924cf7c	10196	if (xaes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	10197	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	10198	}
wolfSSL	15:117db924cf7c	10199
wolfSSL	15:117db924cf7c	10200	aes = &xaes->aes;
wolfSSL	15:117db924cf7c	10201	tweak = &xaes->tweak;
wolfSSL	15:117db924cf7c	10202
wolfSSL	15:117db924cf7c	10203	if (iSz < AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	10204	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	10205	}
wolfSSL	15:117db924cf7c	10206
wolfSSL	15:117db924cf7c	10207	if (blocks > 0) {
wolfSSL	15:117db924cf7c	10208	byte tmp[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10209
wolfSSL	15:117db924cf7c	10210	XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES
wolfSSL	15:117db924cf7c	10211	* key setup passed to encrypt direct*/
wolfSSL	15:117db924cf7c	10212
wolfSSL	15:117db924cf7c	10213	wc_AesEncryptDirect(tweak, tmp, i);
wolfSSL	15:117db924cf7c	10214
wolfSSL	15:117db924cf7c	10215	#ifdef HAVE_AES_ECB
wolfSSL	15:117db924cf7c	10216	/* encrypt all of buffer at once when possible */
wolfSSL	15:117db924cf7c	10217	if (in != out) { /* can not handle inline */
wolfSSL	15:117db924cf7c	10218	XMEMCPY(out, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10219	if ((ret = _AesXtsHelper(aes, out, in, sz, AES_ENCRYPTION)) != 0) {
wolfSSL	15:117db924cf7c	10220	return ret;
wolfSSL	15:117db924cf7c	10221	}
wolfSSL	15:117db924cf7c	10222	}
wolfSSL	15:117db924cf7c	10223	#endif
wolfSSL	15:117db924cf7c	10224
wolfSSL	15:117db924cf7c	10225	while (blocks > 0) {
wolfSSL	15:117db924cf7c	10226	word32 j;
wolfSSL	15:117db924cf7c	10227	byte carry = 0;
wolfSSL	15:117db924cf7c	10228	byte buf[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10229
wolfSSL	15:117db924cf7c	10230	#ifdef HAVE_AES_ECB
wolfSSL	15:117db924cf7c	10231	if (in == out) { /* check for if inline */
wolfSSL	15:117db924cf7c	10232	#endif
wolfSSL	15:117db924cf7c	10233	XMEMCPY(buf, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10234	xorbuf(buf, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10235	wc_AesEncryptDirect(aes, out, buf);
wolfSSL	15:117db924cf7c	10236	#ifdef HAVE_AES_ECB
wolfSSL	15:117db924cf7c	10237	}
wolfSSL	15:117db924cf7c	10238	#endif
wolfSSL	15:117db924cf7c	10239	xorbuf(out, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10240
wolfSSL	15:117db924cf7c	10241	/* multiply by shift left and propogate carry */
wolfSSL	15:117db924cf7c	10242	for (j = 0; j < AES_BLOCK_SIZE; j++) {
wolfSSL	15:117db924cf7c	10243	byte tmpC;
wolfSSL	15:117db924cf7c	10244
wolfSSL	15:117db924cf7c	10245	tmpC = (tmp[j] >> 7) & 0x01;
wolfSSL	15:117db924cf7c	10246	tmp[j] = ((tmp[j] << 1) + carry) & 0xFF;
wolfSSL	15:117db924cf7c	10247	carry = tmpC;
wolfSSL	15:117db924cf7c	10248	}
wolfSSL	15:117db924cf7c	10249	if (carry) {
wolfSSL	15:117db924cf7c	10250	tmp[0] ^= GF_XTS;
wolfSSL	15:117db924cf7c	10251	}
wolfSSL	15:117db924cf7c	10252
wolfSSL	15:117db924cf7c	10253	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10254	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10255	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10256	blocks--;
wolfSSL	15:117db924cf7c	10257	}
wolfSSL	15:117db924cf7c	10258
wolfSSL	15:117db924cf7c	10259	/* stealing operation of XTS to handle left overs */
wolfSSL	15:117db924cf7c	10260	if (sz > 0) {
wolfSSL	15:117db924cf7c	10261	byte buf[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10262
wolfSSL	15:117db924cf7c	10263	XMEMCPY(buf, out - AES_BLOCK_SIZE, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10264	if (sz >= AES_BLOCK_SIZE) { /* extra sanity check before copy */
wolfSSL	15:117db924cf7c	10265	return BUFFER_E;
wolfSSL	15:117db924cf7c	10266	}
wolfSSL	15:117db924cf7c	10267	XMEMCPY(out, buf, sz);
wolfSSL	15:117db924cf7c	10268	XMEMCPY(buf, in, sz);
wolfSSL	15:117db924cf7c	10269
wolfSSL	15:117db924cf7c	10270	xorbuf(buf, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10271	wc_AesEncryptDirect(aes, out - AES_BLOCK_SIZE, buf);
wolfSSL	15:117db924cf7c	10272	xorbuf(out - AES_BLOCK_SIZE, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10273	}
wolfSSL	15:117db924cf7c	10274	}
wolfSSL	15:117db924cf7c	10275	else {
wolfSSL	15:117db924cf7c	10276	WOLFSSL_MSG("Plain text input too small for encryption");
wolfSSL	15:117db924cf7c	10277	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	10278	}
wolfSSL	15:117db924cf7c	10279
wolfSSL	15:117db924cf7c	10280	return ret;
wolfSSL	15:117db924cf7c	10281	}
wolfSSL	15:117db924cf7c	10282
wolfSSL	15:117db924cf7c	10283
wolfSSL	15:117db924cf7c	10284	/* Same process as encryption but Aes key is AES_DECRYPTION type.
wolfSSL	15:117db924cf7c	10285	*
wolfSSL	15:117db924cf7c	10286	* xaes AES keys to use for block encrypt/decrypt
wolfSSL	15:117db924cf7c	10287	* out output buffer to hold plain text
wolfSSL	15:117db924cf7c	10288	* in input cipher text buffer to decrypt
wolfSSL	15:117db924cf7c	10289	* sz size of both out and in buffers
wolfSSL	15:117db924cf7c	10290	* i value to use for tweak
wolfSSL	15:117db924cf7c	10291	* iSz size of i buffer, should always be AES_BLOCK_SIZE but having this input
wolfSSL	15:117db924cf7c	10292	* adds a sanity check on how the user calls the function.
wolfSSL	15:117db924cf7c	10293	*
wolfSSL	15:117db924cf7c	10294	* returns 0 on success
wolfSSL	15:117db924cf7c	10295	*/
wolfSSL	15:117db924cf7c	10296	int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz,
wolfSSL	15:117db924cf7c	10297	const byte* i, word32 iSz)
wolfSSL	15:117db924cf7c	10298	{
wolfSSL	15:117db924cf7c	10299	int ret = 0;
wolfSSL	15:117db924cf7c	10300	word32 blocks = (sz / AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10301	Aes aes, tweak;
wolfSSL	15:117db924cf7c	10302
wolfSSL	15:117db924cf7c	10303	if (xaes == NULL \|\| out == NULL \|\| in == NULL) {
wolfSSL	15:117db924cf7c	10304	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	10305	}
wolfSSL	15:117db924cf7c	10306
wolfSSL	15:117db924cf7c	10307	aes = &xaes->aes;
wolfSSL	15:117db924cf7c	10308	tweak = &xaes->tweak;
wolfSSL	15:117db924cf7c	10309
wolfSSL	15:117db924cf7c	10310	if (iSz < AES_BLOCK_SIZE) {
wolfSSL	15:117db924cf7c	10311	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	10312	}
wolfSSL	15:117db924cf7c	10313
wolfSSL	15:117db924cf7c	10314	if (blocks > 0) {
wolfSSL	15:117db924cf7c	10315	word32 j;
wolfSSL	15:117db924cf7c	10316	byte carry = 0;
wolfSSL	15:117db924cf7c	10317	byte tmp[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10318	byte stl = (sz % AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10319
wolfSSL	15:117db924cf7c	10320	XMEMSET(tmp, 0, AES_BLOCK_SIZE); /* set to 0's in case of improper AES
wolfSSL	15:117db924cf7c	10321	* key setup passed to decrypt direct*/
wolfSSL	15:117db924cf7c	10322
wolfSSL	15:117db924cf7c	10323	wc_AesEncryptDirect(tweak, tmp, i);
wolfSSL	15:117db924cf7c	10324
wolfSSL	15:117db924cf7c	10325	/* if Stealing then break out of loop one block early to handle special
wolfSSL	15:117db924cf7c	10326	* case */
wolfSSL	15:117db924cf7c	10327	if (stl > 0) {
wolfSSL	15:117db924cf7c	10328	blocks--;
wolfSSL	15:117db924cf7c	10329	}
wolfSSL	15:117db924cf7c	10330
wolfSSL	15:117db924cf7c	10331	#ifdef HAVE_AES_ECB
wolfSSL	15:117db924cf7c	10332	/* decrypt all of buffer at once when possible */
wolfSSL	15:117db924cf7c	10333	if (in != out) { /* can not handle inline */
wolfSSL	15:117db924cf7c	10334	XMEMCPY(out, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10335	if ((ret = _AesXtsHelper(aes, out, in, sz, AES_DECRYPTION)) != 0) {
wolfSSL	15:117db924cf7c	10336	return ret;
wolfSSL	15:117db924cf7c	10337	}
wolfSSL	15:117db924cf7c	10338	}
wolfSSL	15:117db924cf7c	10339	#endif
wolfSSL	15:117db924cf7c	10340
wolfSSL	15:117db924cf7c	10341	while (blocks > 0) {
wolfSSL	15:117db924cf7c	10342	byte buf[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10343
wolfSSL	15:117db924cf7c	10344	#ifdef HAVE_AES_ECB
wolfSSL	15:117db924cf7c	10345	if (in == out) { /* check for if inline */
wolfSSL	15:117db924cf7c	10346	#endif
wolfSSL	15:117db924cf7c	10347	XMEMCPY(buf, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10348	xorbuf(buf, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10349	wc_AesDecryptDirect(aes, out, buf);
wolfSSL	15:117db924cf7c	10350	#ifdef HAVE_AES_ECB
wolfSSL	15:117db924cf7c	10351	}
wolfSSL	15:117db924cf7c	10352	#endif
wolfSSL	15:117db924cf7c	10353	xorbuf(out, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10354
wolfSSL	15:117db924cf7c	10355	/* multiply by shift left and propogate carry */
wolfSSL	15:117db924cf7c	10356	for (j = 0; j < AES_BLOCK_SIZE; j++) {
wolfSSL	15:117db924cf7c	10357	byte tmpC;
wolfSSL	15:117db924cf7c	10358
wolfSSL	15:117db924cf7c	10359	tmpC = (tmp[j] >> 7) & 0x01;
wolfSSL	15:117db924cf7c	10360	tmp[j] = ((tmp[j] << 1) + carry) & 0xFF;
wolfSSL	15:117db924cf7c	10361	carry = tmpC;
wolfSSL	15:117db924cf7c	10362	}
wolfSSL	15:117db924cf7c	10363	if (carry) {
wolfSSL	15:117db924cf7c	10364	tmp[0] ^= GF_XTS;
wolfSSL	15:117db924cf7c	10365	}
wolfSSL	15:117db924cf7c	10366	carry = 0;
wolfSSL	15:117db924cf7c	10367
wolfSSL	15:117db924cf7c	10368	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10369	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10370	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10371	blocks--;
wolfSSL	15:117db924cf7c	10372	}
wolfSSL	15:117db924cf7c	10373
wolfSSL	15:117db924cf7c	10374	/* stealing operation of XTS to handle left overs */
wolfSSL	15:117db924cf7c	10375	if (sz > 0) {
wolfSSL	15:117db924cf7c	10376	byte buf[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10377	byte tmp2[AES_BLOCK_SIZE];
wolfSSL	15:117db924cf7c	10378
wolfSSL	15:117db924cf7c	10379	/* multiply by shift left and propogate carry */
wolfSSL	15:117db924cf7c	10380	for (j = 0; j < AES_BLOCK_SIZE; j++) {
wolfSSL	15:117db924cf7c	10381	byte tmpC;
wolfSSL	15:117db924cf7c	10382
wolfSSL	15:117db924cf7c	10383	tmpC = (tmp[j] >> 7) & 0x01;
wolfSSL	15:117db924cf7c	10384	tmp2[j] = ((tmp[j] << 1) + carry) & 0xFF;
wolfSSL	15:117db924cf7c	10385	carry = tmpC;
wolfSSL	15:117db924cf7c	10386	}
wolfSSL	15:117db924cf7c	10387	if (carry) {
wolfSSL	15:117db924cf7c	10388	tmp2[0] ^= GF_XTS;
wolfSSL	15:117db924cf7c	10389	}
wolfSSL	15:117db924cf7c	10390
wolfSSL	15:117db924cf7c	10391	XMEMCPY(buf, in, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10392	xorbuf(buf, tmp2, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10393	wc_AesDecryptDirect(aes, out, buf);
wolfSSL	15:117db924cf7c	10394	xorbuf(out, tmp2, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10395
wolfSSL	15:117db924cf7c	10396	/* tmp2 holds partial \| last */
wolfSSL	15:117db924cf7c	10397	XMEMCPY(tmp2, out, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10398	in += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10399	out += AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10400	sz -= AES_BLOCK_SIZE;
wolfSSL	15:117db924cf7c	10401
wolfSSL	15:117db924cf7c	10402	/* Make buffer with end of cipher text \| last */
wolfSSL	15:117db924cf7c	10403	XMEMCPY(buf, tmp2, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10404	if (sz >= AES_BLOCK_SIZE) { /* extra sanity check before copy */
wolfSSL	15:117db924cf7c	10405	return BUFFER_E;
wolfSSL	15:117db924cf7c	10406	}
wolfSSL	15:117db924cf7c	10407	XMEMCPY(buf, in, sz);
wolfSSL	15:117db924cf7c	10408	XMEMCPY(out, tmp2, sz);
wolfSSL	15:117db924cf7c	10409
wolfSSL	15:117db924cf7c	10410	xorbuf(buf, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10411	wc_AesDecryptDirect(aes, tmp2, buf);
wolfSSL	15:117db924cf7c	10412	xorbuf(tmp2, tmp, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10413	XMEMCPY(out - AES_BLOCK_SIZE, tmp2, AES_BLOCK_SIZE);
wolfSSL	15:117db924cf7c	10414	}
wolfSSL	15:117db924cf7c	10415	}
wolfSSL	15:117db924cf7c	10416	else {
wolfSSL	15:117db924cf7c	10417	WOLFSSL_MSG("Plain text input too small for encryption");
wolfSSL	15:117db924cf7c	10418	return BAD_FUNC_ARG;
wolfSSL	15:117db924cf7c	10419	}
wolfSSL	15:117db924cf7c	10420
wolfSSL	15:117db924cf7c	10421	return ret;
wolfSSL	15:117db924cf7c	10422	}
wolfSSL	15:117db924cf7c	10423
wolfSSL	15:117db924cf7c	10424	#endif /* WOLFSSL_AES_XTS */
wolfSSL	15:117db924cf7c	10425
wolfSSL	15:117db924cf7c	10426	#endif /* HAVE_FIPS */
wolfSSL	15:117db924cf7c	10427	#endif /* !NO_AES */
wolfSSL	15:117db924cf7c	10428

Repository toolbox

Repository details

Type:	Library
Created:	19 Nov 2019
Imports:	7
Forks:	0
Commits:	18
Dependents:	1
Dependencies:	0
Followers:	1

Important changes to repositories hosted on mbed.com

wolfcrypt/src/aes.c@15:117db924cf7c, 2018-08-18 (annotated)

Who changed what in which revision?

Repository toolbox

Repository details

Important Information for this Arm website

Important changes to repositories hosted on mbed.com

wolfcrypt/src/aes.c@15:117db924cf7c, 2018-08-18 (annotated)

Who changed what in which revision?

Repository toolbox

Repository details

Important Information for this Arm website

Access Warning