wolfSSL - wolfSSL SSL/TLS library, support up to TLS1.3

Users » wolfSSL » Code » wolfSSL

wolfSSL SSL/TLS library, support up to TLS1.3

Dependents: CyaSSL-Twitter-OAuth4Tw Example-client-tls-cert TwitterReader TweetTest ... more

wolfcrypt/src/sha512.c@4:1b0d80432c79, 2016-04-28 (annotated)

Committer:: wolfSSL
Date:: Thu Apr 28 00:57:21 2016 +0000
Revision:: 4:1b0d80432c79

wolfSSL 3.9.0

Who changed what in which revision?

User	Revision	Line number	New contents of line
wolfSSL	4:1b0d80432c79	1	/* sha512.c
wolfSSL	4:1b0d80432c79	2	*
wolfSSL	4:1b0d80432c79	3	* Copyright (C) 2006-2016 wolfSSL Inc.
wolfSSL	4:1b0d80432c79	4	*
wolfSSL	4:1b0d80432c79	5	* This file is part of wolfSSL.
wolfSSL	4:1b0d80432c79	6	*
wolfSSL	4:1b0d80432c79	7	* wolfSSL is free software; you can redistribute it and/or modify
wolfSSL	4:1b0d80432c79	8	* it under the terms of the GNU General Public License as published by
wolfSSL	4:1b0d80432c79	9	* the Free Software Foundation; either version 2 of the License, or
wolfSSL	4:1b0d80432c79	10	* (at your option) any later version.
wolfSSL	4:1b0d80432c79	11	*
wolfSSL	4:1b0d80432c79	12	* wolfSSL is distributed in the hope that it will be useful,
wolfSSL	4:1b0d80432c79	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
wolfSSL	4:1b0d80432c79	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
wolfSSL	4:1b0d80432c79	15	* GNU General Public License for more details.
wolfSSL	4:1b0d80432c79	16	*
wolfSSL	4:1b0d80432c79	17	* You should have received a copy of the GNU General Public License
wolfSSL	4:1b0d80432c79	18	* along with this program; if not, write to the Free Software
wolfSSL	4:1b0d80432c79	19	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA
wolfSSL	4:1b0d80432c79	20	*/
wolfSSL	4:1b0d80432c79	21
wolfSSL	4:1b0d80432c79	22
wolfSSL	4:1b0d80432c79	23	#ifdef HAVE_CONFIG_H
wolfSSL	4:1b0d80432c79	24	#include <config.h>
wolfSSL	4:1b0d80432c79	25	#endif
wolfSSL	4:1b0d80432c79	26
wolfSSL	4:1b0d80432c79	27	#include <wolfssl/wolfcrypt/settings.h>
wolfSSL	4:1b0d80432c79	28	#include <wolfssl/wolfcrypt/sha512.h>
wolfSSL	4:1b0d80432c79	29
wolfSSL	4:1b0d80432c79	30	#ifdef WOLFSSL_SHA512
wolfSSL	4:1b0d80432c79	31
wolfSSL	4:1b0d80432c79	32	#ifdef HAVE_FIPS
wolfSSL	4:1b0d80432c79	33	int wc_InitSha512(Sha512* sha)
wolfSSL	4:1b0d80432c79	34	{
wolfSSL	4:1b0d80432c79	35	return InitSha512_fips(sha);
wolfSSL	4:1b0d80432c79	36	}
wolfSSL	4:1b0d80432c79	37
wolfSSL	4:1b0d80432c79	38
wolfSSL	4:1b0d80432c79	39	int wc_Sha512Update(Sha512* sha, const byte* data, word32 len)
wolfSSL	4:1b0d80432c79	40	{
wolfSSL	4:1b0d80432c79	41	return Sha512Update_fips(sha, data, len);
wolfSSL	4:1b0d80432c79	42	}
wolfSSL	4:1b0d80432c79	43
wolfSSL	4:1b0d80432c79	44
wolfSSL	4:1b0d80432c79	45	int wc_Sha512Final(Sha512* sha, byte* out)
wolfSSL	4:1b0d80432c79	46	{
wolfSSL	4:1b0d80432c79	47	return Sha512Final_fips(sha, out);
wolfSSL	4:1b0d80432c79	48	}
wolfSSL	4:1b0d80432c79	49
wolfSSL	4:1b0d80432c79	50
wolfSSL	4:1b0d80432c79	51	#if defined(WOLFSSL_SHA384) \|\| defined(HAVE_AESGCM)
wolfSSL	4:1b0d80432c79	52
wolfSSL	4:1b0d80432c79	53	int wc_InitSha384(Sha384* sha)
wolfSSL	4:1b0d80432c79	54	{
wolfSSL	4:1b0d80432c79	55	return InitSha384_fips(sha);
wolfSSL	4:1b0d80432c79	56	}
wolfSSL	4:1b0d80432c79	57
wolfSSL	4:1b0d80432c79	58
wolfSSL	4:1b0d80432c79	59	int wc_Sha384Update(Sha384* sha, const byte* data, word32 len)
wolfSSL	4:1b0d80432c79	60	{
wolfSSL	4:1b0d80432c79	61	return Sha384Update_fips(sha, data, len);
wolfSSL	4:1b0d80432c79	62	}
wolfSSL	4:1b0d80432c79	63
wolfSSL	4:1b0d80432c79	64
wolfSSL	4:1b0d80432c79	65	int wc_Sha384Final(Sha384* sha, byte* out)
wolfSSL	4:1b0d80432c79	66	{
wolfSSL	4:1b0d80432c79	67	return Sha384Final_fips(sha, out);
wolfSSL	4:1b0d80432c79	68	}
wolfSSL	4:1b0d80432c79	69
wolfSSL	4:1b0d80432c79	70
wolfSSL	4:1b0d80432c79	71	#endif /* WOLFSSL_SHA384 */
wolfSSL	4:1b0d80432c79	72	#else /* else build without using fips */
wolfSSL	4:1b0d80432c79	73	#include <wolfssl/wolfcrypt/logging.h>
wolfSSL	4:1b0d80432c79	74	#include <wolfssl/wolfcrypt/error-crypt.h>
wolfSSL	4:1b0d80432c79	75
wolfSSL	4:1b0d80432c79	76	#ifdef NO_INLINE
wolfSSL	4:1b0d80432c79	77	#include <wolfssl/wolfcrypt/misc.h>
wolfSSL	4:1b0d80432c79	78	#else
wolfSSL	4:1b0d80432c79	79	#include <wolfcrypt/src/misc.c>
wolfSSL	4:1b0d80432c79	80	#endif
wolfSSL	4:1b0d80432c79	81
wolfSSL	4:1b0d80432c79	82
wolfSSL	4:1b0d80432c79	83	#ifndef WOLFSSL_HAVE_MIN
wolfSSL	4:1b0d80432c79	84	#define WOLFSSL_HAVE_MIN
wolfSSL	4:1b0d80432c79	85
wolfSSL	4:1b0d80432c79	86	static INLINE word32 min(word32 a, word32 b)
wolfSSL	4:1b0d80432c79	87	{
wolfSSL	4:1b0d80432c79	88	return a > b ? b : a;
wolfSSL	4:1b0d80432c79	89	}
wolfSSL	4:1b0d80432c79	90
wolfSSL	4:1b0d80432c79	91	#endif /* WOLFSSL_HAVE_MIN */
wolfSSL	4:1b0d80432c79	92
wolfSSL	4:1b0d80432c79	93	#if defined(USE_INTEL_SPEEDUP)
wolfSSL	4:1b0d80432c79	94	#define HAVE_INTEL_AVX1
wolfSSL	4:1b0d80432c79	95	#define HAVE_INTEL_AVX2
wolfSSL	4:1b0d80432c79	96	#endif
wolfSSL	4:1b0d80432c79	97
wolfSSL	4:1b0d80432c79	98	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	99	/* #define DEBUG_XMM */
wolfSSL	4:1b0d80432c79	100	#endif
wolfSSL	4:1b0d80432c79	101
wolfSSL	4:1b0d80432c79	102	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	103	#define HAVE_INTEL_RORX
wolfSSL	4:1b0d80432c79	104	/* #define DEBUG_YMM */
wolfSSL	4:1b0d80432c79	105	#endif
wolfSSL	4:1b0d80432c79	106
wolfSSL	4:1b0d80432c79	107	/*****
wolfSSL	4:1b0d80432c79	108	Intel AVX1/AVX2 Macro Control Structure
wolfSSL	4:1b0d80432c79	109
wolfSSL	4:1b0d80432c79	110	#if defined(HAVE_INteL_SPEEDUP)
wolfSSL	4:1b0d80432c79	111	#define HAVE_INTEL_AVX1
wolfSSL	4:1b0d80432c79	112	#define HAVE_INTEL_AVX2
wolfSSL	4:1b0d80432c79	113	#endif
wolfSSL	4:1b0d80432c79	114
wolfSSL	4:1b0d80432c79	115	int InitSha512(Sha512* sha512) {
wolfSSL	4:1b0d80432c79	116	Save/Recover XMM, YMM
wolfSSL	4:1b0d80432c79	117	...
wolfSSL	4:1b0d80432c79	118
wolfSSL	4:1b0d80432c79	119	Check Intel AVX cpuid flags
wolfSSL	4:1b0d80432c79	120	}
wolfSSL	4:1b0d80432c79	121
wolfSSL	4:1b0d80432c79	122	#if defined(HAVE_INTEL_AVX1)\|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	123	Transform_AVX1() ; # Function prototype
wolfSSL	4:1b0d80432c79	124	Transform_AVX2() ; #
wolfSSL	4:1b0d80432c79	125	#endif
wolfSSL	4:1b0d80432c79	126
wolfSSL	4:1b0d80432c79	127	_Transform() { # Native Transform Function body
wolfSSL	4:1b0d80432c79	128
wolfSSL	4:1b0d80432c79	129	}
wolfSSL	4:1b0d80432c79	130
wolfSSL	4:1b0d80432c79	131	int Sha512Update() {
wolfSSL	4:1b0d80432c79	132	Save/Recover XMM, YMM
wolfSSL	4:1b0d80432c79	133	...
wolfSSL	4:1b0d80432c79	134	}
wolfSSL	4:1b0d80432c79	135
wolfSSL	4:1b0d80432c79	136	int Sha512Final() {
wolfSSL	4:1b0d80432c79	137	Save/Recover XMM, YMM
wolfSSL	4:1b0d80432c79	138	...
wolfSSL	4:1b0d80432c79	139	}
wolfSSL	4:1b0d80432c79	140
wolfSSL	4:1b0d80432c79	141
wolfSSL	4:1b0d80432c79	142	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	143
wolfSSL	4:1b0d80432c79	144	XMM Instructions/INLINE asm Definitions
wolfSSL	4:1b0d80432c79	145
wolfSSL	4:1b0d80432c79	146	#endif
wolfSSL	4:1b0d80432c79	147
wolfSSL	4:1b0d80432c79	148	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	149
wolfSSL	4:1b0d80432c79	150	YMM Instructions/INLINE asm Definitions
wolfSSL	4:1b0d80432c79	151
wolfSSL	4:1b0d80432c79	152	#endif
wolfSSL	4:1b0d80432c79	153
wolfSSL	4:1b0d80432c79	154	#if defnied(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	155
wolfSSL	4:1b0d80432c79	156	int Transform_AVX1() {
wolfSSL	4:1b0d80432c79	157	Stitched Message Sched/Round
wolfSSL	4:1b0d80432c79	158	}
wolfSSL	4:1b0d80432c79	159
wolfSSL	4:1b0d80432c79	160	#endif
wolfSSL	4:1b0d80432c79	161
wolfSSL	4:1b0d80432c79	162	#if defnied(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	163
wolfSSL	4:1b0d80432c79	164	int Transform_AVX2() {
wolfSSL	4:1b0d80432c79	165	Stitched Message Sched/Round
wolfSSL	4:1b0d80432c79	166	}
wolfSSL	4:1b0d80432c79	167	#endif
wolfSSL	4:1b0d80432c79	168
wolfSSL	4:1b0d80432c79	169
wolfSSL	4:1b0d80432c79	170	*/
wolfSSL	4:1b0d80432c79	171
wolfSSL	4:1b0d80432c79	172	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	173
wolfSSL	4:1b0d80432c79	174
wolfSSL	4:1b0d80432c79	175	/* Each platform needs to query info type 1 from cpuid to see if aesni is
wolfSSL	4:1b0d80432c79	176	* supported. Also, let's setup a macro for proper linkage w/o ABI conflicts
wolfSSL	4:1b0d80432c79	177	*/
wolfSSL	4:1b0d80432c79	178
wolfSSL	4:1b0d80432c79	179	#ifndef _MSC_VER
wolfSSL	4:1b0d80432c79	180	#define cpuid(reg, leaf, sub)\
wolfSSL	4:1b0d80432c79	181	__asm__ __volatile__ ("cpuid":\
wolfSSL	4:1b0d80432c79	182	"=a" (reg[0]), "=b" (reg[1]), "=c" (reg[2]), "=d" (reg[3]) :\
wolfSSL	4:1b0d80432c79	183	"a" (leaf), "c"(sub));
wolfSSL	4:1b0d80432c79	184
wolfSSL	4:1b0d80432c79	185	#define XASM_LINK(f) asm(f)
wolfSSL	4:1b0d80432c79	186	#else
wolfSSL	4:1b0d80432c79	187
wolfSSL	4:1b0d80432c79	188	#include <intrin.h>
wolfSSL	4:1b0d80432c79	189	#define cpuid(a,b) __cpuid((int*)a,b)
wolfSSL	4:1b0d80432c79	190
wolfSSL	4:1b0d80432c79	191	#define XASM_LINK(f)
wolfSSL	4:1b0d80432c79	192
wolfSSL	4:1b0d80432c79	193	#endif /* _MSC_VER */
wolfSSL	4:1b0d80432c79	194
wolfSSL	4:1b0d80432c79	195	#define EAX 0
wolfSSL	4:1b0d80432c79	196	#define EBX 1
wolfSSL	4:1b0d80432c79	197	#define ECX 2
wolfSSL	4:1b0d80432c79	198	#define EDX 3
wolfSSL	4:1b0d80432c79	199
wolfSSL	4:1b0d80432c79	200	#define CPUID_AVX1 0x1
wolfSSL	4:1b0d80432c79	201	#define CPUID_AVX2 0x2
wolfSSL	4:1b0d80432c79	202	#define CPUID_RDRAND 0x4
wolfSSL	4:1b0d80432c79	203	#define CPUID_RDSEED 0x8
wolfSSL	4:1b0d80432c79	204	#define CPUID_BMI2 0x10 /* MULX, RORX */
wolfSSL	4:1b0d80432c79	205
wolfSSL	4:1b0d80432c79	206	#define IS_INTEL_AVX1 (cpuid_flags&CPUID_AVX1)
wolfSSL	4:1b0d80432c79	207	#define IS_INTEL_AVX2 (cpuid_flags&CPUID_AVX2)
wolfSSL	4:1b0d80432c79	208	#define IS_INTEL_BMI2 (cpuid_flags&CPUID_BMI2)
wolfSSL	4:1b0d80432c79	209	#define IS_INTEL_RDRAND (cpuid_flags&CPUID_RDRAND)
wolfSSL	4:1b0d80432c79	210	#define IS_INTEL_RDSEED (cpuid_flags&CPUID_RDSEED)
wolfSSL	4:1b0d80432c79	211
wolfSSL	4:1b0d80432c79	212	static word32 cpuid_check = 0 ;
wolfSSL	4:1b0d80432c79	213	static word32 cpuid_flags = 0 ;
wolfSSL	4:1b0d80432c79	214
wolfSSL	4:1b0d80432c79	215	static word32 cpuid_flag(word32 leaf, word32 sub, word32 num, word32 bit) {
wolfSSL	4:1b0d80432c79	216	int got_intel_cpu=0;
wolfSSL	4:1b0d80432c79	217	unsigned int reg[5];
wolfSSL	4:1b0d80432c79	218
wolfSSL	4:1b0d80432c79	219	reg[4] = '\0' ;
wolfSSL	4:1b0d80432c79	220	cpuid(reg, 0, 0);
wolfSSL	4:1b0d80432c79	221	if(memcmp((char *)&(reg[EBX]), "Genu", 4) == 0 &&
wolfSSL	4:1b0d80432c79	222	memcmp((char *)&(reg[EDX]), "ineI", 4) == 0 &&
wolfSSL	4:1b0d80432c79	223	memcmp((char *)&(reg[ECX]), "ntel", 4) == 0) {
wolfSSL	4:1b0d80432c79	224	got_intel_cpu = 1;
wolfSSL	4:1b0d80432c79	225	}
wolfSSL	4:1b0d80432c79	226	if (got_intel_cpu) {
wolfSSL	4:1b0d80432c79	227	cpuid(reg, leaf, sub);
wolfSSL	4:1b0d80432c79	228	return((reg[num]>>bit)&0x1) ;
wolfSSL	4:1b0d80432c79	229	}
wolfSSL	4:1b0d80432c79	230	return 0 ;
wolfSSL	4:1b0d80432c79	231	}
wolfSSL	4:1b0d80432c79	232
wolfSSL	4:1b0d80432c79	233	#define CHECK_SHA512 0x1
wolfSSL	4:1b0d80432c79	234	#define CHECK_SHA384 0x2
wolfSSL	4:1b0d80432c79	235
wolfSSL	4:1b0d80432c79	236	static int set_cpuid_flags(int sha) {
wolfSSL	4:1b0d80432c79	237	if((cpuid_check & sha) ==0) {
wolfSSL	4:1b0d80432c79	238	if(cpuid_flag(1, 0, ECX, 28)){ cpuid_flags \|= CPUID_AVX1 ;}
wolfSSL	4:1b0d80432c79	239	if(cpuid_flag(7, 0, EBX, 5)){ cpuid_flags \|= CPUID_AVX2 ; }
wolfSSL	4:1b0d80432c79	240	if(cpuid_flag(7, 0, EBX, 8)) { cpuid_flags \|= CPUID_BMI2 ; }
wolfSSL	4:1b0d80432c79	241	if(cpuid_flag(1, 0, ECX, 30)){ cpuid_flags \|= CPUID_RDRAND ; }
wolfSSL	4:1b0d80432c79	242	if(cpuid_flag(7, 0, EBX, 18)){ cpuid_flags \|= CPUID_RDSEED ; }
wolfSSL	4:1b0d80432c79	243	cpuid_check \|= sha ;
wolfSSL	4:1b0d80432c79	244	return 0 ;
wolfSSL	4:1b0d80432c79	245	}
wolfSSL	4:1b0d80432c79	246	return 1 ;
wolfSSL	4:1b0d80432c79	247	}
wolfSSL	4:1b0d80432c79	248
wolfSSL	4:1b0d80432c79	249
wolfSSL	4:1b0d80432c79	250	/* #if defined(HAVE_INTEL_AVX1/2) at the tail of sha512 */
wolfSSL	4:1b0d80432c79	251
wolfSSL	4:1b0d80432c79	252	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	253	static int Transform_AVX1(Sha512 *sha512) ;
wolfSSL	4:1b0d80432c79	254	#endif
wolfSSL	4:1b0d80432c79	255
wolfSSL	4:1b0d80432c79	256	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	257	static int Transform_AVX2(Sha512 *sha512) ;
wolfSSL	4:1b0d80432c79	258
wolfSSL	4:1b0d80432c79	259	#if defined(HAVE_INTEL_AVX1) && defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_RORX)
wolfSSL	4:1b0d80432c79	260	static int Transform_AVX1_RORX(Sha512 *sha512) ;
wolfSSL	4:1b0d80432c79	261	#endif
wolfSSL	4:1b0d80432c79	262
wolfSSL	4:1b0d80432c79	263	#endif
wolfSSL	4:1b0d80432c79	264
wolfSSL	4:1b0d80432c79	265	static int _Transform(Sha512 *sha512) ;
wolfSSL	4:1b0d80432c79	266
wolfSSL	4:1b0d80432c79	267	static int (Transform_p)(Sha512 sha512) = _Transform ;
wolfSSL	4:1b0d80432c79	268
wolfSSL	4:1b0d80432c79	269	#define Transform(sha512) (*Transform_p)(sha512)
wolfSSL	4:1b0d80432c79	270
wolfSSL	4:1b0d80432c79	271	static void set_Transform(void) {
wolfSSL	4:1b0d80432c79	272	if(set_cpuid_flags(CHECK_SHA512)) return ;
wolfSSL	4:1b0d80432c79	273
wolfSSL	4:1b0d80432c79	274	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	275	if(IS_INTEL_AVX2 && IS_INTEL_BMI2){
wolfSSL	4:1b0d80432c79	276	Transform_p = Transform_AVX1_RORX; return ;
wolfSSL	4:1b0d80432c79	277	Transform_p = Transform_AVX2 ;
wolfSSL	4:1b0d80432c79	278	/* for avoiding warning,"not used" */
wolfSSL	4:1b0d80432c79	279	}
wolfSSL	4:1b0d80432c79	280	#endif
wolfSSL	4:1b0d80432c79	281	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	282	Transform_p = ((IS_INTEL_AVX1) ? Transform_AVX1 : _Transform) ; return ;
wolfSSL	4:1b0d80432c79	283	#endif
wolfSSL	4:1b0d80432c79	284	Transform_p = _Transform ; return ;
wolfSSL	4:1b0d80432c79	285	}
wolfSSL	4:1b0d80432c79	286
wolfSSL	4:1b0d80432c79	287	#else
wolfSSL	4:1b0d80432c79	288	#define Transform(sha512) _Transform(sha512)
wolfSSL	4:1b0d80432c79	289	#endif
wolfSSL	4:1b0d80432c79	290
wolfSSL	4:1b0d80432c79	291	/* Dummy for saving MM_REGs on behalf of Transform */
wolfSSL	4:1b0d80432c79	292	/* #if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	293	#define SAVE_XMM_YMM __asm__ volatile("orq %%r8, %%r8":::\
wolfSSL	4:1b0d80432c79	294	"%ymm0","%ymm1","%ymm2","%ymm3","%ymm4","%ymm5","%ymm6","%ymm7","%ymm8","%ymm9","%ymm10","%ymm11",\
wolfSSL	4:1b0d80432c79	295	"%ymm12","%ymm13","%ymm14","%ymm15")
wolfSSL	4:1b0d80432c79	296	*/
wolfSSL	4:1b0d80432c79	297	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	298	#define SAVE_XMM_YMM __asm__ volatile("orq %%r8, %%r8":::\
wolfSSL	4:1b0d80432c79	299	"xmm0","xmm1","xmm2","xmm3","xmm4","xmm5","xmm6","xmm7","xmm8","xmm9","xmm10","xmm11","xmm12","xmm13","xmm14","xmm15")
wolfSSL	4:1b0d80432c79	300	#else
wolfSSL	4:1b0d80432c79	301	#define SAVE_XMM_YMM
wolfSSL	4:1b0d80432c79	302	#endif
wolfSSL	4:1b0d80432c79	303
wolfSSL	4:1b0d80432c79	304	#if defined(HAVE_INTEL_AVX1)\|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	305
wolfSSL	4:1b0d80432c79	306	#include <string.h>
wolfSSL	4:1b0d80432c79	307
wolfSSL	4:1b0d80432c79	308	#endif /* defined(HAVE_INTEL_AVX1)\|\| defined(HAVE_INTEL_AVX2) */
wolfSSL	4:1b0d80432c79	309
wolfSSL	4:1b0d80432c79	310
wolfSSL	4:1b0d80432c79	311	#if defined(HAVE_INTEL_RORX)
wolfSSL	4:1b0d80432c79	312	#define ROTR(func, bits, x) \
wolfSSL	4:1b0d80432c79	313	word64 func(word64 x) { word64 ret ;\
wolfSSL	4:1b0d80432c79	314	__asm__ ("rorx $"#bits", %1, %0\n\t":"=r"(ret):"r"(x):) ;\
wolfSSL	4:1b0d80432c79	315	return ret ;\
wolfSSL	4:1b0d80432c79	316	}
wolfSSL	4:1b0d80432c79	317
wolfSSL	4:1b0d80432c79	318	static INLINE ROTR(rotrFixed64_28, 28, x)
wolfSSL	4:1b0d80432c79	319	static INLINE ROTR(rotrFixed64_34, 34, x)
wolfSSL	4:1b0d80432c79	320	static INLINE ROTR(rotrFixed64_39, 39, x)
wolfSSL	4:1b0d80432c79	321	static INLINE ROTR(rotrFixed64_14, 14, x)
wolfSSL	4:1b0d80432c79	322	static INLINE ROTR(rotrFixed64_18, 18, x)
wolfSSL	4:1b0d80432c79	323	static INLINE ROTR(rotrFixed64_41, 41, x)
wolfSSL	4:1b0d80432c79	324
wolfSSL	4:1b0d80432c79	325	#define S0_RORX(x) (rotrFixed64_28(x)^rotrFixed64_34(x)^rotrFixed64_39(x))
wolfSSL	4:1b0d80432c79	326	#define S1_RORX(x) (rotrFixed64_14(x)^rotrFixed64_18(x)^rotrFixed64_41(x))
wolfSSL	4:1b0d80432c79	327	#endif
wolfSSL	4:1b0d80432c79	328
wolfSSL	4:1b0d80432c79	329	#if defined(HAVE_BYTEREVERSE64) && !defined(HAVE_INTEL_AVX1) && !defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	330	#define ByteReverseWords64(out, in, size) ByteReverseWords64_1(out, size)
wolfSSL	4:1b0d80432c79	331	#define ByteReverseWords64_1(buf, size)\
wolfSSL	4:1b0d80432c79	332	{ unsigned int i ;\
wolfSSL	4:1b0d80432c79	333	for(i=0; i< size/sizeof(word64); i++){\
wolfSSL	4:1b0d80432c79	334	__asm__ volatile("bswapq %0":"+r"(buf[i])::) ;\
wolfSSL	4:1b0d80432c79	335	}\
wolfSSL	4:1b0d80432c79	336	}
wolfSSL	4:1b0d80432c79	337	#endif
wolfSSL	4:1b0d80432c79	338
wolfSSL	4:1b0d80432c79	339
wolfSSL	4:1b0d80432c79	340	int wc_InitSha512(Sha512* sha512)
wolfSSL	4:1b0d80432c79	341	{
wolfSSL	4:1b0d80432c79	342	sha512->digest[0] = W64LIT(0x6a09e667f3bcc908);
wolfSSL	4:1b0d80432c79	343	sha512->digest[1] = W64LIT(0xbb67ae8584caa73b);
wolfSSL	4:1b0d80432c79	344	sha512->digest[2] = W64LIT(0x3c6ef372fe94f82b);
wolfSSL	4:1b0d80432c79	345	sha512->digest[3] = W64LIT(0xa54ff53a5f1d36f1);
wolfSSL	4:1b0d80432c79	346	sha512->digest[4] = W64LIT(0x510e527fade682d1);
wolfSSL	4:1b0d80432c79	347	sha512->digest[5] = W64LIT(0x9b05688c2b3e6c1f);
wolfSSL	4:1b0d80432c79	348	sha512->digest[6] = W64LIT(0x1f83d9abfb41bd6b);
wolfSSL	4:1b0d80432c79	349	sha512->digest[7] = W64LIT(0x5be0cd19137e2179);
wolfSSL	4:1b0d80432c79	350
wolfSSL	4:1b0d80432c79	351	sha512->buffLen = 0;
wolfSSL	4:1b0d80432c79	352	sha512->loLen = 0;
wolfSSL	4:1b0d80432c79	353	sha512->hiLen = 0;
wolfSSL	4:1b0d80432c79	354
wolfSSL	4:1b0d80432c79	355	#if defined(HAVE_INTEL_AVX1)\|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	356	set_Transform() ; /* choose best Transform function under this runtime environment */
wolfSSL	4:1b0d80432c79	357	#endif
wolfSSL	4:1b0d80432c79	358
wolfSSL	4:1b0d80432c79	359	return 0 ;
wolfSSL	4:1b0d80432c79	360	}
wolfSSL	4:1b0d80432c79	361
wolfSSL	4:1b0d80432c79	362
wolfSSL	4:1b0d80432c79	363	static const word64 K512[80] = {
wolfSSL	4:1b0d80432c79	364	W64LIT(0x428a2f98d728ae22), W64LIT(0x7137449123ef65cd),
wolfSSL	4:1b0d80432c79	365	W64LIT(0xb5c0fbcfec4d3b2f), W64LIT(0xe9b5dba58189dbbc),
wolfSSL	4:1b0d80432c79	366	W64LIT(0x3956c25bf348b538), W64LIT(0x59f111f1b605d019),
wolfSSL	4:1b0d80432c79	367	W64LIT(0x923f82a4af194f9b), W64LIT(0xab1c5ed5da6d8118),
wolfSSL	4:1b0d80432c79	368	W64LIT(0xd807aa98a3030242), W64LIT(0x12835b0145706fbe),
wolfSSL	4:1b0d80432c79	369	W64LIT(0x243185be4ee4b28c), W64LIT(0x550c7dc3d5ffb4e2),
wolfSSL	4:1b0d80432c79	370	W64LIT(0x72be5d74f27b896f), W64LIT(0x80deb1fe3b1696b1),
wolfSSL	4:1b0d80432c79	371	W64LIT(0x9bdc06a725c71235), W64LIT(0xc19bf174cf692694),
wolfSSL	4:1b0d80432c79	372	W64LIT(0xe49b69c19ef14ad2), W64LIT(0xefbe4786384f25e3),
wolfSSL	4:1b0d80432c79	373	W64LIT(0x0fc19dc68b8cd5b5), W64LIT(0x240ca1cc77ac9c65),
wolfSSL	4:1b0d80432c79	374	W64LIT(0x2de92c6f592b0275), W64LIT(0x4a7484aa6ea6e483),
wolfSSL	4:1b0d80432c79	375	W64LIT(0x5cb0a9dcbd41fbd4), W64LIT(0x76f988da831153b5),
wolfSSL	4:1b0d80432c79	376	W64LIT(0x983e5152ee66dfab), W64LIT(0xa831c66d2db43210),
wolfSSL	4:1b0d80432c79	377	W64LIT(0xb00327c898fb213f), W64LIT(0xbf597fc7beef0ee4),
wolfSSL	4:1b0d80432c79	378	W64LIT(0xc6e00bf33da88fc2), W64LIT(0xd5a79147930aa725),
wolfSSL	4:1b0d80432c79	379	W64LIT(0x06ca6351e003826f), W64LIT(0x142929670a0e6e70),
wolfSSL	4:1b0d80432c79	380	W64LIT(0x27b70a8546d22ffc), W64LIT(0x2e1b21385c26c926),
wolfSSL	4:1b0d80432c79	381	W64LIT(0x4d2c6dfc5ac42aed), W64LIT(0x53380d139d95b3df),
wolfSSL	4:1b0d80432c79	382	W64LIT(0x650a73548baf63de), W64LIT(0x766a0abb3c77b2a8),
wolfSSL	4:1b0d80432c79	383	W64LIT(0x81c2c92e47edaee6), W64LIT(0x92722c851482353b),
wolfSSL	4:1b0d80432c79	384	W64LIT(0xa2bfe8a14cf10364), W64LIT(0xa81a664bbc423001),
wolfSSL	4:1b0d80432c79	385	W64LIT(0xc24b8b70d0f89791), W64LIT(0xc76c51a30654be30),
wolfSSL	4:1b0d80432c79	386	W64LIT(0xd192e819d6ef5218), W64LIT(0xd69906245565a910),
wolfSSL	4:1b0d80432c79	387	W64LIT(0xf40e35855771202a), W64LIT(0x106aa07032bbd1b8),
wolfSSL	4:1b0d80432c79	388	W64LIT(0x19a4c116b8d2d0c8), W64LIT(0x1e376c085141ab53),
wolfSSL	4:1b0d80432c79	389	W64LIT(0x2748774cdf8eeb99), W64LIT(0x34b0bcb5e19b48a8),
wolfSSL	4:1b0d80432c79	390	W64LIT(0x391c0cb3c5c95a63), W64LIT(0x4ed8aa4ae3418acb),
wolfSSL	4:1b0d80432c79	391	W64LIT(0x5b9cca4f7763e373), W64LIT(0x682e6ff3d6b2b8a3),
wolfSSL	4:1b0d80432c79	392	W64LIT(0x748f82ee5defb2fc), W64LIT(0x78a5636f43172f60),
wolfSSL	4:1b0d80432c79	393	W64LIT(0x84c87814a1f0ab72), W64LIT(0x8cc702081a6439ec),
wolfSSL	4:1b0d80432c79	394	W64LIT(0x90befffa23631e28), W64LIT(0xa4506cebde82bde9),
wolfSSL	4:1b0d80432c79	395	W64LIT(0xbef9a3f7b2c67915), W64LIT(0xc67178f2e372532b),
wolfSSL	4:1b0d80432c79	396	W64LIT(0xca273eceea26619c), W64LIT(0xd186b8c721c0c207),
wolfSSL	4:1b0d80432c79	397	W64LIT(0xeada7dd6cde0eb1e), W64LIT(0xf57d4f7fee6ed178),
wolfSSL	4:1b0d80432c79	398	W64LIT(0x06f067aa72176fba), W64LIT(0x0a637dc5a2c898a6),
wolfSSL	4:1b0d80432c79	399	W64LIT(0x113f9804bef90dae), W64LIT(0x1b710b35131c471b),
wolfSSL	4:1b0d80432c79	400	W64LIT(0x28db77f523047d84), W64LIT(0x32caab7b40c72493),
wolfSSL	4:1b0d80432c79	401	W64LIT(0x3c9ebe0a15c9bebc), W64LIT(0x431d67c49c100d4c),
wolfSSL	4:1b0d80432c79	402	W64LIT(0x4cc5d4becb3e42b6), W64LIT(0x597f299cfc657e2a),
wolfSSL	4:1b0d80432c79	403	W64LIT(0x5fcb6fab3ad6faec), W64LIT(0x6c44198c4a475817)
wolfSSL	4:1b0d80432c79	404	};
wolfSSL	4:1b0d80432c79	405
wolfSSL	4:1b0d80432c79	406
wolfSSL	4:1b0d80432c79	407
wolfSSL	4:1b0d80432c79	408	#define blk0(i) (W[i] = sha512->buffer[i])
wolfSSL	4:1b0d80432c79	409
wolfSSL	4:1b0d80432c79	410	#define blk2(i) (W[i&15]+=s1(W[(i-2)&15])+W[(i-7)&15]+s0(W[(i-15)&15]))
wolfSSL	4:1b0d80432c79	411
wolfSSL	4:1b0d80432c79	412	#define Ch(x,y,z) (z^(x&(y^z)))
wolfSSL	4:1b0d80432c79	413	#define Maj(x,y,z) ((x&y)\|(z&(x\|y)))
wolfSSL	4:1b0d80432c79	414
wolfSSL	4:1b0d80432c79	415	#define a(i) T[(0-i)&7]
wolfSSL	4:1b0d80432c79	416	#define b(i) T[(1-i)&7]
wolfSSL	4:1b0d80432c79	417	#define c(i) T[(2-i)&7]
wolfSSL	4:1b0d80432c79	418	#define d(i) T[(3-i)&7]
wolfSSL	4:1b0d80432c79	419	#define e(i) T[(4-i)&7]
wolfSSL	4:1b0d80432c79	420	#define f(i) T[(5-i)&7]
wolfSSL	4:1b0d80432c79	421	#define g(i) T[(6-i)&7]
wolfSSL	4:1b0d80432c79	422	#define h(i) T[(7-i)&7]
wolfSSL	4:1b0d80432c79	423
wolfSSL	4:1b0d80432c79	424	#define S0(x) (rotrFixed64(x,28)^rotrFixed64(x,34)^rotrFixed64(x,39))
wolfSSL	4:1b0d80432c79	425	#define S1(x) (rotrFixed64(x,14)^rotrFixed64(x,18)^rotrFixed64(x,41))
wolfSSL	4:1b0d80432c79	426	#define s0(x) (rotrFixed64(x,1)^rotrFixed64(x,8)^(x>>7))
wolfSSL	4:1b0d80432c79	427	#define s1(x) (rotrFixed64(x,19)^rotrFixed64(x,61)^(x>>6))
wolfSSL	4:1b0d80432c79	428
wolfSSL	4:1b0d80432c79	429	#define R(i) h(i)+=S1(e(i))+Ch(e(i),f(i),g(i))+K[i+j]+(j?blk2(i):blk0(i));\
wolfSSL	4:1b0d80432c79	430	d(i)+=h(i);h(i)+=S0(a(i))+Maj(a(i),b(i),c(i))
wolfSSL	4:1b0d80432c79	431
wolfSSL	4:1b0d80432c79	432	#define blk384(i) (W[i] = sha384->buffer[i])
wolfSSL	4:1b0d80432c79	433
wolfSSL	4:1b0d80432c79	434	#define R2(i) h(i)+=S1(e(i))+Ch(e(i),f(i),g(i))+K[i+j]+(j?blk2(i):blk384(i));\
wolfSSL	4:1b0d80432c79	435	d(i)+=h(i);h(i)+=S0(a(i))+Maj(a(i),b(i),c(i))
wolfSSL	4:1b0d80432c79	436
wolfSSL	4:1b0d80432c79	437	static int _Transform(Sha512* sha512)
wolfSSL	4:1b0d80432c79	438	{
wolfSSL	4:1b0d80432c79	439	const word64* K = K512;
wolfSSL	4:1b0d80432c79	440
wolfSSL	4:1b0d80432c79	441	word32 j;
wolfSSL	4:1b0d80432c79	442	word64 T[8];
wolfSSL	4:1b0d80432c79	443
wolfSSL	4:1b0d80432c79	444
wolfSSL	4:1b0d80432c79	445	#ifdef WOLFSSL_SMALL_STACK
wolfSSL	4:1b0d80432c79	446	word64* W;
wolfSSL	4:1b0d80432c79	447	W = (word64) XMALLOC(sizeof(word64) 16, NULL, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	4:1b0d80432c79	448	if (W == NULL)
wolfSSL	4:1b0d80432c79	449	return MEMORY_E;
wolfSSL	4:1b0d80432c79	450	#else
wolfSSL	4:1b0d80432c79	451	word64 W[16];
wolfSSL	4:1b0d80432c79	452	#endif
wolfSSL	4:1b0d80432c79	453
wolfSSL	4:1b0d80432c79	454	/* Copy digest to working vars */
wolfSSL	4:1b0d80432c79	455	XMEMCPY(T, sha512->digest, sizeof(T));
wolfSSL	4:1b0d80432c79	456
wolfSSL	4:1b0d80432c79	457	#ifdef USE_SLOW_SHA2
wolfSSL	4:1b0d80432c79	458	/* over twice as small, but 50% slower */
wolfSSL	4:1b0d80432c79	459	/* 80 operations, not unrolled */
wolfSSL	4:1b0d80432c79	460	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	461	int m;
wolfSSL	4:1b0d80432c79	462	for (m = 0; m < 16; m++) { /* braces needed here for macros {} */
wolfSSL	4:1b0d80432c79	463	R(m);
wolfSSL	4:1b0d80432c79	464	}
wolfSSL	4:1b0d80432c79	465	}
wolfSSL	4:1b0d80432c79	466	#else
wolfSSL	4:1b0d80432c79	467	/* 80 operations, partially loop unrolled */
wolfSSL	4:1b0d80432c79	468	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	469	R( 0); R( 1); R( 2); R( 3);
wolfSSL	4:1b0d80432c79	470	R( 4); R( 5); R( 6); R( 7);
wolfSSL	4:1b0d80432c79	471	R( 8); R( 9); R(10); R(11);
wolfSSL	4:1b0d80432c79	472	R(12); R(13); R(14); R(15);
wolfSSL	4:1b0d80432c79	473	}
wolfSSL	4:1b0d80432c79	474	#endif /* USE_SLOW_SHA2 */
wolfSSL	4:1b0d80432c79	475
wolfSSL	4:1b0d80432c79	476	/* Add the working vars back into digest */
wolfSSL	4:1b0d80432c79	477
wolfSSL	4:1b0d80432c79	478	sha512->digest[0] += a(0);
wolfSSL	4:1b0d80432c79	479	sha512->digest[1] += b(0);
wolfSSL	4:1b0d80432c79	480	sha512->digest[2] += c(0);
wolfSSL	4:1b0d80432c79	481	sha512->digest[3] += d(0);
wolfSSL	4:1b0d80432c79	482	sha512->digest[4] += e(0);
wolfSSL	4:1b0d80432c79	483	sha512->digest[5] += f(0);
wolfSSL	4:1b0d80432c79	484	sha512->digest[6] += g(0);
wolfSSL	4:1b0d80432c79	485	sha512->digest[7] += h(0);
wolfSSL	4:1b0d80432c79	486
wolfSSL	4:1b0d80432c79	487	/* Wipe variables */
wolfSSL	4:1b0d80432c79	488	ForceZero(W, sizeof(word64) * 16);
wolfSSL	4:1b0d80432c79	489	ForceZero(T, sizeof(T));
wolfSSL	4:1b0d80432c79	490
wolfSSL	4:1b0d80432c79	491	#ifdef WOLFSSL_SMALL_STACK
wolfSSL	4:1b0d80432c79	492	XFREE(W, NULL, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	4:1b0d80432c79	493	#endif
wolfSSL	4:1b0d80432c79	494
wolfSSL	4:1b0d80432c79	495	return 0;
wolfSSL	4:1b0d80432c79	496	}
wolfSSL	4:1b0d80432c79	497
wolfSSL	4:1b0d80432c79	498
wolfSSL	4:1b0d80432c79	499	static INLINE void AddLength(Sha512* sha512, word32 len)
wolfSSL	4:1b0d80432c79	500	{
wolfSSL	4:1b0d80432c79	501	word32 tmp = sha512->loLen;
wolfSSL	4:1b0d80432c79	502	if ( (sha512->loLen += len) < tmp)
wolfSSL	4:1b0d80432c79	503	sha512->hiLen++; /* carry low to high */
wolfSSL	4:1b0d80432c79	504	}
wolfSSL	4:1b0d80432c79	505
wolfSSL	4:1b0d80432c79	506	int wc_Sha512Update(Sha512* sha512, const byte* data, word32 len)
wolfSSL	4:1b0d80432c79	507	{
wolfSSL	4:1b0d80432c79	508	/* do block size increments */
wolfSSL	4:1b0d80432c79	509	byte* local = (byte*)sha512->buffer;
wolfSSL	4:1b0d80432c79	510	SAVE_XMM_YMM ; /* for Intel AVX */
wolfSSL	4:1b0d80432c79	511
wolfSSL	4:1b0d80432c79	512	while (len) {
wolfSSL	4:1b0d80432c79	513	word32 add = min(len, SHA512_BLOCK_SIZE - sha512->buffLen);
wolfSSL	4:1b0d80432c79	514	XMEMCPY(&local[sha512->buffLen], data, add);
wolfSSL	4:1b0d80432c79	515
wolfSSL	4:1b0d80432c79	516	sha512->buffLen += add;
wolfSSL	4:1b0d80432c79	517	data += add;
wolfSSL	4:1b0d80432c79	518	len -= add;
wolfSSL	4:1b0d80432c79	519
wolfSSL	4:1b0d80432c79	520	if (sha512->buffLen == SHA512_BLOCK_SIZE) {
wolfSSL	4:1b0d80432c79	521	int ret;
wolfSSL	4:1b0d80432c79	522	#if defined(LITTLE_ENDIAN_ORDER)
wolfSSL	4:1b0d80432c79	523	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	524	if(!IS_INTEL_AVX1 && !IS_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	525	#endif
wolfSSL	4:1b0d80432c79	526	ByteReverseWords64(sha512->buffer, sha512->buffer,
wolfSSL	4:1b0d80432c79	527	SHA512_BLOCK_SIZE);
wolfSSL	4:1b0d80432c79	528	#endif
wolfSSL	4:1b0d80432c79	529	ret = Transform(sha512);
wolfSSL	4:1b0d80432c79	530	if (ret != 0)
wolfSSL	4:1b0d80432c79	531	return ret;
wolfSSL	4:1b0d80432c79	532
wolfSSL	4:1b0d80432c79	533	AddLength(sha512, SHA512_BLOCK_SIZE);
wolfSSL	4:1b0d80432c79	534	sha512->buffLen = 0;
wolfSSL	4:1b0d80432c79	535	}
wolfSSL	4:1b0d80432c79	536	}
wolfSSL	4:1b0d80432c79	537	return 0;
wolfSSL	4:1b0d80432c79	538	}
wolfSSL	4:1b0d80432c79	539
wolfSSL	4:1b0d80432c79	540
wolfSSL	4:1b0d80432c79	541	int wc_Sha512Final(Sha512* sha512, byte* hash)
wolfSSL	4:1b0d80432c79	542	{
wolfSSL	4:1b0d80432c79	543	byte* local = (byte*)sha512->buffer;
wolfSSL	4:1b0d80432c79	544	int ret;
wolfSSL	4:1b0d80432c79	545
wolfSSL	4:1b0d80432c79	546	SAVE_XMM_YMM ; /* for Intel AVX */
wolfSSL	4:1b0d80432c79	547	AddLength(sha512, sha512->buffLen); /* before adding pads */
wolfSSL	4:1b0d80432c79	548
wolfSSL	4:1b0d80432c79	549	local[sha512->buffLen++] = 0x80; /* add 1 */
wolfSSL	4:1b0d80432c79	550
wolfSSL	4:1b0d80432c79	551	/* pad with zeros */
wolfSSL	4:1b0d80432c79	552	if (sha512->buffLen > SHA512_PAD_SIZE) {
wolfSSL	4:1b0d80432c79	553	XMEMSET(&local[sha512->buffLen], 0, SHA512_BLOCK_SIZE -sha512->buffLen);
wolfSSL	4:1b0d80432c79	554	sha512->buffLen += SHA512_BLOCK_SIZE - sha512->buffLen;
wolfSSL	4:1b0d80432c79	555	#if defined(LITTLE_ENDIAN_ORDER)
wolfSSL	4:1b0d80432c79	556	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	557	if(!IS_INTEL_AVX1 && !IS_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	558	#endif
wolfSSL	4:1b0d80432c79	559	ByteReverseWords64(sha512->buffer,sha512->buffer,SHA512_BLOCK_SIZE);
wolfSSL	4:1b0d80432c79	560	#endif
wolfSSL	4:1b0d80432c79	561	ret = Transform(sha512);
wolfSSL	4:1b0d80432c79	562	if (ret != 0)
wolfSSL	4:1b0d80432c79	563	return ret;
wolfSSL	4:1b0d80432c79	564
wolfSSL	4:1b0d80432c79	565	sha512->buffLen = 0;
wolfSSL	4:1b0d80432c79	566	}
wolfSSL	4:1b0d80432c79	567	XMEMSET(&local[sha512->buffLen], 0, SHA512_PAD_SIZE - sha512->buffLen);
wolfSSL	4:1b0d80432c79	568
wolfSSL	4:1b0d80432c79	569	/* put lengths in bits */
wolfSSL	4:1b0d80432c79	570	sha512->hiLen = (sha512->loLen >> (8*sizeof(sha512->loLen) - 3)) +
wolfSSL	4:1b0d80432c79	571	(sha512->hiLen << 3);
wolfSSL	4:1b0d80432c79	572	sha512->loLen = sha512->loLen << 3;
wolfSSL	4:1b0d80432c79	573
wolfSSL	4:1b0d80432c79	574	/* store lengths */
wolfSSL	4:1b0d80432c79	575	#if defined(LITTLE_ENDIAN_ORDER)
wolfSSL	4:1b0d80432c79	576	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	577	if(!IS_INTEL_AVX1 && !IS_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	578	#endif
wolfSSL	4:1b0d80432c79	579	ByteReverseWords64(sha512->buffer, sha512->buffer, SHA512_PAD_SIZE);
wolfSSL	4:1b0d80432c79	580	#endif
wolfSSL	4:1b0d80432c79	581	/* ! length ordering dependent on digest endian type ! */
wolfSSL	4:1b0d80432c79	582
wolfSSL	4:1b0d80432c79	583	sha512->buffer[SHA512_BLOCK_SIZE / sizeof(word64) - 2] = sha512->hiLen;
wolfSSL	4:1b0d80432c79	584	sha512->buffer[SHA512_BLOCK_SIZE / sizeof(word64) - 1] = sha512->loLen;
wolfSSL	4:1b0d80432c79	585	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	586	if(IS_INTEL_AVX1 \|\| IS_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	587	ByteReverseWords64(&(sha512->buffer[SHA512_BLOCK_SIZE / sizeof(word64) - 2]),
wolfSSL	4:1b0d80432c79	588	&(sha512->buffer[SHA512_BLOCK_SIZE / sizeof(word64) - 2]),
wolfSSL	4:1b0d80432c79	589	SHA512_BLOCK_SIZE - SHA512_PAD_SIZE);
wolfSSL	4:1b0d80432c79	590	#endif
wolfSSL	4:1b0d80432c79	591	ret = Transform(sha512);
wolfSSL	4:1b0d80432c79	592	if (ret != 0)
wolfSSL	4:1b0d80432c79	593	return ret;
wolfSSL	4:1b0d80432c79	594
wolfSSL	4:1b0d80432c79	595	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	4:1b0d80432c79	596	ByteReverseWords64(sha512->digest, sha512->digest, SHA512_DIGEST_SIZE);
wolfSSL	4:1b0d80432c79	597	#endif
wolfSSL	4:1b0d80432c79	598	XMEMCPY(hash, sha512->digest, SHA512_DIGEST_SIZE);
wolfSSL	4:1b0d80432c79	599
wolfSSL	4:1b0d80432c79	600	return wc_InitSha512(sha512); /* reset state */
wolfSSL	4:1b0d80432c79	601	}
wolfSSL	4:1b0d80432c79	602
wolfSSL	4:1b0d80432c79	603
wolfSSL	4:1b0d80432c79	604
wolfSSL	4:1b0d80432c79	605	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	606
wolfSSL	4:1b0d80432c79	607	#define Rx_1(i) h(i)+=S1(e(i))+Ch(e(i),f(i),g(i))+K[i+j] + W_X[i] ;
wolfSSL	4:1b0d80432c79	608	#define Rx_2(i) d(i)+=h(i);
wolfSSL	4:1b0d80432c79	609	#define Rx_3(i) h(i)+=S0(a(i))+Maj(a(i),b(i),c(i));
wolfSSL	4:1b0d80432c79	610
wolfSSL	4:1b0d80432c79	611	#if defined(HAVE_INTEL_RORX)
wolfSSL	4:1b0d80432c79	612	#define Rx_RORX_1(i) h(i)+=S1_RORX(e(i))+Ch(e(i),f(i),g(i))+K[i+j] + W_X[i] ;
wolfSSL	4:1b0d80432c79	613	#define Rx_RORX_2(i) d(i)+=h(i);
wolfSSL	4:1b0d80432c79	614	#define Rx_RORX_3(i) h(i)+=S0_RORX(a(i))+Maj(a(i),b(i),c(i));
wolfSSL	4:1b0d80432c79	615	#endif
wolfSSL	4:1b0d80432c79	616
wolfSSL	4:1b0d80432c79	617	#endif
wolfSSL	4:1b0d80432c79	618
wolfSSL	4:1b0d80432c79	619	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	620	#define Ry_1(i, w) h(i)+=S1(e(i))+Ch(e(i),f(i),g(i))+K[i+j] + w ;
wolfSSL	4:1b0d80432c79	621	#define Ry_2(i, w) d(i)+=h(i);
wolfSSL	4:1b0d80432c79	622	#define Ry_3(i, w) h(i)+=S0(a(i))+Maj(a(i),b(i),c(i));
wolfSSL	4:1b0d80432c79	623	#endif
wolfSSL	4:1b0d80432c79	624
wolfSSL	4:1b0d80432c79	625	#if defined(HAVE_INTEL_AVX1) /* INLINE Assember for Intel AVX1 instructions */
wolfSSL	4:1b0d80432c79	626	#if defined(DEBUG_XMM)
wolfSSL	4:1b0d80432c79	627
wolfSSL	4:1b0d80432c79	628	#define SAVE_REG(i) __asm__ volatile("vmovdqu %%xmm"#i", %0 \n\t":"=m"(reg[i][0])::XMM_REGs);
wolfSSL	4:1b0d80432c79	629	#define RECV_REG(i) __asm__ volatile("vmovdqu %0, %%xmm"#i" \n\t"::"m"(reg[i][0]):XMM_REGs);
wolfSSL	4:1b0d80432c79	630
wolfSSL	4:1b0d80432c79	631	#define _DUMP_REG(REG, name)\
wolfSSL	4:1b0d80432c79	632	{ word64 buf[16] ;word64 reg[16][2];int k ;\
wolfSSL	4:1b0d80432c79	633	SAVE_REG(0); SAVE_REG(1); SAVE_REG(2); SAVE_REG(3); SAVE_REG(4); \
wolfSSL	4:1b0d80432c79	634	SAVE_REG(5); SAVE_REG(6); SAVE_REG(7);SAVE_REG(8); SAVE_REG(9); SAVE_REG(10);\
wolfSSL	4:1b0d80432c79	635	SAVE_REG(11); SAVE_REG(12); SAVE_REG(13); SAVE_REG(14); SAVE_REG(15); \
wolfSSL	4:1b0d80432c79	636	__asm__ volatile("vmovdqu %%"#REG", %0 \n\t":"=m"(buf[0])::XMM_REGs);\
wolfSSL	4:1b0d80432c79	637	printf(" "#name":\t") ; for(k=0; k<2; k++) printf("%016lx.", (word64)(buf[k])); printf("\n") ; \
wolfSSL	4:1b0d80432c79	638	RECV_REG(0); RECV_REG(1); RECV_REG(2); RECV_REG(3); RECV_REG(4);\
wolfSSL	4:1b0d80432c79	639	RECV_REG(5); RECV_REG(6); RECV_REG(7); RECV_REG(8); RECV_REG(9);\
wolfSSL	4:1b0d80432c79	640	RECV_REG(10); RECV_REG(11); RECV_REG(12); RECV_REG(13); RECV_REG(14); RECV_REG(15);\
wolfSSL	4:1b0d80432c79	641	}
wolfSSL	4:1b0d80432c79	642
wolfSSL	4:1b0d80432c79	643	#define DUMP_REG(REG) _DUMP_REG(REG, #REG)
wolfSSL	4:1b0d80432c79	644	#define PRINTF(fmt, ...)
wolfSSL	4:1b0d80432c79	645
wolfSSL	4:1b0d80432c79	646	#else
wolfSSL	4:1b0d80432c79	647
wolfSSL	4:1b0d80432c79	648	#define DUMP_REG(REG)
wolfSSL	4:1b0d80432c79	649	#define PRINTF(fmt, ...)
wolfSSL	4:1b0d80432c79	650
wolfSSL	4:1b0d80432c79	651	#endif
wolfSSL	4:1b0d80432c79	652
wolfSSL	4:1b0d80432c79	653	#define _MOVE_to_REG(xymm, mem) __asm__ volatile("vmovdqu %0, %%"#xymm" "\
wolfSSL	4:1b0d80432c79	654	:: "m"(mem):XMM_REGs) ;
wolfSSL	4:1b0d80432c79	655	#define _MOVE_to_MEM(mem,i, xymm) __asm__ volatile("vmovdqu %%"#xymm", %0" :\
wolfSSL	4:1b0d80432c79	656	"=m"(mem[i]),"=m"(mem[i+1]),"=m"(mem[i+2]),"=m"(mem[i+3])::XMM_REGs) ;
wolfSSL	4:1b0d80432c79	657	#define _MOVE(dest, src) __asm__ volatile("vmovdqu %%"#src", %%"\
wolfSSL	4:1b0d80432c79	658	#dest" ":::XMM_REGs) ;
wolfSSL	4:1b0d80432c79	659
wolfSSL	4:1b0d80432c79	660	#define _S_TEMP(dest, src, bits, temp) __asm__ volatile("vpsrlq $"#bits", %%"\
wolfSSL	4:1b0d80432c79	661	#src", %%"#dest"\n\tvpsllq $64-"#bits", %%"#src", %%"#temp"\n\tvpor %%"\
wolfSSL	4:1b0d80432c79	662	#temp",%%"#dest", %%"#dest" ":::XMM_REGs) ;
wolfSSL	4:1b0d80432c79	663	#define _AVX1_R(dest, src, bits) __asm__ volatile("vpsrlq $"#bits", %%"\
wolfSSL	4:1b0d80432c79	664	#src", %%"#dest" ":::XMM_REGs) ;
wolfSSL	4:1b0d80432c79	665	#define _XOR(dest, src1, src2) __asm__ volatile("vpxor %%"#src1", %%"\
wolfSSL	4:1b0d80432c79	666	#src2", %%"#dest" ":::XMM_REGs) ;
wolfSSL	4:1b0d80432c79	667	#define _OR(dest, src1, src2) __asm__ volatile("vpor %%"#src1", %%"\
wolfSSL	4:1b0d80432c79	668	#src2", %%"#dest" ":::XMM_REGs) ;
wolfSSL	4:1b0d80432c79	669	#define _ADD(dest, src1, src2) __asm__ volatile("vpaddq %%"#src1", %%"\
wolfSSL	4:1b0d80432c79	670	#src2", %%"#dest" ":::XMM_REGs) ;
wolfSSL	4:1b0d80432c79	671	#define _ADD_MEM(dest, src1, mem) __asm__ volatile("vpaddq %0, %%"#src1", %%"\
wolfSSL	4:1b0d80432c79	672	#dest" "::"m"(mem):XMM_REGs) ;
wolfSSL	4:1b0d80432c79	673
wolfSSL	4:1b0d80432c79	674	#define MOVE_to_REG(xymm, mem) _MOVE_to_REG(xymm, mem)
wolfSSL	4:1b0d80432c79	675	#define MOVE_to_MEM(mem, i, xymm) _MOVE_to_MEM(mem, i, xymm)
wolfSSL	4:1b0d80432c79	676	#define MOVE(dest, src) _MOVE(dest, src)
wolfSSL	4:1b0d80432c79	677
wolfSSL	4:1b0d80432c79	678	#define XOR(dest, src1, src2) _XOR(dest, src1, src2)
wolfSSL	4:1b0d80432c79	679	#define OR(dest, src1, src2) _OR(dest, src1, src2)
wolfSSL	4:1b0d80432c79	680	#define ADD(dest, src1, src2) _ADD(dest, src1, src2)
wolfSSL	4:1b0d80432c79	681
wolfSSL	4:1b0d80432c79	682	#define S_TMP(dest, src, bits, temp) _S_TEMP(dest, src, bits, temp);
wolfSSL	4:1b0d80432c79	683	#define AVX1_S(dest, src, bits) S_TMP(dest, src, bits, S_TEMP)
wolfSSL	4:1b0d80432c79	684	#define AVX1_R(dest, src, bits) _AVX1_R(dest, src, bits)
wolfSSL	4:1b0d80432c79	685
wolfSSL	4:1b0d80432c79	686	#define Init_Mask(mask) \
wolfSSL	4:1b0d80432c79	687	__asm__ volatile("vmovdqu %0, %%xmm1\n\t"::"m"(mask):"%xmm1") ;
wolfSSL	4:1b0d80432c79	688
wolfSSL	4:1b0d80432c79	689	#define _W_from_buff1(w, buff, xmm) \
wolfSSL	4:1b0d80432c79	690	/* X0..3(xmm4..7), W[0..15] = sha512->buffer[0.15]; */\
wolfSSL	4:1b0d80432c79	691	__asm__ volatile("vmovdqu %1, %%"#xmm"\n\t"\
wolfSSL	4:1b0d80432c79	692	"vpshufb %%xmm1, %%"#xmm", %%"#xmm"\n\t"\
wolfSSL	4:1b0d80432c79	693	"vmovdqu %%"#xmm", %0"\
wolfSSL	4:1b0d80432c79	694	:"=m"(w): "m"(buff):"%xmm0") ;
wolfSSL	4:1b0d80432c79	695
wolfSSL	4:1b0d80432c79	696	#define W_from_buff1(w, buff, xmm) _W_from_buff1(w, buff, xmm)
wolfSSL	4:1b0d80432c79	697
wolfSSL	4:1b0d80432c79	698	#define W_from_buff(w, buff)\
wolfSSL	4:1b0d80432c79	699	Init_Mask(mBYTE_FLIP_MASK[0]) ;\
wolfSSL	4:1b0d80432c79	700	W_from_buff1(w[0], buff[0], W_0);\
wolfSSL	4:1b0d80432c79	701	W_from_buff1(w[2], buff[2], W_2);\
wolfSSL	4:1b0d80432c79	702	W_from_buff1(w[4], buff[4], W_4);\
wolfSSL	4:1b0d80432c79	703	W_from_buff1(w[6], buff[6], W_6);\
wolfSSL	4:1b0d80432c79	704	W_from_buff1(w[8], buff[8], W_8);\
wolfSSL	4:1b0d80432c79	705	W_from_buff1(w[10],buff[10],W_10);\
wolfSSL	4:1b0d80432c79	706	W_from_buff1(w[12],buff[12],W_12);\
wolfSSL	4:1b0d80432c79	707	W_from_buff1(w[14],buff[14],W_14);
wolfSSL	4:1b0d80432c79	708
wolfSSL	4:1b0d80432c79	709	static word64 mBYTE_FLIP_MASK[] = { 0x0001020304050607, 0x08090a0b0c0d0e0f } ;
wolfSSL	4:1b0d80432c79	710
wolfSSL	4:1b0d80432c79	711	#define W_I_15 xmm14
wolfSSL	4:1b0d80432c79	712	#define W_I_7 xmm11
wolfSSL	4:1b0d80432c79	713	#define W_I_2 xmm13
wolfSSL	4:1b0d80432c79	714	#define W_I xmm12
wolfSSL	4:1b0d80432c79	715	#define G_TEMP xmm0
wolfSSL	4:1b0d80432c79	716	#define S_TEMP xmm1
wolfSSL	4:1b0d80432c79	717	#define XMM_TEMP0 xmm2
wolfSSL	4:1b0d80432c79	718
wolfSSL	4:1b0d80432c79	719	#define W_0 xmm12
wolfSSL	4:1b0d80432c79	720	#define W_2 xmm3
wolfSSL	4:1b0d80432c79	721	#define W_4 xmm4
wolfSSL	4:1b0d80432c79	722	#define W_6 xmm5
wolfSSL	4:1b0d80432c79	723	#define W_8 xmm6
wolfSSL	4:1b0d80432c79	724	#define W_10 xmm7
wolfSSL	4:1b0d80432c79	725	#define W_12 xmm8
wolfSSL	4:1b0d80432c79	726	#define W_14 xmm9
wolfSSL	4:1b0d80432c79	727
wolfSSL	4:1b0d80432c79	728	#define XMM_REGs
wolfSSL	4:1b0d80432c79	729
wolfSSL	4:1b0d80432c79	730	#define s0_1(dest, src) AVX1_S(dest, src, 1);
wolfSSL	4:1b0d80432c79	731	#define s0_2(dest, src) AVX1_S(G_TEMP, src, 8); XOR(dest, G_TEMP, dest) ;
wolfSSL	4:1b0d80432c79	732	#define s0_3(dest, src) AVX1_R(G_TEMP, src, 7); XOR(dest, G_TEMP, dest) ;
wolfSSL	4:1b0d80432c79	733
wolfSSL	4:1b0d80432c79	734	#define s1_1(dest, src) AVX1_S(dest, src, 19);
wolfSSL	4:1b0d80432c79	735	#define s1_2(dest, src) AVX1_S(G_TEMP, src, 61); XOR(dest, G_TEMP, dest) ;
wolfSSL	4:1b0d80432c79	736	#define s1_3(dest, src) AVX1_R(G_TEMP, src, 6); XOR(dest, G_TEMP, dest) ;
wolfSSL	4:1b0d80432c79	737
wolfSSL	4:1b0d80432c79	738	#define s0_(dest, src) s0_1(dest, src) ; s0_2(dest, src) ; s0_3(dest, src)
wolfSSL	4:1b0d80432c79	739	#define s1_(dest, src) s1_1(dest, src) ; s1_2(dest, src) ; s1_3(dest, src)
wolfSSL	4:1b0d80432c79	740
wolfSSL	4:1b0d80432c79	741	#define Block_xx_1(i) \
wolfSSL	4:1b0d80432c79	742	MOVE_to_REG(W_I_15, W_X[(i-15)&15]) ;\
wolfSSL	4:1b0d80432c79	743	MOVE_to_REG(W_I_7, W_X[(i- 7)&15]) ;\
wolfSSL	4:1b0d80432c79	744
wolfSSL	4:1b0d80432c79	745	#define Block_xx_2(i) \
wolfSSL	4:1b0d80432c79	746	MOVE_to_REG(W_I_2, W_X[(i- 2)&15]) ;\
wolfSSL	4:1b0d80432c79	747	MOVE_to_REG(W_I, W_X[(i)]) ;\
wolfSSL	4:1b0d80432c79	748
wolfSSL	4:1b0d80432c79	749	#define Block_xx_3(i) \
wolfSSL	4:1b0d80432c79	750	s0_ (XMM_TEMP0, W_I_15) ;\
wolfSSL	4:1b0d80432c79	751
wolfSSL	4:1b0d80432c79	752	#define Block_xx_4(i) \
wolfSSL	4:1b0d80432c79	753	ADD(W_I, W_I, XMM_TEMP0) ;\
wolfSSL	4:1b0d80432c79	754	ADD(W_I, W_I, W_I_7) ;\
wolfSSL	4:1b0d80432c79	755
wolfSSL	4:1b0d80432c79	756	#define Block_xx_5(i) \
wolfSSL	4:1b0d80432c79	757	s1_ (XMM_TEMP0, W_I_2) ;\
wolfSSL	4:1b0d80432c79	758
wolfSSL	4:1b0d80432c79	759	#define Block_xx_6(i) \
wolfSSL	4:1b0d80432c79	760	ADD(W_I, W_I, XMM_TEMP0) ;\
wolfSSL	4:1b0d80432c79	761	MOVE_to_MEM(W_X,i, W_I) ;\
wolfSSL	4:1b0d80432c79	762	if(i==0)\
wolfSSL	4:1b0d80432c79	763	MOVE_to_MEM(W_X,16, W_I) ;\
wolfSSL	4:1b0d80432c79	764
wolfSSL	4:1b0d80432c79	765	#define Block_xx_7(i) \
wolfSSL	4:1b0d80432c79	766	MOVE_to_REG(W_I_15, W_X[(i-15)&15]) ;\
wolfSSL	4:1b0d80432c79	767	MOVE_to_REG(W_I_7, W_X[(i- 7)&15]) ;\
wolfSSL	4:1b0d80432c79	768
wolfSSL	4:1b0d80432c79	769	#define Block_xx_8(i) \
wolfSSL	4:1b0d80432c79	770	MOVE_to_REG(W_I_2, W_X[(i- 2)&15]) ;\
wolfSSL	4:1b0d80432c79	771	MOVE_to_REG(W_I, W_X[(i)]) ;\
wolfSSL	4:1b0d80432c79	772
wolfSSL	4:1b0d80432c79	773	#define Block_xx_9(i) \
wolfSSL	4:1b0d80432c79	774	s0_ (XMM_TEMP0, W_I_15) ;\
wolfSSL	4:1b0d80432c79	775
wolfSSL	4:1b0d80432c79	776	#define Block_xx_10(i) \
wolfSSL	4:1b0d80432c79	777	ADD(W_I, W_I, XMM_TEMP0) ;\
wolfSSL	4:1b0d80432c79	778	ADD(W_I, W_I, W_I_7) ;\
wolfSSL	4:1b0d80432c79	779
wolfSSL	4:1b0d80432c79	780	#define Block_xx_11(i) \
wolfSSL	4:1b0d80432c79	781	s1_ (XMM_TEMP0, W_I_2) ;\
wolfSSL	4:1b0d80432c79	782
wolfSSL	4:1b0d80432c79	783	#define Block_xx_12(i) \
wolfSSL	4:1b0d80432c79	784	ADD(W_I, W_I, XMM_TEMP0) ;\
wolfSSL	4:1b0d80432c79	785	MOVE_to_MEM(W_X,i, W_I) ;\
wolfSSL	4:1b0d80432c79	786	if((i)==0)\
wolfSSL	4:1b0d80432c79	787	MOVE_to_MEM(W_X,16, W_I) ;\
wolfSSL	4:1b0d80432c79	788
wolfSSL	4:1b0d80432c79	789	static INLINE void Block_0_1(word64 *W_X) { Block_xx_1(0) ; }
wolfSSL	4:1b0d80432c79	790	static INLINE void Block_0_2(word64 *W_X) { Block_xx_2(0) ; }
wolfSSL	4:1b0d80432c79	791	static INLINE void Block_0_3(void) { Block_xx_3(0) ; }
wolfSSL	4:1b0d80432c79	792	static INLINE void Block_0_4(void) { Block_xx_4(0) ; }
wolfSSL	4:1b0d80432c79	793	static INLINE void Block_0_5(void) { Block_xx_5(0) ; }
wolfSSL	4:1b0d80432c79	794	static INLINE void Block_0_6(word64 *W_X) { Block_xx_6(0) ; }
wolfSSL	4:1b0d80432c79	795	static INLINE void Block_0_7(word64 *W_X) { Block_xx_7(2) ; }
wolfSSL	4:1b0d80432c79	796	static INLINE void Block_0_8(word64 *W_X) { Block_xx_8(2) ; }
wolfSSL	4:1b0d80432c79	797	static INLINE void Block_0_9(void) { Block_xx_9(2) ; }
wolfSSL	4:1b0d80432c79	798	static INLINE void Block_0_10(void){ Block_xx_10(2) ; }
wolfSSL	4:1b0d80432c79	799	static INLINE void Block_0_11(void){ Block_xx_11(2) ; }
wolfSSL	4:1b0d80432c79	800	static INLINE void Block_0_12(word64 *W_X){ Block_xx_12(2) ; }
wolfSSL	4:1b0d80432c79	801
wolfSSL	4:1b0d80432c79	802	static INLINE void Block_4_1(word64 *W_X) { Block_xx_1(4) ; }
wolfSSL	4:1b0d80432c79	803	static INLINE void Block_4_2(word64 *W_X) { Block_xx_2(4) ; }
wolfSSL	4:1b0d80432c79	804	static INLINE void Block_4_3(void) { Block_xx_3(4) ; }
wolfSSL	4:1b0d80432c79	805	static INLINE void Block_4_4(void) { Block_xx_4(4) ; }
wolfSSL	4:1b0d80432c79	806	static INLINE void Block_4_5(void) { Block_xx_5(4) ; }
wolfSSL	4:1b0d80432c79	807	static INLINE void Block_4_6(word64 *W_X) { Block_xx_6(4) ; }
wolfSSL	4:1b0d80432c79	808	static INLINE void Block_4_7(word64 *W_X) { Block_xx_7(6) ; }
wolfSSL	4:1b0d80432c79	809	static INLINE void Block_4_8(word64 *W_X) { Block_xx_8(6) ; }
wolfSSL	4:1b0d80432c79	810	static INLINE void Block_4_9(void) { Block_xx_9(6) ; }
wolfSSL	4:1b0d80432c79	811	static INLINE void Block_4_10(void){ Block_xx_10(6) ; }
wolfSSL	4:1b0d80432c79	812	static INLINE void Block_4_11(void){ Block_xx_11(6) ; }
wolfSSL	4:1b0d80432c79	813	static INLINE void Block_4_12(word64 *W_X){ Block_xx_12(6) ; }
wolfSSL	4:1b0d80432c79	814
wolfSSL	4:1b0d80432c79	815	static INLINE void Block_8_1(word64 *W_X) { Block_xx_1(8) ; }
wolfSSL	4:1b0d80432c79	816	static INLINE void Block_8_2(word64 *W_X) { Block_xx_2(8) ; }
wolfSSL	4:1b0d80432c79	817	static INLINE void Block_8_3(void) { Block_xx_3(8) ; }
wolfSSL	4:1b0d80432c79	818	static INLINE void Block_8_4(void) { Block_xx_4(8) ; }
wolfSSL	4:1b0d80432c79	819	static INLINE void Block_8_5(void) { Block_xx_5(8) ; }
wolfSSL	4:1b0d80432c79	820	static INLINE void Block_8_6(word64 *W_X) { Block_xx_6(8) ; }
wolfSSL	4:1b0d80432c79	821	static INLINE void Block_8_7(word64 *W_X) { Block_xx_7(10) ; }
wolfSSL	4:1b0d80432c79	822	static INLINE void Block_8_8(word64 *W_X) { Block_xx_8(10) ; }
wolfSSL	4:1b0d80432c79	823	static INLINE void Block_8_9(void) { Block_xx_9(10) ; }
wolfSSL	4:1b0d80432c79	824	static INLINE void Block_8_10(void){ Block_xx_10(10) ; }
wolfSSL	4:1b0d80432c79	825	static INLINE void Block_8_11(void){ Block_xx_11(10) ; }
wolfSSL	4:1b0d80432c79	826	static INLINE void Block_8_12(word64 *W_X){ Block_xx_12(10) ; }
wolfSSL	4:1b0d80432c79	827
wolfSSL	4:1b0d80432c79	828	static INLINE void Block_12_1(word64 *W_X) { Block_xx_1(12) ; }
wolfSSL	4:1b0d80432c79	829	static INLINE void Block_12_2(word64 *W_X) { Block_xx_2(12) ; }
wolfSSL	4:1b0d80432c79	830	static INLINE void Block_12_3(void) { Block_xx_3(12) ; }
wolfSSL	4:1b0d80432c79	831	static INLINE void Block_12_4(void) { Block_xx_4(12) ; }
wolfSSL	4:1b0d80432c79	832	static INLINE void Block_12_5(void) { Block_xx_5(12) ; }
wolfSSL	4:1b0d80432c79	833	static INLINE void Block_12_6(word64 *W_X) { Block_xx_6(12) ; }
wolfSSL	4:1b0d80432c79	834	static INLINE void Block_12_7(word64 *W_X) { Block_xx_7(14) ; }
wolfSSL	4:1b0d80432c79	835	static INLINE void Block_12_8(word64 *W_X) { Block_xx_8(14) ; }
wolfSSL	4:1b0d80432c79	836	static INLINE void Block_12_9(void) { Block_xx_9(14) ; }
wolfSSL	4:1b0d80432c79	837	static INLINE void Block_12_10(void){ Block_xx_10(14) ; }
wolfSSL	4:1b0d80432c79	838	static INLINE void Block_12_11(void){ Block_xx_11(14) ; }
wolfSSL	4:1b0d80432c79	839	static INLINE void Block_12_12(word64 *W_X){ Block_xx_12(14) ; }
wolfSSL	4:1b0d80432c79	840
wolfSSL	4:1b0d80432c79	841	#endif
wolfSSL	4:1b0d80432c79	842
wolfSSL	4:1b0d80432c79	843	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	844	static const unsigned long mBYTE_FLIP_MASK_Y[] =
wolfSSL	4:1b0d80432c79	845	{ 0x0001020304050607, 0x08090a0b0c0d0e0f, 0x0001020304050607, 0x08090a0b0c0d0e0f } ;
wolfSSL	4:1b0d80432c79	846
wolfSSL	4:1b0d80432c79	847	#define W_from_buff_Y(buff)\
wolfSSL	4:1b0d80432c79	848	{ /* X0..3(ymm9..12), W_X[0..15] = sha512->buffer[0.15]; */\
wolfSSL	4:1b0d80432c79	849	__asm__ volatile("vmovdqu %0, %%ymm8\n\t"::"m"(mBYTE_FLIP_MASK_Y[0]):YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	850	__asm__ volatile("vmovdqu %0, %%ymm12\n\t"\
wolfSSL	4:1b0d80432c79	851	"vmovdqu %1, %%ymm4\n\t"\
wolfSSL	4:1b0d80432c79	852	"vpshufb %%ymm8, %%ymm12, %%ymm12\n\t"\
wolfSSL	4:1b0d80432c79	853	"vpshufb %%ymm8, %%ymm4, %%ymm4\n\t"\
wolfSSL	4:1b0d80432c79	854	:: "m"(buff[0]), "m"(buff[4]):YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	855	__asm__ volatile("vmovdqu %0, %%ymm5\n\t"\
wolfSSL	4:1b0d80432c79	856	"vmovdqu %1, %%ymm6\n\t"\
wolfSSL	4:1b0d80432c79	857	"vpshufb %%ymm8, %%ymm5, %%ymm5\n\t"\
wolfSSL	4:1b0d80432c79	858	"vpshufb %%ymm8, %%ymm6, %%ymm6\n\t"\
wolfSSL	4:1b0d80432c79	859	:: "m"(buff[8]), "m"(buff[12]):YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	860	}
wolfSSL	4:1b0d80432c79	861
wolfSSL	4:1b0d80432c79	862	#if defined(DEBUG_YMM)
wolfSSL	4:1b0d80432c79	863
wolfSSL	4:1b0d80432c79	864	#define SAVE_REG_Y(i) __asm__ volatile("vmovdqu %%ymm"#i", %0 \n\t":"=m"(reg[i-4][0])::YMM_REGs);
wolfSSL	4:1b0d80432c79	865	#define RECV_REG_Y(i) __asm__ volatile("vmovdqu %0, %%ymm"#i" \n\t"::"m"(reg[i-4][0]):YMM_REGs);
wolfSSL	4:1b0d80432c79	866
wolfSSL	4:1b0d80432c79	867	#define _DUMP_REG_Y(REG, name)\
wolfSSL	4:1b0d80432c79	868	{ word64 buf[16] ;word64 reg[16][2];int k ;\
wolfSSL	4:1b0d80432c79	869	SAVE_REG_Y(4); SAVE_REG_Y(5); SAVE_REG_Y(6); SAVE_REG_Y(7); \
wolfSSL	4:1b0d80432c79	870	SAVE_REG_Y(8); SAVE_REG_Y(9); SAVE_REG_Y(10); SAVE_REG_Y(11); SAVE_REG_Y(12);\
wolfSSL	4:1b0d80432c79	871	SAVE_REG_Y(13); SAVE_REG_Y(14); SAVE_REG_Y(15); \
wolfSSL	4:1b0d80432c79	872	__asm__ volatile("vmovdqu %%"#REG", %0 \n\t":"=m"(buf[0])::YMM_REGs);\
wolfSSL	4:1b0d80432c79	873	printf(" "#name":\t") ; for(k=0; k<4; k++) printf("%016lx.", (word64)buf[k]) ; printf("\n") ; \
wolfSSL	4:1b0d80432c79	874	RECV_REG_Y(4); RECV_REG_Y(5); RECV_REG_Y(6); RECV_REG_Y(7); \
wolfSSL	4:1b0d80432c79	875	RECV_REG_Y(8); RECV_REG_Y(9); RECV_REG_Y(10); RECV_REG_Y(11); RECV_REG_Y(12); \
wolfSSL	4:1b0d80432c79	876	RECV_REG_Y(13); RECV_REG_Y(14); RECV_REG_Y(15);\
wolfSSL	4:1b0d80432c79	877	}
wolfSSL	4:1b0d80432c79	878
wolfSSL	4:1b0d80432c79	879	#define DUMP_REG_Y(REG) _DUMP_REG_Y(REG, #REG)
wolfSSL	4:1b0d80432c79	880	#define DUMP_REG2_Y(REG) _DUMP_REG_Y(REG, #REG)
wolfSSL	4:1b0d80432c79	881	#define PRINTF_Y(fmt, ...)
wolfSSL	4:1b0d80432c79	882
wolfSSL	4:1b0d80432c79	883	#else
wolfSSL	4:1b0d80432c79	884
wolfSSL	4:1b0d80432c79	885	#define DUMP_REG_Y(REG)
wolfSSL	4:1b0d80432c79	886	#define DUMP_REG2_Y(REG)
wolfSSL	4:1b0d80432c79	887	#define PRINTF_Y(fmt, ...)
wolfSSL	4:1b0d80432c79	888
wolfSSL	4:1b0d80432c79	889	#endif
wolfSSL	4:1b0d80432c79	890
wolfSSL	4:1b0d80432c79	891	#define _MOVE_to_REGy(ymm, mem) __asm__ volatile("vmovdqu %0, %%"#ymm" "\
wolfSSL	4:1b0d80432c79	892	:: "m"(mem):YMM_REGs) ;
wolfSSL	4:1b0d80432c79	893	#define _MOVE_to_MEMy(mem,i, ymm) __asm__ volatile("vmovdqu %%"#ymm", %0" \
wolfSSL	4:1b0d80432c79	894	: "=m"(mem[i]),"=m"(mem[i+1]),"=m"(mem[i+2]),"=m"(mem[i+3])::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	895	#define _MOVE_128y(ymm0, ymm1, ymm2, map) __asm__ volatile("vperm2i128 $"\
wolfSSL	4:1b0d80432c79	896	#map", %%"#ymm2", %%"#ymm1", %%"#ymm0" ":::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	897	#define _S_TEMPy(dest, src, bits, temp) \
wolfSSL	4:1b0d80432c79	898	__asm__ volatile("vpsrlq $"#bits", %%"#src", %%"#dest"\n\tvpsllq $64-"#bits\
wolfSSL	4:1b0d80432c79	899	", %%"#src", %%"#temp"\n\tvpor %%"#temp",%%"#dest", %%"#dest" ":::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	900	#define _AVX2_R(dest, src, bits) __asm__ volatile("vpsrlq $"#bits", %%"\
wolfSSL	4:1b0d80432c79	901	#src", %%"#dest" ":::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	902	#define _XORy(dest, src1, src2) __asm__ volatile("vpxor %%"#src1", %%"\
wolfSSL	4:1b0d80432c79	903	#src2", %%"#dest" ":::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	904	#define _ADDy(dest, src1, src2) __asm__ volatile("vpaddq %%"#src1", %%"\
wolfSSL	4:1b0d80432c79	905	#src2", %%"#dest" ":::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	906	#define _BLENDy(map, dest, src1, src2) __asm__ volatile("vpblendd $"#map", %%"\
wolfSSL	4:1b0d80432c79	907	#src1", %%"#src2", %%"#dest" ":::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	908	#define _BLENDQy(map, dest, src1, src2) __asm__ volatile("vblendpd $"#map", %%"\
wolfSSL	4:1b0d80432c79	909	#src1", %%"#src2", %%"#dest" ":::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	910	#define _PERMQy(map, dest, src) __asm__ volatile("vpermq $"#map", %%"\
wolfSSL	4:1b0d80432c79	911	#src", %%"#dest" ":::YMM_REGs) ;
wolfSSL	4:1b0d80432c79	912
wolfSSL	4:1b0d80432c79	913	#define MOVE_to_REGy(ymm, mem) _MOVE_to_REGy(ymm, mem)
wolfSSL	4:1b0d80432c79	914	#define MOVE_to_MEMy(mem, i, ymm) _MOVE_to_MEMy(mem, i, ymm)
wolfSSL	4:1b0d80432c79	915
wolfSSL	4:1b0d80432c79	916	#define MOVE_128y(ymm0, ymm1, ymm2, map) _MOVE_128y(ymm0, ymm1, ymm2, map)
wolfSSL	4:1b0d80432c79	917	#define XORy(dest, src1, src2) _XORy(dest, src1, src2)
wolfSSL	4:1b0d80432c79	918	#define ADDy(dest, src1, src2) _ADDy(dest, src1, src2)
wolfSSL	4:1b0d80432c79	919	#define BLENDy(map, dest, src1, src2) _BLENDy(map, dest, src1, src2)
wolfSSL	4:1b0d80432c79	920	#define BLENDQy(map, dest, src1, src2) _BLENDQy(map, dest, src1, src2)
wolfSSL	4:1b0d80432c79	921	#define PERMQy(map, dest, src) _PERMQy(map, dest, src)
wolfSSL	4:1b0d80432c79	922
wolfSSL	4:1b0d80432c79	923
wolfSSL	4:1b0d80432c79	924	#define S_TMPy(dest, src, bits, temp) _S_TEMPy(dest, src, bits, temp);
wolfSSL	4:1b0d80432c79	925	#define AVX2_S(dest, src, bits) S_TMPy(dest, src, bits, S_TEMPy)
wolfSSL	4:1b0d80432c79	926	#define AVX2_R(dest, src, bits) _AVX2_R(dest, src, bits)
wolfSSL	4:1b0d80432c79	927
wolfSSL	4:1b0d80432c79	928
wolfSSL	4:1b0d80432c79	929	#define FEEDBACK1_to_W_I_2(w_i_2, w_i) MOVE_128y(YMM_TEMP0, w_i, w_i, 0x08) ;\
wolfSSL	4:1b0d80432c79	930	BLENDy(0xf0, w_i_2, YMM_TEMP0, w_i_2) ;
wolfSSL	4:1b0d80432c79	931
wolfSSL	4:1b0d80432c79	932	#define MOVE_W_to_W_I_15(w_i_15, w_0, w_4) BLENDQy(0x1, w_i_15, w_4, w_0) ;\
wolfSSL	4:1b0d80432c79	933	PERMQy(0x39, w_i_15, w_i_15) ;
wolfSSL	4:1b0d80432c79	934	#define MOVE_W_to_W_I_7(w_i_7, w_8, w_12) BLENDQy(0x1, w_i_7, w_12, w_8) ;\
wolfSSL	4:1b0d80432c79	935	PERMQy(0x39, w_i_7, w_i_7) ;
wolfSSL	4:1b0d80432c79	936	#define MOVE_W_to_W_I_2(w_i_2, w_12) BLENDQy(0xc, w_i_2, w_12, w_i_2) ;\
wolfSSL	4:1b0d80432c79	937	PERMQy(0x0e, w_i_2, w_i_2) ;
wolfSSL	4:1b0d80432c79	938
wolfSSL	4:1b0d80432c79	939
wolfSSL	4:1b0d80432c79	940	#define W_I_16y ymm8
wolfSSL	4:1b0d80432c79	941	#define W_I_15y ymm9
wolfSSL	4:1b0d80432c79	942	#define W_I_7y ymm10
wolfSSL	4:1b0d80432c79	943	#define W_I_2y ymm11
wolfSSL	4:1b0d80432c79	944	#define W_Iy ymm12
wolfSSL	4:1b0d80432c79	945	#define G_TEMPy ymm13
wolfSSL	4:1b0d80432c79	946	#define S_TEMPy ymm14
wolfSSL	4:1b0d80432c79	947	#define YMM_TEMP0 ymm15
wolfSSL	4:1b0d80432c79	948	#define YMM_TEMP0x xmm15
wolfSSL	4:1b0d80432c79	949	#define W_I_TEMPy ymm7
wolfSSL	4:1b0d80432c79	950	#define W_K_TEMPy ymm15
wolfSSL	4:1b0d80432c79	951	#define W_K_TEMPx xmm15
wolfSSL	4:1b0d80432c79	952	#define W_0y ymm12
wolfSSL	4:1b0d80432c79	953	#define W_4y ymm4
wolfSSL	4:1b0d80432c79	954	#define W_8y ymm5
wolfSSL	4:1b0d80432c79	955	#define W_12y ymm6
wolfSSL	4:1b0d80432c79	956
wolfSSL	4:1b0d80432c79	957	#define YMM_REGs
wolfSSL	4:1b0d80432c79	958	/* Registers are saved in Sha512Update/Final */
wolfSSL	4:1b0d80432c79	959	/* "%ymm7","%ymm8","%ymm9","%ymm10","%ymm11","%ymm12","%ymm13","%ymm14","%ymm15"*/
wolfSSL	4:1b0d80432c79	960
wolfSSL	4:1b0d80432c79	961	#define MOVE_15_to_16(w_i_16, w_i_15, w_i_7)\
wolfSSL	4:1b0d80432c79	962	__asm__ volatile("vperm2i128 $0x01, %%"#w_i_15", %%"#w_i_15", %%"#w_i_15" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	963	__asm__ volatile("vpblendd $0x08, %%"#w_i_15", %%"#w_i_7", %%"#w_i_16" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	964	__asm__ volatile("vperm2i128 $0x01, %%"#w_i_7", %%"#w_i_7", %%"#w_i_15" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	965	__asm__ volatile("vpblendd $0x80, %%"#w_i_15", %%"#w_i_16", %%"#w_i_16" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	966	__asm__ volatile("vpshufd $0x93, %%"#w_i_16", %%"#w_i_16" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	967
wolfSSL	4:1b0d80432c79	968	#define MOVE_7_to_15(w_i_15, w_i_7)\
wolfSSL	4:1b0d80432c79	969	__asm__ volatile("vmovdqu %%"#w_i_7", %%"#w_i_15" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	970
wolfSSL	4:1b0d80432c79	971	#define MOVE_I_to_7(w_i_7, w_i)\
wolfSSL	4:1b0d80432c79	972	__asm__ volatile("vperm2i128 $0x01, %%"#w_i", %%"#w_i", %%"#w_i_7" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	973	__asm__ volatile("vpblendd $0x01, %%"#w_i_7", %%"#w_i", %%"#w_i_7" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	974	__asm__ volatile("vpshufd $0x39, %%"#w_i_7", %%"#w_i_7" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	975
wolfSSL	4:1b0d80432c79	976	#define MOVE_I_to_2(w_i_2, w_i)\
wolfSSL	4:1b0d80432c79	977	__asm__ volatile("vperm2i128 $0x01, %%"#w_i", %%"#w_i", %%"#w_i_2" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	978	__asm__ volatile("vpshufd $0x0e, %%"#w_i_2", %%"#w_i_2" ":::YMM_REGs) ;\
wolfSSL	4:1b0d80432c79	979
wolfSSL	4:1b0d80432c79	980	#endif
wolfSSL	4:1b0d80432c79	981
wolfSSL	4:1b0d80432c79	982
wolfSSL	4:1b0d80432c79	983	/* Transform Body */
wolfSSL	4:1b0d80432c79	984	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	985
wolfSSL	4:1b0d80432c79	986	static int Transform_AVX1(Sha512* sha512)
wolfSSL	4:1b0d80432c79	987	{
wolfSSL	4:1b0d80432c79	988	const word64* K = K512;
wolfSSL	4:1b0d80432c79	989	word64 W_X[16+4];
wolfSSL	4:1b0d80432c79	990	word32 j;
wolfSSL	4:1b0d80432c79	991	word64 T[8];
wolfSSL	4:1b0d80432c79	992	/* Copy digest to working vars */
wolfSSL	4:1b0d80432c79	993	XMEMCPY(T, sha512->digest, sizeof(T));
wolfSSL	4:1b0d80432c79	994
wolfSSL	4:1b0d80432c79	995	W_from_buff(W_X, sha512->buffer) ;
wolfSSL	4:1b0d80432c79	996	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	997	Rx_1( 0); Block_0_1(W_X); Rx_2( 0); Block_0_2(W_X); Rx_3( 0); Block_0_3();
wolfSSL	4:1b0d80432c79	998	Rx_1( 1); Block_0_4(); Rx_2( 1); Block_0_5(); Rx_3( 1); Block_0_6(W_X);
wolfSSL	4:1b0d80432c79	999	Rx_1( 2); Block_0_7(W_X); Rx_2( 2); Block_0_8(W_X); Rx_3( 2); Block_0_9();
wolfSSL	4:1b0d80432c79	1000	Rx_1( 3); Block_0_10();Rx_2( 3); Block_0_11();Rx_3( 3); Block_0_12(W_X);
wolfSSL	4:1b0d80432c79	1001
wolfSSL	4:1b0d80432c79	1002	Rx_1( 4); Block_4_1(W_X); Rx_2( 4); Block_4_2(W_X); Rx_3( 4); Block_4_3();
wolfSSL	4:1b0d80432c79	1003	Rx_1( 5); Block_4_4(); Rx_2( 5); Block_4_5(); Rx_3( 5); Block_4_6(W_X);
wolfSSL	4:1b0d80432c79	1004	Rx_1( 6); Block_4_7(W_X); Rx_2( 6); Block_4_8(W_X); Rx_3( 6); Block_4_9();
wolfSSL	4:1b0d80432c79	1005	Rx_1( 7); Block_4_10();Rx_2( 7); Block_4_11();Rx_3( 7); Block_4_12(W_X);
wolfSSL	4:1b0d80432c79	1006
wolfSSL	4:1b0d80432c79	1007	Rx_1( 8); Block_8_1(W_X); Rx_2( 8); Block_8_2(W_X); Rx_3( 8); Block_8_3();
wolfSSL	4:1b0d80432c79	1008	Rx_1( 9); Block_8_4(); Rx_2( 9); Block_8_5(); Rx_3( 9); Block_8_6(W_X);
wolfSSL	4:1b0d80432c79	1009	Rx_1(10); Block_8_7(W_X); Rx_2(10); Block_8_8(W_X); Rx_3(10); Block_8_9();
wolfSSL	4:1b0d80432c79	1010	Rx_1(11); Block_8_10();Rx_2(11); Block_8_11();Rx_3(11); Block_8_12(W_X);
wolfSSL	4:1b0d80432c79	1011
wolfSSL	4:1b0d80432c79	1012	Rx_1(12); Block_12_1(W_X); Rx_2(12); Block_12_2(W_X); Rx_3(12); Block_12_3();
wolfSSL	4:1b0d80432c79	1013	Rx_1(13); Block_12_4(); Rx_2(13); Block_12_5(); Rx_3(13); Block_12_6(W_X);
wolfSSL	4:1b0d80432c79	1014	Rx_1(14); Block_12_7(W_X); Rx_2(14); Block_12_8(W_X); Rx_3(14); Block_12_9();
wolfSSL	4:1b0d80432c79	1015	Rx_1(15); Block_12_10();Rx_2(15); Block_12_11();Rx_3(15); Block_12_12(W_X);
wolfSSL	4:1b0d80432c79	1016	}
wolfSSL	4:1b0d80432c79	1017
wolfSSL	4:1b0d80432c79	1018	/* Add the working vars back into digest */
wolfSSL	4:1b0d80432c79	1019
wolfSSL	4:1b0d80432c79	1020	sha512->digest[0] += a(0);
wolfSSL	4:1b0d80432c79	1021	sha512->digest[1] += b(0);
wolfSSL	4:1b0d80432c79	1022	sha512->digest[2] += c(0);
wolfSSL	4:1b0d80432c79	1023	sha512->digest[3] += d(0);
wolfSSL	4:1b0d80432c79	1024	sha512->digest[4] += e(0);
wolfSSL	4:1b0d80432c79	1025	sha512->digest[5] += f(0);
wolfSSL	4:1b0d80432c79	1026	sha512->digest[6] += g(0);
wolfSSL	4:1b0d80432c79	1027	sha512->digest[7] += h(0);
wolfSSL	4:1b0d80432c79	1028
wolfSSL	4:1b0d80432c79	1029	/* Wipe variables */
wolfSSL	4:1b0d80432c79	1030	#if !defined(HAVE_INTEL_AVX1)&&!defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1031	XMEMSET(W_X, 0, sizeof(word64) * 16);
wolfSSL	4:1b0d80432c79	1032	#endif
wolfSSL	4:1b0d80432c79	1033	XMEMSET(T, 0, sizeof(T));
wolfSSL	4:1b0d80432c79	1034
wolfSSL	4:1b0d80432c79	1035	return 0;
wolfSSL	4:1b0d80432c79	1036	}
wolfSSL	4:1b0d80432c79	1037
wolfSSL	4:1b0d80432c79	1038	#endif
wolfSSL	4:1b0d80432c79	1039
wolfSSL	4:1b0d80432c79	1040	#if defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_AVX1) && defined(HAVE_INTEL_RORX)
wolfSSL	4:1b0d80432c79	1041
wolfSSL	4:1b0d80432c79	1042	static int Transform_AVX1_RORX(Sha512* sha512)
wolfSSL	4:1b0d80432c79	1043	{
wolfSSL	4:1b0d80432c79	1044	const word64* K = K512;
wolfSSL	4:1b0d80432c79	1045	word64 W_X[16+4];
wolfSSL	4:1b0d80432c79	1046	word32 j;
wolfSSL	4:1b0d80432c79	1047	word64 T[8];
wolfSSL	4:1b0d80432c79	1048	/* Copy digest to working vars */
wolfSSL	4:1b0d80432c79	1049	XMEMCPY(T, sha512->digest, sizeof(T));
wolfSSL	4:1b0d80432c79	1050
wolfSSL	4:1b0d80432c79	1051	W_from_buff(W_X, sha512->buffer) ;
wolfSSL	4:1b0d80432c79	1052	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	1053	Rx_RORX_1( 0); Block_0_1(W_X); Rx_RORX_2( 0); Block_0_2(W_X);
wolfSSL	4:1b0d80432c79	1054	Rx_RORX_3( 0); Block_0_3();
wolfSSL	4:1b0d80432c79	1055	Rx_RORX_1( 1); Block_0_4(); Rx_RORX_2( 1); Block_0_5();
wolfSSL	4:1b0d80432c79	1056	Rx_RORX_3( 1); Block_0_6(W_X);
wolfSSL	4:1b0d80432c79	1057	Rx_RORX_1( 2); Block_0_7(W_X); Rx_RORX_2( 2); Block_0_8(W_X);
wolfSSL	4:1b0d80432c79	1058	Rx_RORX_3( 2); Block_0_9();
wolfSSL	4:1b0d80432c79	1059	Rx_RORX_1( 3); Block_0_10();Rx_RORX_2( 3); Block_0_11();
wolfSSL	4:1b0d80432c79	1060	Rx_RORX_3( 3); Block_0_12(W_X);
wolfSSL	4:1b0d80432c79	1061
wolfSSL	4:1b0d80432c79	1062	Rx_RORX_1( 4); Block_4_1(W_X); Rx_RORX_2( 4); Block_4_2(W_X);
wolfSSL	4:1b0d80432c79	1063	Rx_RORX_3( 4); Block_4_3();
wolfSSL	4:1b0d80432c79	1064	Rx_RORX_1( 5); Block_4_4(); Rx_RORX_2( 5); Block_4_5();
wolfSSL	4:1b0d80432c79	1065	Rx_RORX_3( 5); Block_4_6(W_X);
wolfSSL	4:1b0d80432c79	1066	Rx_RORX_1( 6); Block_4_7(W_X); Rx_RORX_2( 6); Block_4_8(W_X);
wolfSSL	4:1b0d80432c79	1067	Rx_RORX_3( 6); Block_4_9();
wolfSSL	4:1b0d80432c79	1068	Rx_RORX_1( 7); Block_4_10();Rx_RORX_2( 7); Block_4_11();
wolfSSL	4:1b0d80432c79	1069	Rx_RORX_3( 7); Block_4_12(W_X);
wolfSSL	4:1b0d80432c79	1070
wolfSSL	4:1b0d80432c79	1071	Rx_RORX_1( 8); Block_8_1(W_X); Rx_RORX_2( 8); Block_8_2(W_X);
wolfSSL	4:1b0d80432c79	1072	Rx_RORX_3( 8); Block_8_3();
wolfSSL	4:1b0d80432c79	1073	Rx_RORX_1( 9); Block_8_4(); Rx_RORX_2( 9); Block_8_5();
wolfSSL	4:1b0d80432c79	1074	Rx_RORX_3( 9); Block_8_6(W_X);
wolfSSL	4:1b0d80432c79	1075	Rx_RORX_1(10); Block_8_7(W_X); Rx_RORX_2(10); Block_8_8(W_X);
wolfSSL	4:1b0d80432c79	1076	Rx_RORX_3(10); Block_8_9();
wolfSSL	4:1b0d80432c79	1077	Rx_RORX_1(11); Block_8_10();Rx_RORX_2(11); Block_8_11();
wolfSSL	4:1b0d80432c79	1078	Rx_RORX_3(11); Block_8_12(W_X);
wolfSSL	4:1b0d80432c79	1079
wolfSSL	4:1b0d80432c79	1080	Rx_RORX_1(12); Block_12_1(W_X); Rx_RORX_2(12); Block_12_2(W_X);
wolfSSL	4:1b0d80432c79	1081	Rx_RORX_3(12); Block_12_3();
wolfSSL	4:1b0d80432c79	1082	Rx_RORX_1(13); Block_12_4(); Rx_RORX_2(13); Block_12_5();
wolfSSL	4:1b0d80432c79	1083	Rx_RORX_3(13); Block_12_6(W_X);
wolfSSL	4:1b0d80432c79	1084	Rx_RORX_1(14); Block_12_7(W_X); Rx_RORX_2(14); Block_12_8(W_X);
wolfSSL	4:1b0d80432c79	1085	Rx_RORX_3(14); Block_12_9();
wolfSSL	4:1b0d80432c79	1086	Rx_RORX_1(15); Block_12_10();Rx_RORX_2(15); Block_12_11();
wolfSSL	4:1b0d80432c79	1087	Rx_RORX_3(15); Block_12_12(W_X);
wolfSSL	4:1b0d80432c79	1088	}
wolfSSL	4:1b0d80432c79	1089	/* Add the working vars back into digest */
wolfSSL	4:1b0d80432c79	1090
wolfSSL	4:1b0d80432c79	1091	sha512->digest[0] += a(0);
wolfSSL	4:1b0d80432c79	1092	sha512->digest[1] += b(0);
wolfSSL	4:1b0d80432c79	1093	sha512->digest[2] += c(0);
wolfSSL	4:1b0d80432c79	1094	sha512->digest[3] += d(0);
wolfSSL	4:1b0d80432c79	1095	sha512->digest[4] += e(0);
wolfSSL	4:1b0d80432c79	1096	sha512->digest[5] += f(0);
wolfSSL	4:1b0d80432c79	1097	sha512->digest[6] += g(0);
wolfSSL	4:1b0d80432c79	1098	sha512->digest[7] += h(0);
wolfSSL	4:1b0d80432c79	1099
wolfSSL	4:1b0d80432c79	1100	/* Wipe variables */
wolfSSL	4:1b0d80432c79	1101	#if !defined(HAVE_INTEL_AVX1)&&!defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1102	XMEMSET(W_X, 0, sizeof(word64) * 16);
wolfSSL	4:1b0d80432c79	1103	#endif
wolfSSL	4:1b0d80432c79	1104	XMEMSET(T, 0, sizeof(T));
wolfSSL	4:1b0d80432c79	1105
wolfSSL	4:1b0d80432c79	1106	return 0;
wolfSSL	4:1b0d80432c79	1107	}
wolfSSL	4:1b0d80432c79	1108	#endif
wolfSSL	4:1b0d80432c79	1109
wolfSSL	4:1b0d80432c79	1110	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1111
wolfSSL	4:1b0d80432c79	1112	#define s0_1y(dest, src) AVX2_S(dest, src, 1);
wolfSSL	4:1b0d80432c79	1113	#define s0_2y(dest, src) AVX2_S(G_TEMPy, src, 8); XORy(dest, G_TEMPy, dest) ;
wolfSSL	4:1b0d80432c79	1114	#define s0_3y(dest, src) AVX2_R(G_TEMPy, src, 7); XORy(dest, G_TEMPy, dest) ;
wolfSSL	4:1b0d80432c79	1115
wolfSSL	4:1b0d80432c79	1116	#define s1_1y(dest, src) AVX2_S(dest, src, 19);
wolfSSL	4:1b0d80432c79	1117	#define s1_2y(dest, src) AVX2_S(G_TEMPy, src, 61); XORy(dest, G_TEMPy, dest) ;
wolfSSL	4:1b0d80432c79	1118	#define s1_3y(dest, src) AVX2_R(G_TEMPy, src, 6); XORy(dest, G_TEMPy, dest) ;
wolfSSL	4:1b0d80432c79	1119
wolfSSL	4:1b0d80432c79	1120	#define s0_y(dest, src) s0_1y(dest, src) ; s0_2y(dest, src) ; s0_3y(dest, src)
wolfSSL	4:1b0d80432c79	1121	#define s1_y(dest, src) s1_1y(dest, src) ; s1_2y(dest, src) ; s1_3y(dest, src)
wolfSSL	4:1b0d80432c79	1122
wolfSSL	4:1b0d80432c79	1123	#define blk384(i) (W[i] = sha384->buffer[i])
wolfSSL	4:1b0d80432c79	1124
wolfSSL	4:1b0d80432c79	1125
wolfSSL	4:1b0d80432c79	1126	#define Block_Y_xx_1(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1127	MOVE_W_to_W_I_15(W_I_15y, w_0, w_4) ;\
wolfSSL	4:1b0d80432c79	1128	MOVE_W_to_W_I_7 (W_I_7y, w_8, w_12) ;\
wolfSSL	4:1b0d80432c79	1129	MOVE_W_to_W_I_2 (W_I_2y, w_12) ;\
wolfSSL	4:1b0d80432c79	1130
wolfSSL	4:1b0d80432c79	1131	#define Block_Y_xx_2(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1132	s0_1y (YMM_TEMP0, W_I_15y) ;\
wolfSSL	4:1b0d80432c79	1133
wolfSSL	4:1b0d80432c79	1134	#define Block_Y_xx_3(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1135	s0_2y (YMM_TEMP0, W_I_15y) ;\
wolfSSL	4:1b0d80432c79	1136
wolfSSL	4:1b0d80432c79	1137	#define Block_Y_xx_4(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1138	s0_3y (YMM_TEMP0, W_I_15y) ;\
wolfSSL	4:1b0d80432c79	1139
wolfSSL	4:1b0d80432c79	1140	#define Block_Y_xx_5(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1141	ADDy(W_I_TEMPy, w_0, YMM_TEMP0) ;\
wolfSSL	4:1b0d80432c79	1142
wolfSSL	4:1b0d80432c79	1143	#define Block_Y_xx_6(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1144	ADDy(W_I_TEMPy, W_I_TEMPy, W_I_7y) ;\
wolfSSL	4:1b0d80432c79	1145	s1_1y (YMM_TEMP0, W_I_2y) ;\
wolfSSL	4:1b0d80432c79	1146
wolfSSL	4:1b0d80432c79	1147	#define Block_Y_xx_7(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1148	s1_2y (YMM_TEMP0, W_I_2y) ;\
wolfSSL	4:1b0d80432c79	1149
wolfSSL	4:1b0d80432c79	1150	#define Block_Y_xx_8(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1151	s1_3y (YMM_TEMP0, W_I_2y) ;\
wolfSSL	4:1b0d80432c79	1152	ADDy(w_0, W_I_TEMPy, YMM_TEMP0) ;\
wolfSSL	4:1b0d80432c79	1153
wolfSSL	4:1b0d80432c79	1154	#define Block_Y_xx_9(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1155	FEEDBACK1_to_W_I_2(W_I_2y, w_0) ;\
wolfSSL	4:1b0d80432c79	1156
wolfSSL	4:1b0d80432c79	1157	#define Block_Y_xx_10(i, w_0, w_4, w_8, w_12) \
wolfSSL	4:1b0d80432c79	1158	s1_1y (YMM_TEMP0, W_I_2y) ;\
wolfSSL	4:1b0d80432c79	1159
wolfSSL	4:1b0d80432c79	1160	#define Block_Y_xx_11(i, w_0, w_4, w_8, w_12) \
wolfSSL	4:1b0d80432c79	1161	s1_2y (YMM_TEMP0, W_I_2y) ;\
wolfSSL	4:1b0d80432c79	1162
wolfSSL	4:1b0d80432c79	1163	#define Block_Y_xx_12(i, w_0, w_4, w_8, w_12)\
wolfSSL	4:1b0d80432c79	1164	s1_3y (YMM_TEMP0, W_I_2y) ;\
wolfSSL	4:1b0d80432c79	1165	ADDy(w_0, W_I_TEMPy, YMM_TEMP0) ;\
wolfSSL	4:1b0d80432c79	1166	MOVE_to_MEMy(w,0, w_4) ;\
wolfSSL	4:1b0d80432c79	1167
wolfSSL	4:1b0d80432c79	1168
wolfSSL	4:1b0d80432c79	1169	static INLINE void Block_Y_0_1(void) { Block_Y_xx_1(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1170	static INLINE void Block_Y_0_2(void) { Block_Y_xx_2(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1171	static INLINE void Block_Y_0_3(void) { Block_Y_xx_3(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1172	static INLINE void Block_Y_0_4(void) { Block_Y_xx_4(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1173	static INLINE void Block_Y_0_5(void) { Block_Y_xx_5(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1174	static INLINE void Block_Y_0_6(void) { Block_Y_xx_6(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1175	static INLINE void Block_Y_0_7(void) { Block_Y_xx_7(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1176	static INLINE void Block_Y_0_8(void) { Block_Y_xx_8(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1177	static INLINE void Block_Y_0_9(void) { Block_Y_xx_9(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1178	static INLINE void Block_Y_0_10(void){ Block_Y_xx_10(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1179	static INLINE void Block_Y_0_11(void){ Block_Y_xx_11(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1180	static INLINE void Block_Y_0_12(word64 *w){ Block_Y_xx_12(0, W_0y, W_4y, W_8y, W_12y) ; }
wolfSSL	4:1b0d80432c79	1181
wolfSSL	4:1b0d80432c79	1182	static INLINE void Block_Y_4_1(void) { Block_Y_xx_1(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1183	static INLINE void Block_Y_4_2(void) { Block_Y_xx_2(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1184	static INLINE void Block_Y_4_3(void) { Block_Y_xx_3(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1185	static INLINE void Block_Y_4_4(void) { Block_Y_xx_4(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1186	static INLINE void Block_Y_4_5(void) { Block_Y_xx_5(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1187	static INLINE void Block_Y_4_6(void) { Block_Y_xx_6(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1188	static INLINE void Block_Y_4_7(void) { Block_Y_xx_7(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1189	static INLINE void Block_Y_4_8(void) { Block_Y_xx_8(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1190	static INLINE void Block_Y_4_9(void) { Block_Y_xx_9(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1191	static INLINE void Block_Y_4_10(void) { Block_Y_xx_10(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1192	static INLINE void Block_Y_4_11(void) { Block_Y_xx_11(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1193	static INLINE void Block_Y_4_12(word64 *w) { Block_Y_xx_12(4, W_4y, W_8y, W_12y, W_0y) ; }
wolfSSL	4:1b0d80432c79	1194
wolfSSL	4:1b0d80432c79	1195	static INLINE void Block_Y_8_1(void) { Block_Y_xx_1(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1196	static INLINE void Block_Y_8_2(void) { Block_Y_xx_2(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1197	static INLINE void Block_Y_8_3(void) { Block_Y_xx_3(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1198	static INLINE void Block_Y_8_4(void) { Block_Y_xx_4(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1199	static INLINE void Block_Y_8_5(void) { Block_Y_xx_5(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1200	static INLINE void Block_Y_8_6(void) { Block_Y_xx_6(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1201	static INLINE void Block_Y_8_7(void) { Block_Y_xx_7(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1202	static INLINE void Block_Y_8_8(void) { Block_Y_xx_8(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1203	static INLINE void Block_Y_8_9(void) { Block_Y_xx_9(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1204	static INLINE void Block_Y_8_10(void) { Block_Y_xx_10(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1205	static INLINE void Block_Y_8_11(void) { Block_Y_xx_11(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1206	static INLINE void Block_Y_8_12(word64 *w) { Block_Y_xx_12(8, W_8y, W_12y, W_0y, W_4y) ; }
wolfSSL	4:1b0d80432c79	1207
wolfSSL	4:1b0d80432c79	1208	static INLINE void Block_Y_12_1(void) { Block_Y_xx_1(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1209	static INLINE void Block_Y_12_2(void) { Block_Y_xx_2(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1210	static INLINE void Block_Y_12_3(void) { Block_Y_xx_3(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1211	static INLINE void Block_Y_12_4(void) { Block_Y_xx_4(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1212	static INLINE void Block_Y_12_5(void) { Block_Y_xx_5(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1213	static INLINE void Block_Y_12_6(void) { Block_Y_xx_6(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1214	static INLINE void Block_Y_12_7(void) { Block_Y_xx_7(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1215	static INLINE void Block_Y_12_8(void) { Block_Y_xx_8(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1216	static INLINE void Block_Y_12_9(void) { Block_Y_xx_9(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1217	static INLINE void Block_Y_12_10(void) { Block_Y_xx_10(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1218	static INLINE void Block_Y_12_11(void) { Block_Y_xx_11(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1219	static INLINE void Block_Y_12_12(word64 *w) { Block_Y_xx_12(12, W_12y, W_0y, W_4y, W_8y) ; }
wolfSSL	4:1b0d80432c79	1220
wolfSSL	4:1b0d80432c79	1221
wolfSSL	4:1b0d80432c79	1222	static int Transform_AVX2(Sha512* sha512)
wolfSSL	4:1b0d80432c79	1223	{
wolfSSL	4:1b0d80432c79	1224	const word64* K = K512;
wolfSSL	4:1b0d80432c79	1225	word64 w[4] ;
wolfSSL	4:1b0d80432c79	1226	word32 j /, k/;
wolfSSL	4:1b0d80432c79	1227	word64 T[8];
wolfSSL	4:1b0d80432c79	1228	/* Copy digest to working vars */
wolfSSL	4:1b0d80432c79	1229	XMEMCPY(T, sha512->digest, sizeof(T));
wolfSSL	4:1b0d80432c79	1230
wolfSSL	4:1b0d80432c79	1231	W_from_buff_Y(sha512->buffer) ;
wolfSSL	4:1b0d80432c79	1232	MOVE_to_MEMy(w,0, W_0y) ;
wolfSSL	4:1b0d80432c79	1233	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	1234	Ry_1( 0, w[0]); Block_Y_0_1(); Ry_2( 0, w[0]); Block_Y_0_2();
wolfSSL	4:1b0d80432c79	1235	Ry_3( 0, w[0]); Block_Y_0_3();
wolfSSL	4:1b0d80432c79	1236	Ry_1( 1, w[1]); Block_Y_0_4(); Ry_2( 1, w[1]); Block_Y_0_5();
wolfSSL	4:1b0d80432c79	1237	Ry_3( 1, w[1]); Block_Y_0_6();
wolfSSL	4:1b0d80432c79	1238	Ry_1( 2, w[2]); Block_Y_0_7(); Ry_2( 2, w[2]); Block_Y_0_8();
wolfSSL	4:1b0d80432c79	1239	Ry_3( 2, w[2]); Block_Y_0_9();
wolfSSL	4:1b0d80432c79	1240	Ry_1( 3, w[3]); Block_Y_0_10();Ry_2( 3, w[3]); Block_Y_0_11();
wolfSSL	4:1b0d80432c79	1241	Ry_3( 3, w[3]); Block_Y_0_12(w);
wolfSSL	4:1b0d80432c79	1242
wolfSSL	4:1b0d80432c79	1243	Ry_1( 4, w[0]); Block_Y_4_1(); Ry_2( 4, w[0]); Block_Y_4_2();
wolfSSL	4:1b0d80432c79	1244	Ry_3( 4, w[0]); Block_Y_4_3();
wolfSSL	4:1b0d80432c79	1245	Ry_1( 5, w[1]); Block_Y_4_4(); Ry_2( 5, w[1]); Block_Y_4_5();
wolfSSL	4:1b0d80432c79	1246	Ry_3( 5, w[1]); Block_Y_4_6();
wolfSSL	4:1b0d80432c79	1247	Ry_1( 6, w[2]); Block_Y_4_7(); Ry_2( 6, w[2]); Block_Y_4_8();
wolfSSL	4:1b0d80432c79	1248	Ry_3( 6, w[2]); Block_Y_4_9();
wolfSSL	4:1b0d80432c79	1249	Ry_1( 7, w[3]); Block_Y_4_10(); Ry_2( 7, w[3]);Block_Y_4_11();
wolfSSL	4:1b0d80432c79	1250	Ry_3( 7, w[3]);Block_Y_4_12(w);
wolfSSL	4:1b0d80432c79	1251
wolfSSL	4:1b0d80432c79	1252	Ry_1( 8, w[0]); Block_Y_8_1(); Ry_2( 8, w[0]); Block_Y_8_2();
wolfSSL	4:1b0d80432c79	1253	Ry_3( 8, w[0]); Block_Y_8_3();
wolfSSL	4:1b0d80432c79	1254	Ry_1( 9, w[1]); Block_Y_8_4(); Ry_2( 9, w[1]); Block_Y_8_5();
wolfSSL	4:1b0d80432c79	1255	Ry_3( 9, w[1]); Block_Y_8_6();
wolfSSL	4:1b0d80432c79	1256	Ry_1(10, w[2]); Block_Y_8_7(); Ry_2(10, w[2]); Block_Y_8_8();
wolfSSL	4:1b0d80432c79	1257	Ry_3(10, w[2]); Block_Y_8_9();
wolfSSL	4:1b0d80432c79	1258	Ry_1(11, w[3]); Block_Y_8_10();Ry_2(11, w[3]); Block_Y_8_11();
wolfSSL	4:1b0d80432c79	1259	Ry_3(11, w[3]); Block_Y_8_12(w);
wolfSSL	4:1b0d80432c79	1260
wolfSSL	4:1b0d80432c79	1261	Ry_1(12, w[0]); Block_Y_12_1(); Ry_2(12, w[0]); Block_Y_12_2();
wolfSSL	4:1b0d80432c79	1262	Ry_3(12, w[0]); Block_Y_12_3();
wolfSSL	4:1b0d80432c79	1263	Ry_1(13, w[1]); Block_Y_12_4(); Ry_2(13, w[1]); Block_Y_12_5();
wolfSSL	4:1b0d80432c79	1264	Ry_3(13, w[1]); Block_Y_12_6();
wolfSSL	4:1b0d80432c79	1265	Ry_1(14, w[2]); Block_Y_12_7(); Ry_2(14, w[2]); Block_Y_12_8();
wolfSSL	4:1b0d80432c79	1266	Ry_3(14, w[2]); Block_Y_12_9();
wolfSSL	4:1b0d80432c79	1267	Ry_1(15, w[3]); Block_Y_12_10();Ry_2(15, w[3]); Block_Y_12_11();
wolfSSL	4:1b0d80432c79	1268	Ry_3(15, w[3]);Block_Y_12_12(w);
wolfSSL	4:1b0d80432c79	1269	}
wolfSSL	4:1b0d80432c79	1270
wolfSSL	4:1b0d80432c79	1271	/* Add the working vars back into digest */
wolfSSL	4:1b0d80432c79	1272
wolfSSL	4:1b0d80432c79	1273	sha512->digest[0] += a(0);
wolfSSL	4:1b0d80432c79	1274	sha512->digest[1] += b(0);
wolfSSL	4:1b0d80432c79	1275	sha512->digest[2] += c(0);
wolfSSL	4:1b0d80432c79	1276	sha512->digest[3] += d(0);
wolfSSL	4:1b0d80432c79	1277	sha512->digest[4] += e(0);
wolfSSL	4:1b0d80432c79	1278	sha512->digest[5] += f(0);
wolfSSL	4:1b0d80432c79	1279	sha512->digest[6] += g(0);
wolfSSL	4:1b0d80432c79	1280	sha512->digest[7] += h(0);
wolfSSL	4:1b0d80432c79	1281
wolfSSL	4:1b0d80432c79	1282	/* Wipe variables */
wolfSSL	4:1b0d80432c79	1283	#if !defined(HAVE_INTEL_AVX1)&&!defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1284	XMEMSET(W, 0, sizeof(word64) * 16);
wolfSSL	4:1b0d80432c79	1285	#endif
wolfSSL	4:1b0d80432c79	1286	XMEMSET(T, 0, sizeof(T));
wolfSSL	4:1b0d80432c79	1287
wolfSSL	4:1b0d80432c79	1288	return 0;
wolfSSL	4:1b0d80432c79	1289	}
wolfSSL	4:1b0d80432c79	1290
wolfSSL	4:1b0d80432c79	1291	#endif
wolfSSL	4:1b0d80432c79	1292
wolfSSL	4:1b0d80432c79	1293
wolfSSL	4:1b0d80432c79	1294	#ifdef WOLFSSL_SHA384
wolfSSL	4:1b0d80432c79	1295
wolfSSL	4:1b0d80432c79	1296	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1297
wolfSSL	4:1b0d80432c79	1298	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	1299	static int Transform384_AVX1(Sha384 *sha384) ;
wolfSSL	4:1b0d80432c79	1300	#endif
wolfSSL	4:1b0d80432c79	1301	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1302	static int Transform384_AVX2(Sha384 *sha384) ;
wolfSSL	4:1b0d80432c79	1303	#endif
wolfSSL	4:1b0d80432c79	1304
wolfSSL	4:1b0d80432c79	1305	#if defined(HAVE_INTEL_AVX1) && defined(HAVE_INTEL_AVX2) &&defined(HAVE_INTEL_RORX)
wolfSSL	4:1b0d80432c79	1306	static int Transform384_AVX1_RORX(Sha384 *sha384) ;
wolfSSL	4:1b0d80432c79	1307	#endif
wolfSSL	4:1b0d80432c79	1308
wolfSSL	4:1b0d80432c79	1309	static int _Transform384(Sha384 *sha384) ;
wolfSSL	4:1b0d80432c79	1310	static int (Transform384_p)(Sha384 sha384) = _Transform384 ;
wolfSSL	4:1b0d80432c79	1311
wolfSSL	4:1b0d80432c79	1312	#define Transform384(sha384) (*Transform384_p)(sha384)
wolfSSL	4:1b0d80432c79	1313	static void set_Transform384(void) {
wolfSSL	4:1b0d80432c79	1314	if(set_cpuid_flags(CHECK_SHA384))return ;
wolfSSL	4:1b0d80432c79	1315
wolfSSL	4:1b0d80432c79	1316	#if defined(HAVE_INTEL_AVX1) && !defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1317	Transform384_p = ((IS_INTEL_AVX1) ? Transform384_AVX1 : _Transform384) ;
wolfSSL	4:1b0d80432c79	1318	#elif defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1319	#if defined(HAVE_INTEL_AVX1) && defined(HAVE_INTEL_RORX)
wolfSSL	4:1b0d80432c79	1320	if(IS_INTEL_AVX2 && IS_INTEL_BMI2) { Transform384_p = Transform384_AVX1_RORX ; return ; }
wolfSSL	4:1b0d80432c79	1321	#endif
wolfSSL	4:1b0d80432c79	1322	if(IS_INTEL_AVX2) { Transform384_p = Transform384_AVX2 ; return ; }
wolfSSL	4:1b0d80432c79	1323	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	1324	Transform384_p = ((IS_INTEL_AVX1) ? Transform384_AVX1 : _Transform384) ;
wolfSSL	4:1b0d80432c79	1325	#endif
wolfSSL	4:1b0d80432c79	1326	#else
wolfSSL	4:1b0d80432c79	1327	Transform384_p = ((IS_INTEL_AVX1) ? Transform384_AVX1 : _Transform384) ;
wolfSSL	4:1b0d80432c79	1328	#endif
wolfSSL	4:1b0d80432c79	1329	}
wolfSSL	4:1b0d80432c79	1330
wolfSSL	4:1b0d80432c79	1331	#else
wolfSSL	4:1b0d80432c79	1332	#define Transform384(sha512) _Transform384(sha512)
wolfSSL	4:1b0d80432c79	1333	#endif
wolfSSL	4:1b0d80432c79	1334
wolfSSL	4:1b0d80432c79	1335	int wc_InitSha384(Sha384* sha384)
wolfSSL	4:1b0d80432c79	1336	{
wolfSSL	4:1b0d80432c79	1337	sha384->digest[0] = W64LIT(0xcbbb9d5dc1059ed8);
wolfSSL	4:1b0d80432c79	1338	sha384->digest[1] = W64LIT(0x629a292a367cd507);
wolfSSL	4:1b0d80432c79	1339	sha384->digest[2] = W64LIT(0x9159015a3070dd17);
wolfSSL	4:1b0d80432c79	1340	sha384->digest[3] = W64LIT(0x152fecd8f70e5939);
wolfSSL	4:1b0d80432c79	1341	sha384->digest[4] = W64LIT(0x67332667ffc00b31);
wolfSSL	4:1b0d80432c79	1342	sha384->digest[5] = W64LIT(0x8eb44a8768581511);
wolfSSL	4:1b0d80432c79	1343	sha384->digest[6] = W64LIT(0xdb0c2e0d64f98fa7);
wolfSSL	4:1b0d80432c79	1344	sha384->digest[7] = W64LIT(0x47b5481dbefa4fa4);
wolfSSL	4:1b0d80432c79	1345
wolfSSL	4:1b0d80432c79	1346	sha384->buffLen = 0;
wolfSSL	4:1b0d80432c79	1347	sha384->loLen = 0;
wolfSSL	4:1b0d80432c79	1348	sha384->hiLen = 0;
wolfSSL	4:1b0d80432c79	1349
wolfSSL	4:1b0d80432c79	1350	#if defined(HAVE_INTEL_AVX1)\|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1351	set_Transform384() ;
wolfSSL	4:1b0d80432c79	1352	#endif
wolfSSL	4:1b0d80432c79	1353
wolfSSL	4:1b0d80432c79	1354	return 0;
wolfSSL	4:1b0d80432c79	1355	}
wolfSSL	4:1b0d80432c79	1356
wolfSSL	4:1b0d80432c79	1357	static int _Transform384(Sha384* sha384)
wolfSSL	4:1b0d80432c79	1358	{
wolfSSL	4:1b0d80432c79	1359	const word64* K = K512;
wolfSSL	4:1b0d80432c79	1360
wolfSSL	4:1b0d80432c79	1361	word32 j;
wolfSSL	4:1b0d80432c79	1362	word64 T[8];
wolfSSL	4:1b0d80432c79	1363
wolfSSL	4:1b0d80432c79	1364	#ifdef WOLFSSL_SMALL_STACK
wolfSSL	4:1b0d80432c79	1365	word64* W;
wolfSSL	4:1b0d80432c79	1366
wolfSSL	4:1b0d80432c79	1367	W = (word64) XMALLOC(sizeof(word64) 16, NULL, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	4:1b0d80432c79	1368	if (W == NULL)
wolfSSL	4:1b0d80432c79	1369	return MEMORY_E;
wolfSSL	4:1b0d80432c79	1370	#else
wolfSSL	4:1b0d80432c79	1371	word64 W[16];
wolfSSL	4:1b0d80432c79	1372	#endif
wolfSSL	4:1b0d80432c79	1373
wolfSSL	4:1b0d80432c79	1374	/* Copy digest to working vars */
wolfSSL	4:1b0d80432c79	1375	XMEMCPY(T, sha384->digest, sizeof(T));
wolfSSL	4:1b0d80432c79	1376
wolfSSL	4:1b0d80432c79	1377	#ifdef USE_SLOW_SHA2
wolfSSL	4:1b0d80432c79	1378	/* over twice as small, but 50% slower */
wolfSSL	4:1b0d80432c79	1379	/* 80 operations, not unrolled */
wolfSSL	4:1b0d80432c79	1380	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	1381	int m;
wolfSSL	4:1b0d80432c79	1382	for (m = 0; m < 16; m++) { /* braces needed for macros {} */
wolfSSL	4:1b0d80432c79	1383	R2(m);
wolfSSL	4:1b0d80432c79	1384	}
wolfSSL	4:1b0d80432c79	1385	}
wolfSSL	4:1b0d80432c79	1386	#else
wolfSSL	4:1b0d80432c79	1387	/* 80 operations, partially loop unrolled */
wolfSSL	4:1b0d80432c79	1388	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	1389	R2( 0); R2( 1); R2( 2); R2( 3);
wolfSSL	4:1b0d80432c79	1390	R2( 4); R2( 5); R2( 6); R2( 7);
wolfSSL	4:1b0d80432c79	1391	R2( 8); R2( 9); R2(10); R2(11);
wolfSSL	4:1b0d80432c79	1392	R2(12); R2(13); R2(14); R2(15);
wolfSSL	4:1b0d80432c79	1393	}
wolfSSL	4:1b0d80432c79	1394	#endif /* USE_SLOW_SHA2 */
wolfSSL	4:1b0d80432c79	1395
wolfSSL	4:1b0d80432c79	1396	/* Add the working vars back into digest */
wolfSSL	4:1b0d80432c79	1397
wolfSSL	4:1b0d80432c79	1398	sha384->digest[0] += a(0);
wolfSSL	4:1b0d80432c79	1399	sha384->digest[1] += b(0);
wolfSSL	4:1b0d80432c79	1400	sha384->digest[2] += c(0);
wolfSSL	4:1b0d80432c79	1401	sha384->digest[3] += d(0);
wolfSSL	4:1b0d80432c79	1402	sha384->digest[4] += e(0);
wolfSSL	4:1b0d80432c79	1403	sha384->digest[5] += f(0);
wolfSSL	4:1b0d80432c79	1404	sha384->digest[6] += g(0);
wolfSSL	4:1b0d80432c79	1405	sha384->digest[7] += h(0);
wolfSSL	4:1b0d80432c79	1406
wolfSSL	4:1b0d80432c79	1407	/* Wipe variables */
wolfSSL	4:1b0d80432c79	1408	XMEMSET(W, 0, sizeof(word64) * 16);
wolfSSL	4:1b0d80432c79	1409	XMEMSET(T, 0, sizeof(T));
wolfSSL	4:1b0d80432c79	1410
wolfSSL	4:1b0d80432c79	1411	#ifdef WOLFSSL_SMALL_STACK
wolfSSL	4:1b0d80432c79	1412	XFREE(W, NULL, DYNAMIC_TYPE_TMP_BUFFER);
wolfSSL	4:1b0d80432c79	1413	#endif
wolfSSL	4:1b0d80432c79	1414
wolfSSL	4:1b0d80432c79	1415	return 0;
wolfSSL	4:1b0d80432c79	1416	}
wolfSSL	4:1b0d80432c79	1417
wolfSSL	4:1b0d80432c79	1418	static INLINE void AddLength384(Sha384* sha384, word32 len)
wolfSSL	4:1b0d80432c79	1419	{
wolfSSL	4:1b0d80432c79	1420	word32 tmp = sha384->loLen;
wolfSSL	4:1b0d80432c79	1421	if ( (sha384->loLen += len) < tmp)
wolfSSL	4:1b0d80432c79	1422	sha384->hiLen++; /* carry low to high */
wolfSSL	4:1b0d80432c79	1423	}
wolfSSL	4:1b0d80432c79	1424
wolfSSL	4:1b0d80432c79	1425	int wc_Sha384Update(Sha384* sha384, const byte* data, word32 len)
wolfSSL	4:1b0d80432c79	1426	{
wolfSSL	4:1b0d80432c79	1427	/* do block size increments */
wolfSSL	4:1b0d80432c79	1428	byte* local = (byte*)sha384->buffer;
wolfSSL	4:1b0d80432c79	1429
wolfSSL	4:1b0d80432c79	1430	SAVE_XMM_YMM ; /* for Intel AVX */
wolfSSL	4:1b0d80432c79	1431
wolfSSL	4:1b0d80432c79	1432	while (len) {
wolfSSL	4:1b0d80432c79	1433	word32 add = min(len, SHA384_BLOCK_SIZE - sha384->buffLen);
wolfSSL	4:1b0d80432c79	1434	XMEMCPY(&local[sha384->buffLen], data, add);
wolfSSL	4:1b0d80432c79	1435
wolfSSL	4:1b0d80432c79	1436	sha384->buffLen += add;
wolfSSL	4:1b0d80432c79	1437	data += add;
wolfSSL	4:1b0d80432c79	1438	len -= add;
wolfSSL	4:1b0d80432c79	1439
wolfSSL	4:1b0d80432c79	1440	if (sha384->buffLen == SHA384_BLOCK_SIZE) {
wolfSSL	4:1b0d80432c79	1441	int ret;
wolfSSL	4:1b0d80432c79	1442
wolfSSL	4:1b0d80432c79	1443	#if defined(LITTLE_ENDIAN_ORDER)
wolfSSL	4:1b0d80432c79	1444	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1445	if(!IS_INTEL_AVX1 && !IS_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1446	#endif
wolfSSL	4:1b0d80432c79	1447	ByteReverseWords64(sha384->buffer, sha384->buffer,
wolfSSL	4:1b0d80432c79	1448	SHA384_BLOCK_SIZE);
wolfSSL	4:1b0d80432c79	1449	#endif
wolfSSL	4:1b0d80432c79	1450	ret = Transform384(sha384);
wolfSSL	4:1b0d80432c79	1451	if (ret != 0)
wolfSSL	4:1b0d80432c79	1452	return ret;
wolfSSL	4:1b0d80432c79	1453
wolfSSL	4:1b0d80432c79	1454	AddLength384(sha384, SHA384_BLOCK_SIZE);
wolfSSL	4:1b0d80432c79	1455	sha384->buffLen = 0;
wolfSSL	4:1b0d80432c79	1456	}
wolfSSL	4:1b0d80432c79	1457	}
wolfSSL	4:1b0d80432c79	1458	return 0;
wolfSSL	4:1b0d80432c79	1459	}
wolfSSL	4:1b0d80432c79	1460
wolfSSL	4:1b0d80432c79	1461
wolfSSL	4:1b0d80432c79	1462	int wc_Sha384Final(Sha384* sha384, byte* hash)
wolfSSL	4:1b0d80432c79	1463	{
wolfSSL	4:1b0d80432c79	1464	byte* local = (byte*)sha384->buffer;
wolfSSL	4:1b0d80432c79	1465	int ret;
wolfSSL	4:1b0d80432c79	1466
wolfSSL	4:1b0d80432c79	1467	SAVE_XMM_YMM ; /* for Intel AVX */
wolfSSL	4:1b0d80432c79	1468	AddLength384(sha384, sha384->buffLen); /* before adding pads */
wolfSSL	4:1b0d80432c79	1469
wolfSSL	4:1b0d80432c79	1470	local[sha384->buffLen++] = 0x80; /* add 1 */
wolfSSL	4:1b0d80432c79	1471
wolfSSL	4:1b0d80432c79	1472	/* pad with zeros */
wolfSSL	4:1b0d80432c79	1473	if (sha384->buffLen > SHA384_PAD_SIZE) {
wolfSSL	4:1b0d80432c79	1474	XMEMSET(&local[sha384->buffLen], 0, SHA384_BLOCK_SIZE -sha384->buffLen);
wolfSSL	4:1b0d80432c79	1475	sha384->buffLen += SHA384_BLOCK_SIZE - sha384->buffLen;
wolfSSL	4:1b0d80432c79	1476
wolfSSL	4:1b0d80432c79	1477	#if defined(LITTLE_ENDIAN_ORDER)
wolfSSL	4:1b0d80432c79	1478	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1479	if(!IS_INTEL_AVX1 && !IS_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1480	#endif
wolfSSL	4:1b0d80432c79	1481	ByteReverseWords64(sha384->buffer, sha384->buffer,
wolfSSL	4:1b0d80432c79	1482	SHA384_BLOCK_SIZE);
wolfSSL	4:1b0d80432c79	1483	#endif
wolfSSL	4:1b0d80432c79	1484	ret = Transform384(sha384);
wolfSSL	4:1b0d80432c79	1485	if (ret != 0)
wolfSSL	4:1b0d80432c79	1486	return ret;
wolfSSL	4:1b0d80432c79	1487
wolfSSL	4:1b0d80432c79	1488	sha384->buffLen = 0;
wolfSSL	4:1b0d80432c79	1489	}
wolfSSL	4:1b0d80432c79	1490	XMEMSET(&local[sha384->buffLen], 0, SHA384_PAD_SIZE - sha384->buffLen);
wolfSSL	4:1b0d80432c79	1491
wolfSSL	4:1b0d80432c79	1492	/* put lengths in bits */
wolfSSL	4:1b0d80432c79	1493	sha384->hiLen = (sha384->loLen >> (8*sizeof(sha384->loLen) - 3)) +
wolfSSL	4:1b0d80432c79	1494	(sha384->hiLen << 3);
wolfSSL	4:1b0d80432c79	1495	sha384->loLen = sha384->loLen << 3;
wolfSSL	4:1b0d80432c79	1496
wolfSSL	4:1b0d80432c79	1497	/* store lengths */
wolfSSL	4:1b0d80432c79	1498	#if defined(LITTLE_ENDIAN_ORDER)
wolfSSL	4:1b0d80432c79	1499	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1500	if(!IS_INTEL_AVX1 && !IS_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1501	#endif
wolfSSL	4:1b0d80432c79	1502	ByteReverseWords64(sha384->buffer, sha384->buffer,
wolfSSL	4:1b0d80432c79	1503	SHA384_BLOCK_SIZE);
wolfSSL	4:1b0d80432c79	1504	#endif
wolfSSL	4:1b0d80432c79	1505	/* ! length ordering dependent on digest endian type ! */
wolfSSL	4:1b0d80432c79	1506	sha384->buffer[SHA384_BLOCK_SIZE / sizeof(word64) - 2] = sha384->hiLen;
wolfSSL	4:1b0d80432c79	1507	sha384->buffer[SHA384_BLOCK_SIZE / sizeof(word64) - 1] = sha384->loLen;
wolfSSL	4:1b0d80432c79	1508	#if defined(HAVE_INTEL_AVX1) \|\| defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1509	if(IS_INTEL_AVX1 \|\| IS_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1510	ByteReverseWords64(&(sha384->buffer[SHA384_BLOCK_SIZE / sizeof(word64) - 2]),
wolfSSL	4:1b0d80432c79	1511	&(sha384->buffer[SHA384_BLOCK_SIZE / sizeof(word64) - 2]),
wolfSSL	4:1b0d80432c79	1512	SHA384_BLOCK_SIZE - SHA384_PAD_SIZE);
wolfSSL	4:1b0d80432c79	1513	#endif
wolfSSL	4:1b0d80432c79	1514	ret = Transform384(sha384);
wolfSSL	4:1b0d80432c79	1515	if (ret != 0)
wolfSSL	4:1b0d80432c79	1516	return ret;
wolfSSL	4:1b0d80432c79	1517
wolfSSL	4:1b0d80432c79	1518	#ifdef LITTLE_ENDIAN_ORDER
wolfSSL	4:1b0d80432c79	1519	ByteReverseWords64(sha384->digest, sha384->digest, SHA384_DIGEST_SIZE);
wolfSSL	4:1b0d80432c79	1520	#endif
wolfSSL	4:1b0d80432c79	1521	XMEMCPY(hash, sha384->digest, SHA384_DIGEST_SIZE);
wolfSSL	4:1b0d80432c79	1522
wolfSSL	4:1b0d80432c79	1523	return wc_InitSha384(sha384); /* reset state */
wolfSSL	4:1b0d80432c79	1524	}
wolfSSL	4:1b0d80432c79	1525
wolfSSL	4:1b0d80432c79	1526
wolfSSL	4:1b0d80432c79	1527
wolfSSL	4:1b0d80432c79	1528	#if defined(HAVE_INTEL_AVX1)
wolfSSL	4:1b0d80432c79	1529
wolfSSL	4:1b0d80432c79	1530	static int Transform384_AVX1(Sha384* sha384)
wolfSSL	4:1b0d80432c79	1531	{
wolfSSL	4:1b0d80432c79	1532	const word64* K = K512;
wolfSSL	4:1b0d80432c79	1533	word64 W_X[16+4];
wolfSSL	4:1b0d80432c79	1534	word32 j;
wolfSSL	4:1b0d80432c79	1535	word64 T[8];
wolfSSL	4:1b0d80432c79	1536
wolfSSL	4:1b0d80432c79	1537	/* Copy digest to working vars */
wolfSSL	4:1b0d80432c79	1538	XMEMCPY(T, sha384->digest, sizeof(T));
wolfSSL	4:1b0d80432c79	1539	W_from_buff(W_X, sha384->buffer) ;
wolfSSL	4:1b0d80432c79	1540	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	1541	Rx_1( 0); Block_0_1(W_X); Rx_2( 0); Block_0_2(W_X); Rx_3( 0); Block_0_3();
wolfSSL	4:1b0d80432c79	1542	Rx_1( 1); Block_0_4(); Rx_2( 1); Block_0_5(); Rx_3( 1); Block_0_6(W_X);
wolfSSL	4:1b0d80432c79	1543	Rx_1( 2); Block_0_7(W_X); Rx_2( 2); Block_0_8(W_X); Rx_3( 2); Block_0_9();
wolfSSL	4:1b0d80432c79	1544	Rx_1( 3); Block_0_10();Rx_2( 3); Block_0_11();Rx_3( 3); Block_0_12(W_X);
wolfSSL	4:1b0d80432c79	1545
wolfSSL	4:1b0d80432c79	1546	Rx_1( 4); Block_4_1(W_X); Rx_2( 4); Block_4_2(W_X); Rx_3( 4); Block_4_3();
wolfSSL	4:1b0d80432c79	1547	Rx_1( 5); Block_4_4(); Rx_2( 5); Block_4_5(); Rx_3( 5); Block_4_6(W_X);
wolfSSL	4:1b0d80432c79	1548	Rx_1( 6); Block_4_7(W_X); Rx_2( 6); Block_4_8(W_X); Rx_3( 6); Block_4_9();
wolfSSL	4:1b0d80432c79	1549	Rx_1( 7); Block_4_10();Rx_2( 7); Block_4_11();Rx_3( 7); Block_4_12(W_X);
wolfSSL	4:1b0d80432c79	1550
wolfSSL	4:1b0d80432c79	1551	Rx_1( 8); Block_8_1(W_X); Rx_2( 8); Block_8_2(W_X); Rx_3( 8); Block_8_3();
wolfSSL	4:1b0d80432c79	1552	Rx_1( 9); Block_8_4(); Rx_2( 9); Block_8_5(); Rx_3( 9); Block_8_6(W_X);
wolfSSL	4:1b0d80432c79	1553	Rx_1(10); Block_8_7(W_X); Rx_2(10); Block_8_8(W_X); Rx_3(10); Block_8_9();
wolfSSL	4:1b0d80432c79	1554	Rx_1(11); Block_8_10();Rx_2(11); Block_8_11();Rx_3(11); Block_8_12(W_X);
wolfSSL	4:1b0d80432c79	1555
wolfSSL	4:1b0d80432c79	1556	Rx_1(12); Block_12_1(W_X); Rx_2(12); Block_12_2(W_X); Rx_3(12); Block_12_3();
wolfSSL	4:1b0d80432c79	1557	Rx_1(13); Block_12_4(); Rx_2(13); Block_12_5(); Rx_3(13); Block_12_6(W_X);
wolfSSL	4:1b0d80432c79	1558	Rx_1(14); Block_12_7(W_X); Rx_2(14); Block_12_8(W_X); Rx_3(14); Block_12_9();
wolfSSL	4:1b0d80432c79	1559	Rx_1(15); Block_12_10();Rx_2(15); Block_12_11();Rx_3(15); Block_12_12(W_X);
wolfSSL	4:1b0d80432c79	1560	}
wolfSSL	4:1b0d80432c79	1561
wolfSSL	4:1b0d80432c79	1562	/* Add the working vars back into digest */
wolfSSL	4:1b0d80432c79	1563
wolfSSL	4:1b0d80432c79	1564	sha384->digest[0] += a(0);
wolfSSL	4:1b0d80432c79	1565	sha384->digest[1] += b(0);
wolfSSL	4:1b0d80432c79	1566	sha384->digest[2] += c(0);
wolfSSL	4:1b0d80432c79	1567	sha384->digest[3] += d(0);
wolfSSL	4:1b0d80432c79	1568	sha384->digest[4] += e(0);
wolfSSL	4:1b0d80432c79	1569	sha384->digest[5] += f(0);
wolfSSL	4:1b0d80432c79	1570	sha384->digest[6] += g(0);
wolfSSL	4:1b0d80432c79	1571	sha384->digest[7] += h(0);
wolfSSL	4:1b0d80432c79	1572
wolfSSL	4:1b0d80432c79	1573	/* Wipe variables */
wolfSSL	4:1b0d80432c79	1574	#if !defined(HAVE_INTEL_AVX1)&&!defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1575	XMEMSET(W, 0, sizeof(word64) * 16);
wolfSSL	4:1b0d80432c79	1576	#endif
wolfSSL	4:1b0d80432c79	1577	XMEMSET(T, 0, sizeof(T));
wolfSSL	4:1b0d80432c79	1578
wolfSSL	4:1b0d80432c79	1579	return 0;
wolfSSL	4:1b0d80432c79	1580	}
wolfSSL	4:1b0d80432c79	1581
wolfSSL	4:1b0d80432c79	1582	#endif
wolfSSL	4:1b0d80432c79	1583
wolfSSL	4:1b0d80432c79	1584	#if defined(HAVE_INTEL_AVX1) && defined(HAVE_INTEL_AVX2) && defined(HAVE_INTEL_RORX)
wolfSSL	4:1b0d80432c79	1585	static int Transform384_AVX1_RORX(Sha384* sha384)
wolfSSL	4:1b0d80432c79	1586	{
wolfSSL	4:1b0d80432c79	1587	const word64* K = K512;
wolfSSL	4:1b0d80432c79	1588	word64 W_X[16+4];
wolfSSL	4:1b0d80432c79	1589	word32 j;
wolfSSL	4:1b0d80432c79	1590	word64 T[8];
wolfSSL	4:1b0d80432c79	1591
wolfSSL	4:1b0d80432c79	1592	/* Copy digest to working vars */
wolfSSL	4:1b0d80432c79	1593	XMEMCPY(T, sha384->digest, sizeof(T));
wolfSSL	4:1b0d80432c79	1594
wolfSSL	4:1b0d80432c79	1595	W_from_buff(W_X, sha384->buffer) ;
wolfSSL	4:1b0d80432c79	1596	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	1597	Rx_RORX_1( 0); Block_0_1(W_X); Rx_RORX_2( 0);
wolfSSL	4:1b0d80432c79	1598	Block_0_2(W_X); Rx_RORX_3( 0); Block_0_3();
wolfSSL	4:1b0d80432c79	1599	Rx_RORX_1( 1); Block_0_4(); Rx_RORX_2( 1);
wolfSSL	4:1b0d80432c79	1600	Block_0_5(); Rx_RORX_3( 1); Block_0_6(W_X);
wolfSSL	4:1b0d80432c79	1601	Rx_RORX_1( 2); Block_0_7(W_X); Rx_RORX_2( 2);
wolfSSL	4:1b0d80432c79	1602	Block_0_8(W_X); Rx_RORX_3( 2); Block_0_9();
wolfSSL	4:1b0d80432c79	1603	Rx_RORX_1( 3); Block_0_10();Rx_RORX_2( 3);
wolfSSL	4:1b0d80432c79	1604	Block_0_11();Rx_RORX_3( 3); Block_0_12(W_X);
wolfSSL	4:1b0d80432c79	1605
wolfSSL	4:1b0d80432c79	1606	Rx_RORX_1( 4); Block_4_1(W_X); Rx_RORX_2( 4);
wolfSSL	4:1b0d80432c79	1607	Block_4_2(W_X); Rx_RORX_3( 4); Block_4_3();
wolfSSL	4:1b0d80432c79	1608	Rx_RORX_1( 5); Block_4_4(); Rx_RORX_2( 5);
wolfSSL	4:1b0d80432c79	1609	Block_4_5(); Rx_RORX_3( 5); Block_4_6(W_X);
wolfSSL	4:1b0d80432c79	1610	Rx_RORX_1( 6); Block_4_7(W_X); Rx_RORX_2( 6);
wolfSSL	4:1b0d80432c79	1611	Block_4_8(W_X); Rx_RORX_3( 6); Block_4_9();
wolfSSL	4:1b0d80432c79	1612	Rx_RORX_1( 7); Block_4_10();Rx_RORX_2( 7);
wolfSSL	4:1b0d80432c79	1613	Block_4_11();Rx_RORX_3( 7); Block_4_12(W_X);
wolfSSL	4:1b0d80432c79	1614
wolfSSL	4:1b0d80432c79	1615	Rx_RORX_1( 8); Block_8_1(W_X); Rx_RORX_2( 8);
wolfSSL	4:1b0d80432c79	1616	Block_8_2(W_X); Rx_RORX_3( 8); Block_8_3();
wolfSSL	4:1b0d80432c79	1617	Rx_RORX_1( 9); Block_8_4(); Rx_RORX_2( 9);
wolfSSL	4:1b0d80432c79	1618	Block_8_5(); Rx_RORX_3( 9); Block_8_6(W_X);
wolfSSL	4:1b0d80432c79	1619	Rx_RORX_1(10); Block_8_7(W_X); Rx_RORX_2(10);
wolfSSL	4:1b0d80432c79	1620	Block_8_8(W_X); Rx_RORX_3(10); Block_8_9();
wolfSSL	4:1b0d80432c79	1621	Rx_RORX_1(11); Block_8_10();Rx_RORX_2(11);
wolfSSL	4:1b0d80432c79	1622	Block_8_11();Rx_RORX_3(11); Block_8_12(W_X);
wolfSSL	4:1b0d80432c79	1623
wolfSSL	4:1b0d80432c79	1624	Rx_RORX_1(12); Block_12_1(W_X); Rx_RORX_2(12);
wolfSSL	4:1b0d80432c79	1625	Block_12_2(W_X); Rx_RORX_3(12); Block_12_3();
wolfSSL	4:1b0d80432c79	1626	Rx_RORX_1(13); Block_12_4(); Rx_RORX_2(13);
wolfSSL	4:1b0d80432c79	1627	Block_12_5(); Rx_RORX_3(13); Block_12_6(W_X);
wolfSSL	4:1b0d80432c79	1628	Rx_RORX_1(14); Block_12_7(W_X); Rx_RORX_2(14);
wolfSSL	4:1b0d80432c79	1629	Block_12_8(W_X); Rx_RORX_3(14); Block_12_9();
wolfSSL	4:1b0d80432c79	1630	Rx_RORX_1(15); Block_12_10();Rx_RORX_2(15);
wolfSSL	4:1b0d80432c79	1631	Block_12_11();Rx_RORX_3(15); Block_12_12(W_X);
wolfSSL	4:1b0d80432c79	1632	}
wolfSSL	4:1b0d80432c79	1633
wolfSSL	4:1b0d80432c79	1634	/* Add the working vars back into digest */
wolfSSL	4:1b0d80432c79	1635
wolfSSL	4:1b0d80432c79	1636	sha384->digest[0] += a(0);
wolfSSL	4:1b0d80432c79	1637	sha384->digest[1] += b(0);
wolfSSL	4:1b0d80432c79	1638	sha384->digest[2] += c(0);
wolfSSL	4:1b0d80432c79	1639	sha384->digest[3] += d(0);
wolfSSL	4:1b0d80432c79	1640	sha384->digest[4] += e(0);
wolfSSL	4:1b0d80432c79	1641	sha384->digest[5] += f(0);
wolfSSL	4:1b0d80432c79	1642	sha384->digest[6] += g(0);
wolfSSL	4:1b0d80432c79	1643	sha384->digest[7] += h(0);
wolfSSL	4:1b0d80432c79	1644
wolfSSL	4:1b0d80432c79	1645	/* Wipe variables */
wolfSSL	4:1b0d80432c79	1646	#if !defined(HAVE_INTEL_AVX1)&&!defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1647	XMEMSET(W, 0, sizeof(word64) * 16);
wolfSSL	4:1b0d80432c79	1648	#endif
wolfSSL	4:1b0d80432c79	1649	XMEMSET(T, 0, sizeof(T));
wolfSSL	4:1b0d80432c79	1650
wolfSSL	4:1b0d80432c79	1651	return 0;
wolfSSL	4:1b0d80432c79	1652	}
wolfSSL	4:1b0d80432c79	1653	#endif
wolfSSL	4:1b0d80432c79	1654
wolfSSL	4:1b0d80432c79	1655	#if defined(HAVE_INTEL_AVX2)
wolfSSL	4:1b0d80432c79	1656
wolfSSL	4:1b0d80432c79	1657	static int Transform384_AVX2(Sha384* sha384)
wolfSSL	4:1b0d80432c79	1658	{
wolfSSL	4:1b0d80432c79	1659	const word64* K = K512;
wolfSSL	4:1b0d80432c79	1660	word64 w[4] ;
wolfSSL	4:1b0d80432c79	1661	word32 j;
wolfSSL	4:1b0d80432c79	1662	word64 T[8];
wolfSSL	4:1b0d80432c79	1663
wolfSSL	4:1b0d80432c79	1664	/* Copy digest to working vars */
wolfSSL	4:1b0d80432c79	1665	XMEMCPY(T, sha384->digest, sizeof(T));
wolfSSL	4:1b0d80432c79	1666
wolfSSL	4:1b0d80432c79	1667	/* over twice as small, but 50% slower */
wolfSSL	4:1b0d80432c79	1668	/* 80 operations, not unrolled */
wolfSSL	4:1b0d80432c79	1669
wolfSSL	4:1b0d80432c79	1670	W_from_buff_Y(sha384->buffer) ;
wolfSSL	4:1b0d80432c79	1671
wolfSSL	4:1b0d80432c79	1672	MOVE_to_MEMy(w,0, W_0y) ;
wolfSSL	4:1b0d80432c79	1673	for (j = 0; j < 80; j += 16) {
wolfSSL	4:1b0d80432c79	1674	Ry_1( 0, w[0]); Block_Y_0_1(); Ry_2( 0, w[0]);
wolfSSL	4:1b0d80432c79	1675	Block_Y_0_2(); Ry_3( 0, w[0]); Block_Y_0_3();
wolfSSL	4:1b0d80432c79	1676	Ry_1( 1, w[1]); Block_Y_0_4(); Ry_2( 1, w[1]);
wolfSSL	4:1b0d80432c79	1677	Block_Y_0_5(); Ry_3( 1, w[1]); Block_Y_0_6();
wolfSSL	4:1b0d80432c79	1678	Ry_1( 2, w[2]); Block_Y_0_7(); Ry_2( 2, w[2]);
wolfSSL	4:1b0d80432c79	1679	Block_Y_0_8(); Ry_3( 2, w[2]); Block_Y_0_9();
wolfSSL	4:1b0d80432c79	1680	Ry_1( 3, w[3]); Block_Y_0_10();Ry_2( 3, w[3]);
wolfSSL	4:1b0d80432c79	1681	Block_Y_0_11();Ry_3( 3, w[3]); Block_Y_0_12(w);
wolfSSL	4:1b0d80432c79	1682
wolfSSL	4:1b0d80432c79	1683	Ry_1( 4, w[0]); Block_Y_4_1(); Ry_2( 4, w[0]);
wolfSSL	4:1b0d80432c79	1684	Block_Y_4_2(); Ry_3( 4, w[0]); Block_Y_4_3();
wolfSSL	4:1b0d80432c79	1685	Ry_1( 5, w[1]); Block_Y_4_4(); Ry_2( 5, w[1]);
wolfSSL	4:1b0d80432c79	1686	Block_Y_4_5(); Ry_3( 5, w[1]); Block_Y_4_6();
wolfSSL	4:1b0d80432c79	1687	Ry_1( 6, w[2]); Block_Y_4_7(); Ry_2( 6, w[2]);
wolfSSL	4:1b0d80432c79	1688	Block_Y_4_8(); Ry_3( 6, w[2]); Block_Y_4_9();
wolfSSL	4:1b0d80432c79	1689	Ry_1( 7, w[3]); Block_Y_4_10(); Ry_2( 7, w[3]);
wolfSSL	4:1b0d80432c79	1690	Block_Y_4_11(); Ry_3( 7, w[3]);Block_Y_4_12(w);
wolfSSL	4:1b0d80432c79	1691
wolfSSL	4:1b0d80432c79	1692	Ry_1( 8, w[0]); Block_Y_8_1(); Ry_2( 8, w[0]);
wolfSSL	4:1b0d80432c79	1693	Block_Y_8_2(); Ry_3( 8, w[0]); Block_Y_8_3();
wolfSSL	4:1b0d80432c79	1694	Ry_1( 9, w[1]); Block_Y_8_4(); Ry_2( 9, w[1]);
wolfSSL	4:1b0d80432c79	1695	Block_Y_8_5(); Ry_3( 9, w[1]); Block_Y_8_6();
wolfSSL	4:1b0d80432c79	1696	Ry_1(10, w[2]); Block_Y_8_7(); Ry_2(10, w[2]);
wolfSSL	4:1b0d80432c79	1697	Block_Y_8_8(); Ry_3(10, w[2]); Block_Y_8_9();
wolfSSL	4:1b0d80432c79	1698	Ry_1(11, w[3]); Block_Y_8_10();Ry_2(11, w[3]);
wolfSSL	4:1b0d80432c79	1699	Block_Y_8_11();Ry_3(11, w[3]); Block_Y_8_12(w);
wolfSSL	4:1b0d80432c79	1700
wolfSSL	4:1b0d80432c79	1701	Ry_1(12, w[0]); Block_Y_12_1(); Ry_2(12, w[0]);
wolfSSL	4:1b0d80432c79	1702	Block_Y_12_2(); Ry_3(12, w[0]); Block_Y_12_3();
wolfSSL	4:1b0d80432c79	1703	Ry_1(13, w[1]); Block_Y_12_4(); Ry_2(13, w[1]);
wolfSSL	4:1b0d80432c79	1704	Block_Y_12_5(); Ry_3(13, w[1]); Block_Y_12_6();
wolfSSL	4:1b0d80432c79	1705	Ry_1(14, w[2]); Block_Y_12_7(); Ry_2(14, w[2]);
wolfSSL	4:1b0d80432c79	1706	Block_Y_12_8(); Ry_3(14, w[2]); Block_Y_12_9();
wolfSSL	4:1b0d80432c79	1707	Ry_1(15, w[3]); Block_Y_12_10();Ry_2(15, w[3]);
wolfSSL	4:1b0d80432c79	1708	Block_Y_12_11();Ry_3(15, w[3]); Block_Y_12_12(w);
wolfSSL	4:1b0d80432c79	1709	}
wolfSSL	4:1b0d80432c79	1710
wolfSSL	4:1b0d80432c79	1711	/* Add the working vars back into digest */
wolfSSL	4:1b0d80432c79	1712
wolfSSL	4:1b0d80432c79	1713	sha384->digest[0] += a(0);
wolfSSL	4:1b0d80432c79	1714	sha384->digest[1] += b(0);
wolfSSL	4:1b0d80432c79	1715	sha384->digest[2] += c(0);
wolfSSL	4:1b0d80432c79	1716	sha384->digest[3] += d(0);
wolfSSL	4:1b0d80432c79	1717	sha384->digest[4] += e(0);
wolfSSL	4:1b0d80432c79	1718	sha384->digest[5] += f(0);
wolfSSL	4:1b0d80432c79	1719	sha384->digest[6] += g(0);
wolfSSL	4:1b0d80432c79	1720	sha384->digest[7] += h(0);
wolfSSL	4:1b0d80432c79	1721
wolfSSL	4:1b0d80432c79	1722	/* Wipe variables */
wolfSSL	4:1b0d80432c79	1723	XMEMSET(T, 0, sizeof(T));
wolfSSL	4:1b0d80432c79	1724
wolfSSL	4:1b0d80432c79	1725	return 0;
wolfSSL	4:1b0d80432c79	1726	}
wolfSSL	4:1b0d80432c79	1727
wolfSSL	4:1b0d80432c79	1728	#endif
wolfSSL	4:1b0d80432c79	1729
wolfSSL	4:1b0d80432c79	1730	#endif /* WOLFSSL_SHA384 */
wolfSSL	4:1b0d80432c79	1731
wolfSSL	4:1b0d80432c79	1732	#endif /* HAVE_FIPS */
wolfSSL	4:1b0d80432c79	1733
wolfSSL	4:1b0d80432c79	1734	#endif /* WOLFSSL_SHA512 */
wolfSSL	4:1b0d80432c79	1735
wolfSSL	4:1b0d80432c79	1736

Repository toolbox

Export to desktop IDE

Repository details

Type:	Library
Created:	26 Jun 2015
Imports:	673
Forks:	0
Commits:	18
Dependents:	29
Dependencies:	0
Followers:	21

wolfcrypt/src/sha512.c@4:1b0d80432c79, 2016-04-28 (annotated)

Who changed what in which revision?

Repository toolbox

Repository details

Important Information for this Arm website

Access Warning