mbed TLS library
Dependents: HTTPClient-SSL WS_SERVER
polarssl/aesni.c@0:137634ff4186, 2015-06-11 (annotated)
- Committer:
- ansond
- Date:
- Thu Jun 11 03:27:03 2015 +0000
- Revision:
- 0:137634ff4186
initial commit
Who changed what in which revision?
User | Revision | Line number | New contents of line |
---|---|---|---|
ansond | 0:137634ff4186 | 1 | /* |
ansond | 0:137634ff4186 | 2 | * AES-NI support functions |
ansond | 0:137634ff4186 | 3 | * |
ansond | 0:137634ff4186 | 4 | * Copyright (C) 2006-2014, ARM Limited, All Rights Reserved |
ansond | 0:137634ff4186 | 5 | * |
ansond | 0:137634ff4186 | 6 | * This file is part of mbed TLS (https://tls.mbed.org) |
ansond | 0:137634ff4186 | 7 | * |
ansond | 0:137634ff4186 | 8 | * This program is free software; you can redistribute it and/or modify |
ansond | 0:137634ff4186 | 9 | * it under the terms of the GNU General Public License as published by |
ansond | 0:137634ff4186 | 10 | * the Free Software Foundation; either version 2 of the License, or |
ansond | 0:137634ff4186 | 11 | * (at your option) any later version. |
ansond | 0:137634ff4186 | 12 | * |
ansond | 0:137634ff4186 | 13 | * This program is distributed in the hope that it will be useful, |
ansond | 0:137634ff4186 | 14 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
ansond | 0:137634ff4186 | 15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
ansond | 0:137634ff4186 | 16 | * GNU General Public License for more details. |
ansond | 0:137634ff4186 | 17 | * |
ansond | 0:137634ff4186 | 18 | * You should have received a copy of the GNU General Public License along |
ansond | 0:137634ff4186 | 19 | * with this program; if not, write to the Free Software Foundation, Inc., |
ansond | 0:137634ff4186 | 20 | * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. |
ansond | 0:137634ff4186 | 21 | */ |
ansond | 0:137634ff4186 | 22 | |
ansond | 0:137634ff4186 | 23 | /* |
ansond | 0:137634ff4186 | 24 | * [AES-WP] http://software.intel.com/en-us/articles/intel-advanced-encryption-standard-aes-instructions-set |
ansond | 0:137634ff4186 | 25 | * [CLMUL-WP] http://software.intel.com/en-us/articles/intel-carry-less-multiplication-instruction-and-its-usage-for-computing-the-gcm-mode/ |
ansond | 0:137634ff4186 | 26 | */ |
ansond | 0:137634ff4186 | 27 | |
ansond | 0:137634ff4186 | 28 | #if !defined(POLARSSL_CONFIG_FILE) |
ansond | 0:137634ff4186 | 29 | #include "polarssl/config.h" |
ansond | 0:137634ff4186 | 30 | #else |
ansond | 0:137634ff4186 | 31 | #include POLARSSL_CONFIG_FILE |
ansond | 0:137634ff4186 | 32 | #endif |
ansond | 0:137634ff4186 | 33 | |
ansond | 0:137634ff4186 | 34 | #if defined(POLARSSL_AESNI_C) |
ansond | 0:137634ff4186 | 35 | |
ansond | 0:137634ff4186 | 36 | #include "polarssl/aesni.h" |
ansond | 0:137634ff4186 | 37 | |
ansond | 0:137634ff4186 | 38 | #include <string.h> |
ansond | 0:137634ff4186 | 39 | |
ansond | 0:137634ff4186 | 40 | #if defined(POLARSSL_HAVE_X86_64) |
ansond | 0:137634ff4186 | 41 | |
ansond | 0:137634ff4186 | 42 | /* |
ansond | 0:137634ff4186 | 43 | * AES-NI support detection routine |
ansond | 0:137634ff4186 | 44 | */ |
ansond | 0:137634ff4186 | 45 | int aesni_supports( unsigned int what ) |
ansond | 0:137634ff4186 | 46 | { |
ansond | 0:137634ff4186 | 47 | static int done = 0; |
ansond | 0:137634ff4186 | 48 | static unsigned int c = 0; |
ansond | 0:137634ff4186 | 49 | |
ansond | 0:137634ff4186 | 50 | if( ! done ) |
ansond | 0:137634ff4186 | 51 | { |
ansond | 0:137634ff4186 | 52 | asm( "movl $1, %%eax \n\t" |
ansond | 0:137634ff4186 | 53 | "cpuid \n\t" |
ansond | 0:137634ff4186 | 54 | : "=c" (c) |
ansond | 0:137634ff4186 | 55 | : |
ansond | 0:137634ff4186 | 56 | : "eax", "ebx", "edx" ); |
ansond | 0:137634ff4186 | 57 | done = 1; |
ansond | 0:137634ff4186 | 58 | } |
ansond | 0:137634ff4186 | 59 | |
ansond | 0:137634ff4186 | 60 | return( ( c & what ) != 0 ); |
ansond | 0:137634ff4186 | 61 | } |
ansond | 0:137634ff4186 | 62 | |
ansond | 0:137634ff4186 | 63 | /* |
ansond | 0:137634ff4186 | 64 | * Binutils needs to be at least 2.19 to support AES-NI instructions. |
ansond | 0:137634ff4186 | 65 | * Unfortunately, a lot of users have a lower version now (2014-04). |
ansond | 0:137634ff4186 | 66 | * Emit bytecode directly in order to support "old" version of gas. |
ansond | 0:137634ff4186 | 67 | * |
ansond | 0:137634ff4186 | 68 | * Opcodes from the Intel architecture reference manual, vol. 3. |
ansond | 0:137634ff4186 | 69 | * We always use registers, so we don't need prefixes for memory operands. |
ansond | 0:137634ff4186 | 70 | * Operand macros are in gas order (src, dst) as opposed to Intel order |
ansond | 0:137634ff4186 | 71 | * (dst, src) in order to blend better into the surrounding assembly code. |
ansond | 0:137634ff4186 | 72 | */ |
ansond | 0:137634ff4186 | 73 | #define AESDEC ".byte 0x66,0x0F,0x38,0xDE," |
ansond | 0:137634ff4186 | 74 | #define AESDECLAST ".byte 0x66,0x0F,0x38,0xDF," |
ansond | 0:137634ff4186 | 75 | #define AESENC ".byte 0x66,0x0F,0x38,0xDC," |
ansond | 0:137634ff4186 | 76 | #define AESENCLAST ".byte 0x66,0x0F,0x38,0xDD," |
ansond | 0:137634ff4186 | 77 | #define AESIMC ".byte 0x66,0x0F,0x38,0xDB," |
ansond | 0:137634ff4186 | 78 | #define AESKEYGENA ".byte 0x66,0x0F,0x3A,0xDF," |
ansond | 0:137634ff4186 | 79 | #define PCLMULQDQ ".byte 0x66,0x0F,0x3A,0x44," |
ansond | 0:137634ff4186 | 80 | |
ansond | 0:137634ff4186 | 81 | #define xmm0_xmm0 "0xC0" |
ansond | 0:137634ff4186 | 82 | #define xmm0_xmm1 "0xC8" |
ansond | 0:137634ff4186 | 83 | #define xmm0_xmm2 "0xD0" |
ansond | 0:137634ff4186 | 84 | #define xmm0_xmm3 "0xD8" |
ansond | 0:137634ff4186 | 85 | #define xmm0_xmm4 "0xE0" |
ansond | 0:137634ff4186 | 86 | #define xmm1_xmm0 "0xC1" |
ansond | 0:137634ff4186 | 87 | #define xmm1_xmm2 "0xD1" |
ansond | 0:137634ff4186 | 88 | |
ansond | 0:137634ff4186 | 89 | /* |
ansond | 0:137634ff4186 | 90 | * AES-NI AES-ECB block en(de)cryption |
ansond | 0:137634ff4186 | 91 | */ |
ansond | 0:137634ff4186 | 92 | int aesni_crypt_ecb( aes_context *ctx, |
ansond | 0:137634ff4186 | 93 | int mode, |
ansond | 0:137634ff4186 | 94 | const unsigned char input[16], |
ansond | 0:137634ff4186 | 95 | unsigned char output[16] ) |
ansond | 0:137634ff4186 | 96 | { |
ansond | 0:137634ff4186 | 97 | asm( "movdqu (%3), %%xmm0 \n\t" // load input |
ansond | 0:137634ff4186 | 98 | "movdqu (%1), %%xmm1 \n\t" // load round key 0 |
ansond | 0:137634ff4186 | 99 | "pxor %%xmm1, %%xmm0 \n\t" // round 0 |
ansond | 0:137634ff4186 | 100 | "addq $16, %1 \n\t" // point to next round key |
ansond | 0:137634ff4186 | 101 | "subl $1, %0 \n\t" // normal rounds = nr - 1 |
ansond | 0:137634ff4186 | 102 | "test %2, %2 \n\t" // mode? |
ansond | 0:137634ff4186 | 103 | "jz 2f \n\t" // 0 = decrypt |
ansond | 0:137634ff4186 | 104 | |
ansond | 0:137634ff4186 | 105 | "1: \n\t" // encryption loop |
ansond | 0:137634ff4186 | 106 | "movdqu (%1), %%xmm1 \n\t" // load round key |
ansond | 0:137634ff4186 | 107 | AESENC xmm1_xmm0 "\n\t" // do round |
ansond | 0:137634ff4186 | 108 | "addq $16, %1 \n\t" // point to next round key |
ansond | 0:137634ff4186 | 109 | "subl $1, %0 \n\t" // loop |
ansond | 0:137634ff4186 | 110 | "jnz 1b \n\t" |
ansond | 0:137634ff4186 | 111 | "movdqu (%1), %%xmm1 \n\t" // load round key |
ansond | 0:137634ff4186 | 112 | AESENCLAST xmm1_xmm0 "\n\t" // last round |
ansond | 0:137634ff4186 | 113 | "jmp 3f \n\t" |
ansond | 0:137634ff4186 | 114 | |
ansond | 0:137634ff4186 | 115 | "2: \n\t" // decryption loop |
ansond | 0:137634ff4186 | 116 | "movdqu (%1), %%xmm1 \n\t" |
ansond | 0:137634ff4186 | 117 | AESDEC xmm1_xmm0 "\n\t" // do round |
ansond | 0:137634ff4186 | 118 | "addq $16, %1 \n\t" |
ansond | 0:137634ff4186 | 119 | "subl $1, %0 \n\t" |
ansond | 0:137634ff4186 | 120 | "jnz 2b \n\t" |
ansond | 0:137634ff4186 | 121 | "movdqu (%1), %%xmm1 \n\t" // load round key |
ansond | 0:137634ff4186 | 122 | AESDECLAST xmm1_xmm0 "\n\t" // last round |
ansond | 0:137634ff4186 | 123 | |
ansond | 0:137634ff4186 | 124 | "3: \n\t" |
ansond | 0:137634ff4186 | 125 | "movdqu %%xmm0, (%4) \n\t" // export output |
ansond | 0:137634ff4186 | 126 | : |
ansond | 0:137634ff4186 | 127 | : "r" (ctx->nr), "r" (ctx->rk), "r" (mode), "r" (input), "r" (output) |
ansond | 0:137634ff4186 | 128 | : "memory", "cc", "xmm0", "xmm1" ); |
ansond | 0:137634ff4186 | 129 | |
ansond | 0:137634ff4186 | 130 | |
ansond | 0:137634ff4186 | 131 | return( 0 ); |
ansond | 0:137634ff4186 | 132 | } |
ansond | 0:137634ff4186 | 133 | |
ansond | 0:137634ff4186 | 134 | /* |
ansond | 0:137634ff4186 | 135 | * GCM multiplication: c = a times b in GF(2^128) |
ansond | 0:137634ff4186 | 136 | * Based on [CLMUL-WP] algorithms 1 (with equation 27) and 5. |
ansond | 0:137634ff4186 | 137 | */ |
ansond | 0:137634ff4186 | 138 | void aesni_gcm_mult( unsigned char c[16], |
ansond | 0:137634ff4186 | 139 | const unsigned char a[16], |
ansond | 0:137634ff4186 | 140 | const unsigned char b[16] ) |
ansond | 0:137634ff4186 | 141 | { |
ansond | 0:137634ff4186 | 142 | unsigned char aa[16], bb[16], cc[16]; |
ansond | 0:137634ff4186 | 143 | size_t i; |
ansond | 0:137634ff4186 | 144 | |
ansond | 0:137634ff4186 | 145 | /* The inputs are in big-endian order, so byte-reverse them */ |
ansond | 0:137634ff4186 | 146 | for( i = 0; i < 16; i++ ) |
ansond | 0:137634ff4186 | 147 | { |
ansond | 0:137634ff4186 | 148 | aa[i] = a[15 - i]; |
ansond | 0:137634ff4186 | 149 | bb[i] = b[15 - i]; |
ansond | 0:137634ff4186 | 150 | } |
ansond | 0:137634ff4186 | 151 | |
ansond | 0:137634ff4186 | 152 | asm( "movdqu (%0), %%xmm0 \n\t" // a1:a0 |
ansond | 0:137634ff4186 | 153 | "movdqu (%1), %%xmm1 \n\t" // b1:b0 |
ansond | 0:137634ff4186 | 154 | |
ansond | 0:137634ff4186 | 155 | /* |
ansond | 0:137634ff4186 | 156 | * Caryless multiplication xmm2:xmm1 = xmm0 * xmm1 |
ansond | 0:137634ff4186 | 157 | * using [CLMUL-WP] algorithm 1 (p. 13). |
ansond | 0:137634ff4186 | 158 | */ |
ansond | 0:137634ff4186 | 159 | "movdqa %%xmm1, %%xmm2 \n\t" // copy of b1:b0 |
ansond | 0:137634ff4186 | 160 | "movdqa %%xmm1, %%xmm3 \n\t" // same |
ansond | 0:137634ff4186 | 161 | "movdqa %%xmm1, %%xmm4 \n\t" // same |
ansond | 0:137634ff4186 | 162 | PCLMULQDQ xmm0_xmm1 ",0x00 \n\t" // a0*b0 = c1:c0 |
ansond | 0:137634ff4186 | 163 | PCLMULQDQ xmm0_xmm2 ",0x11 \n\t" // a1*b1 = d1:d0 |
ansond | 0:137634ff4186 | 164 | PCLMULQDQ xmm0_xmm3 ",0x10 \n\t" // a0*b1 = e1:e0 |
ansond | 0:137634ff4186 | 165 | PCLMULQDQ xmm0_xmm4 ",0x01 \n\t" // a1*b0 = f1:f0 |
ansond | 0:137634ff4186 | 166 | "pxor %%xmm3, %%xmm4 \n\t" // e1+f1:e0+f0 |
ansond | 0:137634ff4186 | 167 | "movdqa %%xmm4, %%xmm3 \n\t" // same |
ansond | 0:137634ff4186 | 168 | "psrldq $8, %%xmm4 \n\t" // 0:e1+f1 |
ansond | 0:137634ff4186 | 169 | "pslldq $8, %%xmm3 \n\t" // e0+f0:0 |
ansond | 0:137634ff4186 | 170 | "pxor %%xmm4, %%xmm2 \n\t" // d1:d0+e1+f1 |
ansond | 0:137634ff4186 | 171 | "pxor %%xmm3, %%xmm1 \n\t" // c1+e0+f1:c0 |
ansond | 0:137634ff4186 | 172 | |
ansond | 0:137634ff4186 | 173 | /* |
ansond | 0:137634ff4186 | 174 | * Now shift the result one bit to the left, |
ansond | 0:137634ff4186 | 175 | * taking advantage of [CLMUL-WP] eq 27 (p. 20) |
ansond | 0:137634ff4186 | 176 | */ |
ansond | 0:137634ff4186 | 177 | "movdqa %%xmm1, %%xmm3 \n\t" // r1:r0 |
ansond | 0:137634ff4186 | 178 | "movdqa %%xmm2, %%xmm4 \n\t" // r3:r2 |
ansond | 0:137634ff4186 | 179 | "psllq $1, %%xmm1 \n\t" // r1<<1:r0<<1 |
ansond | 0:137634ff4186 | 180 | "psllq $1, %%xmm2 \n\t" // r3<<1:r2<<1 |
ansond | 0:137634ff4186 | 181 | "psrlq $63, %%xmm3 \n\t" // r1>>63:r0>>63 |
ansond | 0:137634ff4186 | 182 | "psrlq $63, %%xmm4 \n\t" // r3>>63:r2>>63 |
ansond | 0:137634ff4186 | 183 | "movdqa %%xmm3, %%xmm5 \n\t" // r1>>63:r0>>63 |
ansond | 0:137634ff4186 | 184 | "pslldq $8, %%xmm3 \n\t" // r0>>63:0 |
ansond | 0:137634ff4186 | 185 | "pslldq $8, %%xmm4 \n\t" // r2>>63:0 |
ansond | 0:137634ff4186 | 186 | "psrldq $8, %%xmm5 \n\t" // 0:r1>>63 |
ansond | 0:137634ff4186 | 187 | "por %%xmm3, %%xmm1 \n\t" // r1<<1|r0>>63:r0<<1 |
ansond | 0:137634ff4186 | 188 | "por %%xmm4, %%xmm2 \n\t" // r3<<1|r2>>62:r2<<1 |
ansond | 0:137634ff4186 | 189 | "por %%xmm5, %%xmm2 \n\t" // r3<<1|r2>>62:r2<<1|r1>>63 |
ansond | 0:137634ff4186 | 190 | |
ansond | 0:137634ff4186 | 191 | /* |
ansond | 0:137634ff4186 | 192 | * Now reduce modulo the GCM polynomial x^128 + x^7 + x^2 + x + 1 |
ansond | 0:137634ff4186 | 193 | * using [CLMUL-WP] algorithm 5 (p. 20). |
ansond | 0:137634ff4186 | 194 | * Currently xmm2:xmm1 holds x3:x2:x1:x0 (already shifted). |
ansond | 0:137634ff4186 | 195 | */ |
ansond | 0:137634ff4186 | 196 | /* Step 2 (1) */ |
ansond | 0:137634ff4186 | 197 | "movdqa %%xmm1, %%xmm3 \n\t" // x1:x0 |
ansond | 0:137634ff4186 | 198 | "movdqa %%xmm1, %%xmm4 \n\t" // same |
ansond | 0:137634ff4186 | 199 | "movdqa %%xmm1, %%xmm5 \n\t" // same |
ansond | 0:137634ff4186 | 200 | "psllq $63, %%xmm3 \n\t" // x1<<63:x0<<63 = stuff:a |
ansond | 0:137634ff4186 | 201 | "psllq $62, %%xmm4 \n\t" // x1<<62:x0<<62 = stuff:b |
ansond | 0:137634ff4186 | 202 | "psllq $57, %%xmm5 \n\t" // x1<<57:x0<<57 = stuff:c |
ansond | 0:137634ff4186 | 203 | |
ansond | 0:137634ff4186 | 204 | /* Step 2 (2) */ |
ansond | 0:137634ff4186 | 205 | "pxor %%xmm4, %%xmm3 \n\t" // stuff:a+b |
ansond | 0:137634ff4186 | 206 | "pxor %%xmm5, %%xmm3 \n\t" // stuff:a+b+c |
ansond | 0:137634ff4186 | 207 | "pslldq $8, %%xmm3 \n\t" // a+b+c:0 |
ansond | 0:137634ff4186 | 208 | "pxor %%xmm3, %%xmm1 \n\t" // x1+a+b+c:x0 = d:x0 |
ansond | 0:137634ff4186 | 209 | |
ansond | 0:137634ff4186 | 210 | /* Steps 3 and 4 */ |
ansond | 0:137634ff4186 | 211 | "movdqa %%xmm1,%%xmm0 \n\t" // d:x0 |
ansond | 0:137634ff4186 | 212 | "movdqa %%xmm1,%%xmm4 \n\t" // same |
ansond | 0:137634ff4186 | 213 | "movdqa %%xmm1,%%xmm5 \n\t" // same |
ansond | 0:137634ff4186 | 214 | "psrlq $1, %%xmm0 \n\t" // e1:x0>>1 = e1:e0' |
ansond | 0:137634ff4186 | 215 | "psrlq $2, %%xmm4 \n\t" // f1:x0>>2 = f1:f0' |
ansond | 0:137634ff4186 | 216 | "psrlq $7, %%xmm5 \n\t" // g1:x0>>7 = g1:g0' |
ansond | 0:137634ff4186 | 217 | "pxor %%xmm4, %%xmm0 \n\t" // e1+f1:e0'+f0' |
ansond | 0:137634ff4186 | 218 | "pxor %%xmm5, %%xmm0 \n\t" // e1+f1+g1:e0'+f0'+g0' |
ansond | 0:137634ff4186 | 219 | // e0'+f0'+g0' is almost e0+f0+g0, ex\tcept for some missing |
ansond | 0:137634ff4186 | 220 | // bits carried from d. Now get those\t bits back in. |
ansond | 0:137634ff4186 | 221 | "movdqa %%xmm1,%%xmm3 \n\t" // d:x0 |
ansond | 0:137634ff4186 | 222 | "movdqa %%xmm1,%%xmm4 \n\t" // same |
ansond | 0:137634ff4186 | 223 | "movdqa %%xmm1,%%xmm5 \n\t" // same |
ansond | 0:137634ff4186 | 224 | "psllq $63, %%xmm3 \n\t" // d<<63:stuff |
ansond | 0:137634ff4186 | 225 | "psllq $62, %%xmm4 \n\t" // d<<62:stuff |
ansond | 0:137634ff4186 | 226 | "psllq $57, %%xmm5 \n\t" // d<<57:stuff |
ansond | 0:137634ff4186 | 227 | "pxor %%xmm4, %%xmm3 \n\t" // d<<63+d<<62:stuff |
ansond | 0:137634ff4186 | 228 | "pxor %%xmm5, %%xmm3 \n\t" // missing bits of d:stuff |
ansond | 0:137634ff4186 | 229 | "psrldq $8, %%xmm3 \n\t" // 0:missing bits of d |
ansond | 0:137634ff4186 | 230 | "pxor %%xmm3, %%xmm0 \n\t" // e1+f1+g1:e0+f0+g0 |
ansond | 0:137634ff4186 | 231 | "pxor %%xmm1, %%xmm0 \n\t" // h1:h0 |
ansond | 0:137634ff4186 | 232 | "pxor %%xmm2, %%xmm0 \n\t" // x3+h1:x2+h0 |
ansond | 0:137634ff4186 | 233 | |
ansond | 0:137634ff4186 | 234 | "movdqu %%xmm0, (%2) \n\t" // done |
ansond | 0:137634ff4186 | 235 | : |
ansond | 0:137634ff4186 | 236 | : "r" (aa), "r" (bb), "r" (cc) |
ansond | 0:137634ff4186 | 237 | : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5" ); |
ansond | 0:137634ff4186 | 238 | |
ansond | 0:137634ff4186 | 239 | /* Now byte-reverse the outputs */ |
ansond | 0:137634ff4186 | 240 | for( i = 0; i < 16; i++ ) |
ansond | 0:137634ff4186 | 241 | c[i] = cc[15 - i]; |
ansond | 0:137634ff4186 | 242 | |
ansond | 0:137634ff4186 | 243 | return; |
ansond | 0:137634ff4186 | 244 | } |
ansond | 0:137634ff4186 | 245 | |
ansond | 0:137634ff4186 | 246 | /* |
ansond | 0:137634ff4186 | 247 | * Compute decryption round keys from encryption round keys |
ansond | 0:137634ff4186 | 248 | */ |
ansond | 0:137634ff4186 | 249 | void aesni_inverse_key( unsigned char *invkey, |
ansond | 0:137634ff4186 | 250 | const unsigned char *fwdkey, int nr ) |
ansond | 0:137634ff4186 | 251 | { |
ansond | 0:137634ff4186 | 252 | unsigned char *ik = invkey; |
ansond | 0:137634ff4186 | 253 | const unsigned char *fk = fwdkey + 16 * nr; |
ansond | 0:137634ff4186 | 254 | |
ansond | 0:137634ff4186 | 255 | memcpy( ik, fk, 16 ); |
ansond | 0:137634ff4186 | 256 | |
ansond | 0:137634ff4186 | 257 | for( fk -= 16, ik += 16; fk > fwdkey; fk -= 16, ik += 16 ) |
ansond | 0:137634ff4186 | 258 | asm( "movdqu (%0), %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 259 | AESIMC xmm0_xmm0 "\n\t" |
ansond | 0:137634ff4186 | 260 | "movdqu %%xmm0, (%1) \n\t" |
ansond | 0:137634ff4186 | 261 | : |
ansond | 0:137634ff4186 | 262 | : "r" (fk), "r" (ik) |
ansond | 0:137634ff4186 | 263 | : "memory", "xmm0" ); |
ansond | 0:137634ff4186 | 264 | |
ansond | 0:137634ff4186 | 265 | memcpy( ik, fk, 16 ); |
ansond | 0:137634ff4186 | 266 | } |
ansond | 0:137634ff4186 | 267 | |
ansond | 0:137634ff4186 | 268 | /* |
ansond | 0:137634ff4186 | 269 | * Key expansion, 128-bit case |
ansond | 0:137634ff4186 | 270 | */ |
ansond | 0:137634ff4186 | 271 | static void aesni_setkey_enc_128( unsigned char *rk, |
ansond | 0:137634ff4186 | 272 | const unsigned char *key ) |
ansond | 0:137634ff4186 | 273 | { |
ansond | 0:137634ff4186 | 274 | asm( "movdqu (%1), %%xmm0 \n\t" // copy the original key |
ansond | 0:137634ff4186 | 275 | "movdqu %%xmm0, (%0) \n\t" // as round key 0 |
ansond | 0:137634ff4186 | 276 | "jmp 2f \n\t" // skip auxiliary routine |
ansond | 0:137634ff4186 | 277 | |
ansond | 0:137634ff4186 | 278 | /* |
ansond | 0:137634ff4186 | 279 | * Finish generating the next round key. |
ansond | 0:137634ff4186 | 280 | * |
ansond | 0:137634ff4186 | 281 | * On entry xmm0 is r3:r2:r1:r0 and xmm1 is X:stuff:stuff:stuff |
ansond | 0:137634ff4186 | 282 | * with X = rot( sub( r3 ) ) ^ RCON. |
ansond | 0:137634ff4186 | 283 | * |
ansond | 0:137634ff4186 | 284 | * On exit, xmm0 is r7:r6:r5:r4 |
ansond | 0:137634ff4186 | 285 | * with r4 = X + r0, r5 = r4 + r1, r6 = r5 + r2, r7 = r6 + r3 |
ansond | 0:137634ff4186 | 286 | * and those are written to the round key buffer. |
ansond | 0:137634ff4186 | 287 | */ |
ansond | 0:137634ff4186 | 288 | "1: \n\t" |
ansond | 0:137634ff4186 | 289 | "pshufd $0xff, %%xmm1, %%xmm1 \n\t" // X:X:X:X |
ansond | 0:137634ff4186 | 290 | "pxor %%xmm0, %%xmm1 \n\t" // X+r3:X+r2:X+r1:r4 |
ansond | 0:137634ff4186 | 291 | "pslldq $4, %%xmm0 \n\t" // r2:r1:r0:0 |
ansond | 0:137634ff4186 | 292 | "pxor %%xmm0, %%xmm1 \n\t" // X+r3+r2:X+r2+r1:r5:r4 |
ansond | 0:137634ff4186 | 293 | "pslldq $4, %%xmm0 \n\t" // etc |
ansond | 0:137634ff4186 | 294 | "pxor %%xmm0, %%xmm1 \n\t" |
ansond | 0:137634ff4186 | 295 | "pslldq $4, %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 296 | "pxor %%xmm1, %%xmm0 \n\t" // update xmm0 for next time! |
ansond | 0:137634ff4186 | 297 | "add $16, %0 \n\t" // point to next round key |
ansond | 0:137634ff4186 | 298 | "movdqu %%xmm0, (%0) \n\t" // write it |
ansond | 0:137634ff4186 | 299 | "ret \n\t" |
ansond | 0:137634ff4186 | 300 | |
ansond | 0:137634ff4186 | 301 | /* Main "loop" */ |
ansond | 0:137634ff4186 | 302 | "2: \n\t" |
ansond | 0:137634ff4186 | 303 | AESKEYGENA xmm0_xmm1 ",0x01 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 304 | AESKEYGENA xmm0_xmm1 ",0x02 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 305 | AESKEYGENA xmm0_xmm1 ",0x04 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 306 | AESKEYGENA xmm0_xmm1 ",0x08 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 307 | AESKEYGENA xmm0_xmm1 ",0x10 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 308 | AESKEYGENA xmm0_xmm1 ",0x20 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 309 | AESKEYGENA xmm0_xmm1 ",0x40 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 310 | AESKEYGENA xmm0_xmm1 ",0x80 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 311 | AESKEYGENA xmm0_xmm1 ",0x1B \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 312 | AESKEYGENA xmm0_xmm1 ",0x36 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 313 | : |
ansond | 0:137634ff4186 | 314 | : "r" (rk), "r" (key) |
ansond | 0:137634ff4186 | 315 | : "memory", "cc", "0" ); |
ansond | 0:137634ff4186 | 316 | } |
ansond | 0:137634ff4186 | 317 | |
ansond | 0:137634ff4186 | 318 | /* |
ansond | 0:137634ff4186 | 319 | * Key expansion, 192-bit case |
ansond | 0:137634ff4186 | 320 | */ |
ansond | 0:137634ff4186 | 321 | static void aesni_setkey_enc_192( unsigned char *rk, |
ansond | 0:137634ff4186 | 322 | const unsigned char *key ) |
ansond | 0:137634ff4186 | 323 | { |
ansond | 0:137634ff4186 | 324 | asm( "movdqu (%1), %%xmm0 \n\t" // copy original round key |
ansond | 0:137634ff4186 | 325 | "movdqu %%xmm0, (%0) \n\t" |
ansond | 0:137634ff4186 | 326 | "add $16, %0 \n\t" |
ansond | 0:137634ff4186 | 327 | "movq 16(%1), %%xmm1 \n\t" |
ansond | 0:137634ff4186 | 328 | "movq %%xmm1, (%0) \n\t" |
ansond | 0:137634ff4186 | 329 | "add $8, %0 \n\t" |
ansond | 0:137634ff4186 | 330 | "jmp 2f \n\t" // skip auxiliary routine |
ansond | 0:137634ff4186 | 331 | |
ansond | 0:137634ff4186 | 332 | /* |
ansond | 0:137634ff4186 | 333 | * Finish generating the next 6 quarter-keys. |
ansond | 0:137634ff4186 | 334 | * |
ansond | 0:137634ff4186 | 335 | * On entry xmm0 is r3:r2:r1:r0, xmm1 is stuff:stuff:r5:r4 |
ansond | 0:137634ff4186 | 336 | * and xmm2 is stuff:stuff:X:stuff with X = rot( sub( r3 ) ) ^ RCON. |
ansond | 0:137634ff4186 | 337 | * |
ansond | 0:137634ff4186 | 338 | * On exit, xmm0 is r9:r8:r7:r6 and xmm1 is stuff:stuff:r11:r10 |
ansond | 0:137634ff4186 | 339 | * and those are written to the round key buffer. |
ansond | 0:137634ff4186 | 340 | */ |
ansond | 0:137634ff4186 | 341 | "1: \n\t" |
ansond | 0:137634ff4186 | 342 | "pshufd $0x55, %%xmm2, %%xmm2 \n\t" // X:X:X:X |
ansond | 0:137634ff4186 | 343 | "pxor %%xmm0, %%xmm2 \n\t" // X+r3:X+r2:X+r1:r4 |
ansond | 0:137634ff4186 | 344 | "pslldq $4, %%xmm0 \n\t" // etc |
ansond | 0:137634ff4186 | 345 | "pxor %%xmm0, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 346 | "pslldq $4, %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 347 | "pxor %%xmm0, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 348 | "pslldq $4, %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 349 | "pxor %%xmm2, %%xmm0 \n\t" // update xmm0 = r9:r8:r7:r6 |
ansond | 0:137634ff4186 | 350 | "movdqu %%xmm0, (%0) \n\t" |
ansond | 0:137634ff4186 | 351 | "add $16, %0 \n\t" |
ansond | 0:137634ff4186 | 352 | "pshufd $0xff, %%xmm0, %%xmm2 \n\t" // r9:r9:r9:r9 |
ansond | 0:137634ff4186 | 353 | "pxor %%xmm1, %%xmm2 \n\t" // stuff:stuff:r9+r5:r10 |
ansond | 0:137634ff4186 | 354 | "pslldq $4, %%xmm1 \n\t" // r2:r1:r0:0 |
ansond | 0:137634ff4186 | 355 | "pxor %%xmm2, %%xmm1 \n\t" // xmm1 = stuff:stuff:r11:r10 |
ansond | 0:137634ff4186 | 356 | "movq %%xmm1, (%0) \n\t" |
ansond | 0:137634ff4186 | 357 | "add $8, %0 \n\t" |
ansond | 0:137634ff4186 | 358 | "ret \n\t" |
ansond | 0:137634ff4186 | 359 | |
ansond | 0:137634ff4186 | 360 | "2: \n\t" |
ansond | 0:137634ff4186 | 361 | AESKEYGENA xmm1_xmm2 ",0x01 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 362 | AESKEYGENA xmm1_xmm2 ",0x02 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 363 | AESKEYGENA xmm1_xmm2 ",0x04 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 364 | AESKEYGENA xmm1_xmm2 ",0x08 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 365 | AESKEYGENA xmm1_xmm2 ",0x10 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 366 | AESKEYGENA xmm1_xmm2 ",0x20 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 367 | AESKEYGENA xmm1_xmm2 ",0x40 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 368 | AESKEYGENA xmm1_xmm2 ",0x80 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 369 | |
ansond | 0:137634ff4186 | 370 | : |
ansond | 0:137634ff4186 | 371 | : "r" (rk), "r" (key) |
ansond | 0:137634ff4186 | 372 | : "memory", "cc", "0" ); |
ansond | 0:137634ff4186 | 373 | } |
ansond | 0:137634ff4186 | 374 | |
ansond | 0:137634ff4186 | 375 | /* |
ansond | 0:137634ff4186 | 376 | * Key expansion, 256-bit case |
ansond | 0:137634ff4186 | 377 | */ |
ansond | 0:137634ff4186 | 378 | static void aesni_setkey_enc_256( unsigned char *rk, |
ansond | 0:137634ff4186 | 379 | const unsigned char *key ) |
ansond | 0:137634ff4186 | 380 | { |
ansond | 0:137634ff4186 | 381 | asm( "movdqu (%1), %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 382 | "movdqu %%xmm0, (%0) \n\t" |
ansond | 0:137634ff4186 | 383 | "add $16, %0 \n\t" |
ansond | 0:137634ff4186 | 384 | "movdqu 16(%1), %%xmm1 \n\t" |
ansond | 0:137634ff4186 | 385 | "movdqu %%xmm1, (%0) \n\t" |
ansond | 0:137634ff4186 | 386 | "jmp 2f \n\t" // skip auxiliary routine |
ansond | 0:137634ff4186 | 387 | |
ansond | 0:137634ff4186 | 388 | /* |
ansond | 0:137634ff4186 | 389 | * Finish generating the next two round keys. |
ansond | 0:137634ff4186 | 390 | * |
ansond | 0:137634ff4186 | 391 | * On entry xmm0 is r3:r2:r1:r0, xmm1 is r7:r6:r5:r4 and |
ansond | 0:137634ff4186 | 392 | * xmm2 is X:stuff:stuff:stuff with X = rot( sub( r7 )) ^ RCON |
ansond | 0:137634ff4186 | 393 | * |
ansond | 0:137634ff4186 | 394 | * On exit, xmm0 is r11:r10:r9:r8 and xmm1 is r15:r14:r13:r12 |
ansond | 0:137634ff4186 | 395 | * and those have been written to the output buffer. |
ansond | 0:137634ff4186 | 396 | */ |
ansond | 0:137634ff4186 | 397 | "1: \n\t" |
ansond | 0:137634ff4186 | 398 | "pshufd $0xff, %%xmm2, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 399 | "pxor %%xmm0, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 400 | "pslldq $4, %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 401 | "pxor %%xmm0, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 402 | "pslldq $4, %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 403 | "pxor %%xmm0, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 404 | "pslldq $4, %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 405 | "pxor %%xmm2, %%xmm0 \n\t" |
ansond | 0:137634ff4186 | 406 | "add $16, %0 \n\t" |
ansond | 0:137634ff4186 | 407 | "movdqu %%xmm0, (%0) \n\t" |
ansond | 0:137634ff4186 | 408 | |
ansond | 0:137634ff4186 | 409 | /* Set xmm2 to stuff:Y:stuff:stuff with Y = subword( r11 ) |
ansond | 0:137634ff4186 | 410 | * and proceed to generate next round key from there */ |
ansond | 0:137634ff4186 | 411 | AESKEYGENA xmm0_xmm2 ",0x00 \n\t" |
ansond | 0:137634ff4186 | 412 | "pshufd $0xaa, %%xmm2, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 413 | "pxor %%xmm1, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 414 | "pslldq $4, %%xmm1 \n\t" |
ansond | 0:137634ff4186 | 415 | "pxor %%xmm1, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 416 | "pslldq $4, %%xmm1 \n\t" |
ansond | 0:137634ff4186 | 417 | "pxor %%xmm1, %%xmm2 \n\t" |
ansond | 0:137634ff4186 | 418 | "pslldq $4, %%xmm1 \n\t" |
ansond | 0:137634ff4186 | 419 | "pxor %%xmm2, %%xmm1 \n\t" |
ansond | 0:137634ff4186 | 420 | "add $16, %0 \n\t" |
ansond | 0:137634ff4186 | 421 | "movdqu %%xmm1, (%0) \n\t" |
ansond | 0:137634ff4186 | 422 | "ret \n\t" |
ansond | 0:137634ff4186 | 423 | |
ansond | 0:137634ff4186 | 424 | /* |
ansond | 0:137634ff4186 | 425 | * Main "loop" - Generating one more key than necessary, |
ansond | 0:137634ff4186 | 426 | * see definition of aes_context.buf |
ansond | 0:137634ff4186 | 427 | */ |
ansond | 0:137634ff4186 | 428 | "2: \n\t" |
ansond | 0:137634ff4186 | 429 | AESKEYGENA xmm1_xmm2 ",0x01 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 430 | AESKEYGENA xmm1_xmm2 ",0x02 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 431 | AESKEYGENA xmm1_xmm2 ",0x04 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 432 | AESKEYGENA xmm1_xmm2 ",0x08 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 433 | AESKEYGENA xmm1_xmm2 ",0x10 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 434 | AESKEYGENA xmm1_xmm2 ",0x20 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 435 | AESKEYGENA xmm1_xmm2 ",0x40 \n\tcall 1b \n\t" |
ansond | 0:137634ff4186 | 436 | : |
ansond | 0:137634ff4186 | 437 | : "r" (rk), "r" (key) |
ansond | 0:137634ff4186 | 438 | : "memory", "cc", "0" ); |
ansond | 0:137634ff4186 | 439 | } |
ansond | 0:137634ff4186 | 440 | |
ansond | 0:137634ff4186 | 441 | /* |
ansond | 0:137634ff4186 | 442 | * Key expansion, wrapper |
ansond | 0:137634ff4186 | 443 | */ |
ansond | 0:137634ff4186 | 444 | int aesni_setkey_enc( unsigned char *rk, |
ansond | 0:137634ff4186 | 445 | const unsigned char *key, |
ansond | 0:137634ff4186 | 446 | size_t bits ) |
ansond | 0:137634ff4186 | 447 | { |
ansond | 0:137634ff4186 | 448 | switch( bits ) |
ansond | 0:137634ff4186 | 449 | { |
ansond | 0:137634ff4186 | 450 | case 128: aesni_setkey_enc_128( rk, key ); break; |
ansond | 0:137634ff4186 | 451 | case 192: aesni_setkey_enc_192( rk, key ); break; |
ansond | 0:137634ff4186 | 452 | case 256: aesni_setkey_enc_256( rk, key ); break; |
ansond | 0:137634ff4186 | 453 | default : return( POLARSSL_ERR_AES_INVALID_KEY_LENGTH ); |
ansond | 0:137634ff4186 | 454 | } |
ansond | 0:137634ff4186 | 455 | |
ansond | 0:137634ff4186 | 456 | return( 0 ); |
ansond | 0:137634ff4186 | 457 | } |
ansond | 0:137634ff4186 | 458 | |
ansond | 0:137634ff4186 | 459 | #endif /* POLARSSL_HAVE_X86_64 */ |
ansond | 0:137634ff4186 | 460 | |
ansond | 0:137634ff4186 | 461 | #endif /* POLARSSL_AESNI_C */ |
ansond | 0:137634ff4186 | 462 |