/* sha1edp.s : this file provides optimized x86_64 and i386 implementation of the sha1 function CoreOS - vector and numerics group cclee 6-21-10 The implementation is based on the principle described in an Intel online article "Improving the Performance of the Secure Hash Algorithm (SHA-1)" http://software.intel.com/en-us/articles/improving-the-performance-of-the-secure-hash-algorithm-1/ Update HASH[] by processing a one 64-byte block in MESSAGE[] can be represented by the following C function void SHA1( int HASH[], int MESSAGE[] ) { int A[81], B[81], C[81], D[81], E[81]; int W[80]; int i, FN; A[0] = HASH[0]; B[0] = HASH[1]; C[0] = HASH[2]; D[0] = HASH[3]; E[0] = HASH[4]; for ( i=0; i<80; ++i ) { if ( i < 16 ) W[i] = BIG_ENDIAN_LOAD( MESSAGE[i] ); else W[i] = ROTATE_LEFT( W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16], 1 ); FN = F( i, B[i], C[i], D[i] ); A[i+1] = FN + E[i] + ROTATE_LEFT( A[i], 5 ) + W[i] + K(i); B[i+1] = A[i]; C[i+1] = ROTATE_LEFT( B[i], 30 ); D[i+1] = C[i]; E[i+1] = D[i]; } HASH[0] += A[80]; HASH[1] += B[80]; HASH[2] += C[80]; HASH[3] += D[80]; HASH[4] += E[80]; } For i=0:15, W[i] is simply big-endian loading of MESSAGE[i]. For i=16:79, W[i] is updated according to W[i] = ROTATE_LEFT( W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16], 1 ); The approach (by Dean Gaudet) can be used to vectorize the computation of W[i] for i=16:79, 1. done on 4 consequtive W[i] values in a single XMM register W[i ] = (W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16]) rol 1 W[i+1] = (W[i-2] ^ W[i-7] ^ W[i-13] ^ W[i-15]) rol 1 W[i+2] = (W[i-1] ^ W[i-6] ^ W[i-12] ^ W[i-14]) rol 1 W[i+3] = ( 0 ^ W[i-5] ^ W[i-11] ^ W[i-13]) rol 1 2. this additional calculation unfortunately requires many additional operations W[i+3] ^= W[i] rol 1 3. once we have 4 W[i] values in XMM we can also add four K values with one instruction W[i:i+3] += {K,K,K,K} Let W0 = {W[i] W[i+1] W[i+2] W[i+3]} be the current W-vector to be computed, W4 = {W[i-4] W[i-3] W[i-2] W[i-1]} be the previous vector, and so on The Dean Gaudet approach can be expressed as 1. W0 = rotate_left(left_shift(W4,32) ^ W8 ^ left_shift(concatenate(W16,W12),64) ^ W16,1); 2. W[i+3] ^= W[i] rol 1 3. W0 += {K,K,K,K} For i>=32, the Intel online article suggests that (using a basic identity (X rol 1) rol 1 = X rol 2) the update equation is equivalent to 1. W0 = rotate_left(left_shift(concatenate(W8,W4),64) ^ W16 ^ W28 ^ W32, 2); Note: 1. In total, we need 8 16-byte registers or memory for W0,W4,...,W28. W0 and W32 can be the same register or memory. 2. The registers are used in a circular buffering mode. For example, we start with W28,W24,...,W0 (with W0 indicating the most recent 16-byte) i=0, W28,W24,...,W0 i=4, W24,W20,...,W28 i=8, W20,W16,...,W24 . . and so forth. 3. 2 ssse3 instructions are used in the Intel article, pshufb and palignr. a. pshufb is used to simplify the BIG_ENDIAN_LOAD operation b. palignr is used to simplify the computation of left_shift(concatenate(W12,W8),64) 4. we probe __cpu_capabilities to detect ssse3 support and dispatch code with ssse3 support when available. If ssse3 is not supported, a suboptimal code (pshufb and palignr workaround) is dispatched. */ /* the code can be compiled into single block (64 bytes) per call mode by setting Multiple_blocks to 0 */ #define Multiple_Blocks 1 #if defined (__x86_64__) || defined(__i386__) // x86_64 or i386 architectures #if defined(__x86_64__) // set up for x86_64 #define stack_size (8+16*11+16*4) // 8 (alignedment) + x0-x10 + 4 128-bits for intermediate WK(t) storage #define sp %rsp // unifying architectural stack pointer representation #define ctx %rdi // 1st input argument, will move to HASH_PTR (%r9) #define buf %rsi // 2nd input argument, will move to BUFFER_PTR (%r10) #define cnt %r11 // will copy from the 3rd input argument (%rdx) #define K_BASE %r8 // an aligned pointer to point to shufb reference numbers of table of K values #define HASH_PTR %r9 // pointer to Hash values (A,B,C,D,E) #define BUFFER_PTR %r10 // pointer to input blocks #else // !__x86_64__ // set up for i386 #define stack_size (12+16*2+16*11+16*4) // 12-bytes (alignment) + extra 2 + 3 (W24/W28/XMM_SHUFB_BSWAP) + 8 (xmm0-xmm7) + 4 (WK(t)) #define sp %esp // unifying architectural stack pointer representation #define HASH_PTR stack_size+16+4(sp) // use 1st input argument from caller function, 16 for (esi/edi/ebx/ebp) #define BUFFER_PTR stack_size+16+8(sp) // use 2nd input argument from caller function #define cnt stack_size+16+12(sp) // use 3rd input argument from caller function #define K_BASE stack_size-4(sp) // use for K_BASE #endif // __x86_64__ // symbolizing registers or stack memory with algorithmic variables W0,W4,...,W28 + W_TMP, W_TMP2, and XMM_SHUFB_BSWAP for code with ssse3 support #define W_TMP %xmm0 #define W_TMP2 %xmm1 #define W0 %xmm2 #define W4 %xmm3 #define W8 %xmm4 #define W12 %xmm5 #define W16 %xmm6 #define W20 %xmm7 #if defined(__x86_64__) #define W24 %xmm8 #define W28 %xmm9 #define XMM_SHUFB_BSWAP %xmm10 // used only when ssse3 is supported #else // defined (__i386__) #define W24 12*16(sp) #define W28 13*16(sp) #define XMM_SHUFB_BSWAP 14*16(sp) // used only when ssse3 is supported #endif #define xmov movaps // aligned 16-byte move #define xmovu movups // unaligned 16-byte move // intermediate hash variables #define A %ecx #define B %esi #define C %edi #define D %ebp #define E %edx // temp variables #define T1 %eax #define T2 %ebx #define WK(t) (t&15)*4(sp) // int F1(int B, int C, int D) { return (D ^ ( B & (C ^ D)); } // result in T1 .macro F1 mov $1, T1 xor $2, T1 and $0, T1 xor $2, T1 .endm // int F2(int B, int C, int D) { return (D ^ B ^ C); } // result in T1 .macro F2 mov $2, T1 xor $1, T1 xor $0, T1 .endm // int F3(int B, int C, int D) { return (B & C) | (D & (B ^ C)); } // result in T1 .macro F3 mov $1, T1 mov $0, T2 or $0, T1 and $1, T2 and $2, T1 or T2, T1 .endm // for i=60:79, F4 is identical to F2 #define F4 F2 /* i=0:15, W[i] = BIG_ENDIAN_LOAD(MESSAGE[i]); with ssse3 support, this is achived via for (i=0;i<16;i+=4) { 1. W_TMP = new 16 bytes from MESSAGE[] 2. W_TMP = pshufb(W_TMP, XMM_SHUFB_BSWAP); save to W circular buffer for updating W 3. WTMP += {K,K,K,K}; 4. save quadruple W[i]+K[i] = W_TMP in the stack memory; } each step is represented in one of the following 4 macro definitions */ .macro W_PRECALC_00_15_0_ssse3 // input argument $0 : 0/4/8/12 #if defined (__x86_64__) // BUFFER_PTR is already an address register in x86_64 xmovu $0*4(BUFFER_PTR), W_TMP // read 16-bytes into W_TMP, BUFFER_PTR possibly not 16-byte aligned #else // BUFFER_PTR is from the argument set up in the caller mov BUFFER_PTR, T1 // T1 = BUFFER_PTR xmovu $0*4(T1), W_TMP // read 16-bytes into W_TMP, BUFFER_PTR possibly not 16-byte aligned #endif .endm .macro W_PRECALC_00_15_1_ssse3 // input argument $0 : current 16-bytes in the circular buffer, one of W0,W4,W8,...,W28 pshufb XMM_SHUFB_BSWAP, W_TMP // convert W_TMP from little-endian into big-endian xmov W_TMP, $0 // save W_TMP in the circular buffer .endm .macro W_PRECALC_00_15_2 // K_BASE points to the current K quadruple. #if defined (__x86_64__) // K_BASE is already an address register in x86_64 paddd (K_BASE), W_TMP // W_TMP += {K,K,K,K}; #else // K_BASE is previously set up in the stack memory mov K_BASE, T1 // T1 = K_BASE paddd (T1), W_TMP // W_TMP += {K,K,K,K}; #endif .endm .macro W_PRECALC_00_15_3 xmov W_TMP, WK($0&~3) // save quadruple W[i]+K in the stack memory, which would be used later for updating the hashes A/B/C/D/E .endm /* without ssse3 support, steps 1 and 2 need to be modified 1. sequentially load 4 words into T1, bswap T1, and save it to 4-bytes in the stack space 2. load the 16-bytes from the aligned stack memory into W_TMP */ .macro W_PRECALC_00_15_0_nossse3 // input argument $0 : 0/4/8/12 #if defined (__x86_64__) #define BUFFERP BUFFER_PTR #else mov BUFFER_PTR, T2 // copy BUFFER_PTR (from caller 2nd argument) to T2 #define BUFFERP T2 #endif // load 1st word, bswap it, save it to stack mov $0*4(BUFFERP), T1 bswap T1 mov T1, 14*16(sp) // load 2nd word, bswap it, save it to stack mov 4+$0*4(BUFFERP), T1 bswap T1 mov T1, 4+14*16(sp) // load 3rd word, bswap it, save it to stack mov 8+$0*4(BUFFERP), T1 bswap T1 mov T1, 8+14*16(sp) // load 4th word, bswap it, save it to stack mov 12+$0*4(BUFFERP), T1 bswap T1 mov T1, 12+14*16(sp) .endm .macro W_PRECALC_00_15_1_nossse3 // input argument $0 : current 16-bytes in the circular buffer, one of W0,W4,W8,...,W28 xmov 14*16(sp), W_TMP // load the bswapped 16-bytes from the aligned stack memory xmov W_TMP, $0 // save W = W_TMP in the circular buffer .endm // rounds 16-31 compute W[0] using the vectorization approach by Dean Gaudet /* W[i ] = (W[i-3] ^ W[i-8] ^ W[i-14] ^ W[i-16]) rol 1 W[i+1] = (W[i-2] ^ W[i-7] ^ W[i-13] ^ W[i-15]) rol 1 W[i+2] = (W[i-1] ^ W[i-6] ^ W[i-12] ^ W[i-14]) rol 1 W[i+3] = ( 0 ^ W[i-5] ^ W[i-11] ^ W[i-13]) rol 1 W[i+3] ^= W[i] rol 1; // this W[i] is already rol by 1, if we are taking from the intial W before rol 1, we should rol this by 2 The operation (updating W and W+K) is scheduled as and divided into 4 steps 0. W_tmp = W3; W = W14 ^ W8 1. W = W3 ^ W8 ^ W14 ^ W16; W_TMP = W; W_TMP2 = (W[i] 0 0 0); 2. W_TMP = (W3 ^ W8 ^ W14 ^ W16) rol 1; split (W[i] 0 0 0) rol 2 in W_TMP2 and W 3. W = W_TMP = W_TMP ^ W_TMP2 ^ W = (W3 ^ W8 ^ W14 ^ W16) rol 1 ^ (W[i] 0 0 0) rol 2; WK = W _TMP+K; */ .macro W_PRECALC_16_31_0_ssse3 // input arguments : W16,W12,W8,W4,W xmov $1, $4 // W = W12 palignr $$8, $0, $4 // W = W14 xmov $3, W_TMP // W_TMP = W4 psrldq $$4, W_TMP // W_TMP = W3 pxor $2, $4 // W = W8 ^ W14 .endm .macro W_PRECALC_16_31_1 // input arguments : W16,W pxor $0, W_TMP // W_TMP = W3 ^ W16 pxor W_TMP, $1 // W = W3 ^ W16 ^ W8 ^ W14 xmov $1, W_TMP2 // W_TMP2 = W3 ^ W16 ^ W8 ^ W14 xmov $1, W_TMP // W_TMP = W3 ^ W16 ^ W8 ^ W14 pslldq $$12, W_TMP2 // W_TMP2 = (W[i] 0 0 0) .endm .macro W_PRECALC_16_31_2 // input argument : W psrld $$31, $0 // (W3 ^ W16 ^ W8 ^ W14)>>31 pslld $$1, W_TMP // (W3 ^ W16 ^ W8 ^ W14)<<1 por $0, W_TMP // W_TMP = (W3 ^ W16 ^ W8 ^ W14) rol 1 xmov W_TMP2, $0 // copy W[i] at location of W[i+3] psrld $$30, W_TMP2 // W_TMP2 = W[i] lower 2 bits after rol 2 pslld $$2, $0 // W = W[i] higher 30 bits after rol 2 .endm .macro W_PRECALC_16_31_3 // input arguments: W, i, K_XMM #if defined (__i386__) mov K_BASE, T1 // K_BASE is store in the stack memory for i386 #endif pxor $0, W_TMP pxor W_TMP2, W_TMP // W_TMP = (W3 ^ W16 ^ W8 ^ W14) rol 1 ^ (W[i] 0 0 0) rol 2 xmov W_TMP, $0 // save W = W_TMP in the W circular buffer #if defined (__x86_64__) paddd $2(K_BASE), W_TMP // W+K #else paddd $2(T1), W_TMP // W+K #endif xmov W_TMP, WK($1&~3) // save WK = W+K for later update of the hashes A/B/C/D/E .endm // the following is a variant of W_PRECALC_16_31_0_ssse3 to be used for system without ssse3, palignr is replaced with 4 instructions .macro W_PRECALC_16_31_0_nossse3 // input arguments : W16,W12,W8,W4,W xmov $1, $4 // W = W12 = (w9 w10 w11 w12) // the following is a wrokaround for palignr xmov $0, W_TMP // W16 = (w13 w14 w15 w16) pslldq $$8, $4 // shift left to make (w11 w12 0 0) psrldq $$8, W_TMP // shift right to make (0 0 w13 w14) por W_TMP, $4 // W = W14 = (w11 w12 w13 w14) xmov $3, W_TMP // W_TMP = W4 = (w1 w2 w3 w4) psrldq $$4, W_TMP // W_TMP = W3 = (0 w1 w2 w3) pxor $2, $4 // W = W8 ^ W14 .endm /* rounds 32-79 compute W und W+K iusing the vectorization approach from the Intel article W = rotate_left(left_shift(concatenate(W8,W4),64) ^ W16 ^ W28 ^ W32, 2); where left_shift(concatenate(W8,W4),64) is equivalent to W6. Note also that W32 and W use the same register. 0. W_tmp = W6; W = W28 ^ W32; 1. W = W_tmp = W6 ^ W16 ^ W28 ^ W32; 2. W_tmp = (W6 ^ W16 ^ W28 ^ W32) rol 2; 3. W = W_Tmp; WK = W_tmp + K; */ .macro W_PRECALC_32_79_0_ssse3 // inputr arguments : W28,W8,W4,W xmov $2, W_TMP // (w1 w2 w3 w4) pxor $0, $3 // W = W28 ^ W32; palignr $$8, $1, W_TMP // W_tmp = (w3 w4 w5 w6) = W6; .endm // the following is a variant and will be used for system without ssse3 support .macro W_PRECALC_32_79_0_nossse3 // input arguments : W28,W8,W4,W xmov $2, W_TMP // (w1 w2 w3 w4) xmov $1, W_TMP2 // (w5 w6 w7 w8) pxor $0, $3 // W = W28 ^ W32 pslldq $$8, W_TMP // (w3 w4 0 0) psrldq $$8, W_TMP2 // (0 0 w5 w6) por W_TMP2, W_TMP // W_tmp = (w3 w4 w5 w6) = W6 .endm // this is a variant of W_PRECALC_32_79_0_ssse3 for i386 (as W24/W28 are stored in memory, not in registers) .macro W_PRECALC_32_79_0_i386_ssse3 // input arguments : W28,W8,W4,W xmov $3, W_TMP // W32 pxor $0, W_TMP // W28 ^ W32 xmov W_TMP, $3 // W = W28 ^ W32; xmov $2, W_TMP // W4 palignr $$8, $1, W_TMP // W_tmp = (w3 w4 w5 w6) = W6; .endm // this is a variant of W_PRECALC_32_79_0_nossse3 for i386 (as W24/W28 are stored in memory, not in registers) .macro W_PRECALC_32_79_0_i386_nossse3 // input arguments : W28,W8,W4,W xmov $3, W_TMP // W32 pxor $0, W_TMP // W28 ^ W32 xmov W_TMP, $3 // W = W28 ^ W32 xmov $2, W_TMP // W4 = (w1 w2 w3 w4) xmov $1, W_TMP2 // W8 = (w5 w6 w7 w8) pslldq $$8, W_TMP // (w3 w4 0 0) psrldq $$8, W_TMP2 // (0 0 w5 w6) por W_TMP2, W_TMP // W_tmp = (w3 w4 w5 w6) = W6 .endm .macro W_PRECALC_32_79_1 // input arguments : W16,W pxor $0, W_TMP // W_tmp = W6 ^ W16 pxor $1, W_TMP // W_tmp = W6 ^ W16 ^ W28 ^ W32 xmov W_TMP, $1 // W = W_tmp = W6 ^ W16 ^ W28 ^ W32 .endm .macro W_PRECALC_32_79_2 // input argument : W psrld $$30, $0 // W >> 30 pslld $$2, W_TMP // W << 2 por $0, W_TMP // W_tmp = (W6 ^ W16 ^ W28 ^ W32) rol 2 .endm // this is a variant of W_PRECALC_32_79_2 for i386 (as W24/W28 are stored in memory, not in registers) // this should be used when the input is either W24 or W28 on i386 architecture .macro W_PRECALC_32_79_2_i386 // input argument : W xmov $0, W_TMP2 // W psrld $$30, W_TMP2 // W >> 30 xmov W_TMP2, $0 // save (W >> 30) at W pslld $$2, W_TMP // W_tmp << 2 por $0, W_TMP // W_tmp = (W6 ^ W16 ^ W28 ^ W32) rol 2 .endm .macro W_PRECALC_32_79_3 // input argument W, i, K_XMM #if defined (__x86_64__) xmov W_TMP, $0 // W = (W6 ^ W16 ^ W28 ^ W32) rol 2 paddd $2(K_BASE), W_TMP // W + K xmov W_TMP, WK($1&~3) // write W+K #else mov K_BASE, T1 // T1 = K_BASE (which is in the caller argument) xmov W_TMP, $0 // W = (W6 ^ W16 ^ W28 ^ W32) rol 2 paddd $2(T1), W_TMP // W_tmp = W + K xmov W_TMP, WK($1&~3) // write WK #endif .endm /* The hash update operation is completed by the following statements. A[i+1] = FN + E[i] + ROTATE_LEFT( A[i], 5 ) + WK(i); B[i+1] = A[i]; C[i+1] = ROTATE_LEFT( B[i], 30 ); D[i+1] = C[i]; E[i+1] = D[i]; Suppose we start with A0,B0,C0,D0,E0. The 1st iteration can be expressed as follows: A1 = FN + E0 + rol(A0,5) + WK; B1 = A0; C1 = rol(B0, 30); D1 = C0; E1 = D0; to avoid excessive memory movement between registers, 1. A1 = FN + E0 + rol(A0,5) + WK; can be temporarily saved in E0, 2. C1 = rol(B0,30) can be temporarily saved in B0. Therefore, ignoring the time index, the update operation is equivalent to 1. E = FN(B,C,D) + E + rol(A,5) + WK(i) 2. B = rol(B,30) 3. the hashes are now stored in the order of E,A,B,C,D To pack 2 hash update operations in 1 iteration, starting with A,B,C,D,E 1. E = FN(B,C,D) + E + rol(A,5) + WK(i) 2. B = rol(B,30) // now the hashes are in the order of E,A,B,C,D 3. D = FN(A,B,C) + D + rol(E,5) + WK(i+1) 4. A = rol(A,30) // now the hashes are in the order of D,E,A,B,C These operations are distributed into the following 2 macro definitions RR0 and RR1. */ .macro RR0 // input arguments : FN, A, B, C, D, E, i $0 $2, $3, $4 // T1 = FN(B,C,D) add WK($6), $5 // E + WK(i) rol $$30, $2 // B = rol(B,30) mov $1, T2 // T2 = A add WK($6+1), $4 // D + WK(i+1) rol $$5, T2 // rol(A,5) add T1, $5 // E = FN(B,C,D) + E + WK(i) .endm .macro RR1 add $5, T2 // T2 = FN(B,C,D) + E + rol(A,5) + WK(i) mov T2, $5 // E = FN(B,C,D) + E + rol(A,5) + WK(i) rol $$5, T2 // rol(E,5) add T2, $4 // D + WK(i+1) + rol(E,5) $0 $1, $2, $3 // FN(A,B,C) add T1, $4 // D = FN(A,B,C) + D + rol(E,5) + WK(i+1) rol $$30, $1 // A = rol(A,30) .endm /* The following macro definitions are used to expand code for the per-block sha1 operation. INITIAL_W_PRECALC_ssse3 : BIG_ENDIAN_LOAD(64 bytes block) into W (i=0:15) and store W+K into the stack memory INTERNAL_ssse3 : updating W (16:79) and update the digests A/B/C/D/E (i=0:63, based on W+K stored in the stack memory) ENDING : finishing up update the digests A/B/C/D/E (i=64:79) For multiple-block sha1 operation (Multiple_Blocks = 1), INITIAL_W_PRECALC_ssse3 and ENDING are combined into 1 macro definition for software pipeling. SOFTWARE_PIPELINING_ssse3 : BIG_ENDIAN_LOAD(64 bytes block) into W (i=0:15) and store W+K into the stack, and finishing up update the digests A/B/C/D/E (i=64:79) assume cnt (the number of blocks) >= 1, the main code body should look like INITIAL_W_PRECALC_ssse3 // W = big_endian_load and pre-compute W+K (i=0:15) do { INTERNAL_ssse3 // update W(i=16:79), and update hash digests A/B/C/D/E (i=0:63) cnt--; if (cnt==0) break; BUFFER_PTR += 64; SOFTWARE_PIPELINING_ssse3; // update hash digests A/B/C/D/E (i=64:79) + W = big_endian_load and pre-compute W+K (i=0:15) } ENDING // update hash digests A/B/C/D/E (i=64:79) */ #define W_PRECALC_00_15_0 W_PRECALC_00_15_0_ssse3 #define W_PRECALC_00_15_1 W_PRECALC_00_15_1_ssse3 #define W_PRECALC_16_31_0 W_PRECALC_16_31_0_ssse3 #define W_PRECALC_32_79_0 W_PRECALC_32_79_0_ssse3 #define W_PRECALC_32_79_0_i386 W_PRECALC_32_79_0_i386_ssse3 .macro INITIAL_W_PRECALC_ssse3 // BIG_ENDIAN_LOAD(64 bytes block) into W (i=0:15) and store W+K into the stack memory // i=0 : W28,W24,W20,W16,W12,W8,W4,W0 W_PRECALC_00_15_0 0 // W_TMP = (BUFFER_PTR) W_PRECALC_00_15_1 W0 // convert W_TMP to big-endian, and save W0 = W_TMP W_PRECALC_00_15_2 // W_TMP = W0 + K W_PRECALC_00_15_3 3 // (sp) = W_TMP = W0 + K // i=4 : W24,W20,W16,W12,W8,W4,W0,W28 W_PRECALC_00_15_0 4 // W_TMP = 16(BUFFER_PTR) W_PRECALC_00_15_1 W28 // convert W_TMP to big-endian, and save W28 = W_TMP W_PRECALC_00_15_2 // W_TMP = W28 + K W_PRECALC_00_15_3 7 // 16(sp) = W_TMP = W28 + K // i=8 : W20,W16,W12,W8,W4,W0,W28,W24 W_PRECALC_00_15_0 8 // W_TMP = 32(BUFFER_PTR) W_PRECALC_00_15_1 W24 // convert W_TMP to big-endian, and save W24 = W_TMP W_PRECALC_00_15_2 // W_TMP = W24 + K W_PRECALC_00_15_3 11 // 32(sp) = W_TMP = W24 + K // i=12 : W16,W12,W8,W4,W0,W28,W24,W20 W_PRECALC_00_15_0 12 // W_TMP = 48(BUFFER_PTR) W_PRECALC_00_15_1 W20 // convert W_TMP to big-endian, and save W20 = W_TMP W_PRECALC_00_15_2 // W_TMP = W20 + K W_PRECALC_00_15_3 15 // 48(sp) = W_TMP = W20 + K .endm .macro INTERNAL_ssse3 // updating W (16:79) and update the digests A/B/C/D/E (i=0:63, based on W+K stored in the stack memory) // i=16 : W12,W8,W4,W0,W28,W24,W20,W16 W_PRECALC_16_31_0 W0,W28,W24,W20,W16 RR0 F1,A,B,C,D,E,0 W_PRECALC_16_31_1 W0,W16 RR1 F1,A,B,C,D,E,0 W_PRECALC_16_31_2 W16 RR0 F1,D,E,A,B,C,2 W_PRECALC_16_31_3 W16, 2, 0 RR1 F1,D,E,A,B,C,2 // i=20 : W8,W4,W0,W28,W24,W20,W16,W12 W_PRECALC_16_31_0 W28,W24,W20,W16,W12 RR0 F1,B,C,D,E,A,4 W_PRECALC_16_31_1 W28,W12 RR1 F1,B,C,D,E,A,4 W_PRECALC_16_31_2 W12 RR0 F1,E,A,B,C,D,6 W_PRECALC_16_31_3 W12, 6, 16 RR1 F1,E,A,B,C,D,6 // i=24 : W4,W0,W28,W24,W20,W16,W12,W8 W_PRECALC_16_31_0 W24,W20,W16,W12,W8 RR0 F1,C,D,E,A,B,8 W_PRECALC_16_31_1 W24,W8 RR1 F1,C,D,E,A,B,8 W_PRECALC_16_31_2 W8 RR0 F1,A,B,C,D,E,10 W_PRECALC_16_31_3 W8,10,16 RR1 F1,A,B,C,D,E,10 // i=28 : W0,W28,W24,W20,W16,W12,W8,W4 W_PRECALC_16_31_0 W20,W16,W12,W8,W4 RR0 F1,D,E,A,B,C,12 W_PRECALC_16_31_1 W20,W4 RR1 F1,D,E,A,B,C,12 W_PRECALC_16_31_2 W4 RR0 F1,B,C,D,E,A,14 W_PRECALC_16_31_3 W4,14,16 RR1 F1,B,C,D,E,A,14 // i=32 : W28,W24,W20,W16,W12,W8,W4,W0 W_PRECALC_32_79_0 W28,W8,W4,W0 RR0 F1,E,A,B,C,D,16 W_PRECALC_32_79_1 W16,W0 RR1 F1,E,A,B,C,D,16 W_PRECALC_32_79_2 W0 RR0 F1,C,D,E,A,B,18 W_PRECALC_32_79_3 W0,18,16 RR1 F1,C,D,E,A,B,18 // starting using F2 // i=36 : W24,W20,W16,W12,W8,W4,W0,W28 #if defined (__x86_64__) W_PRECALC_32_79_0 W24,W4,W0,W28 #else W_PRECALC_32_79_0_i386 W24,W4,W0,W28 #endif RR0 F2,A,B,C,D,E,20 W_PRECALC_32_79_1 W12,W28 RR1 F2,A,B,C,D,E,20 #if defined (__x86_64__) W_PRECALC_32_79_2 W28 #else W_PRECALC_32_79_2_i386 W28 #endif RR0 F2,D,E,A,B,C,22 W_PRECALC_32_79_3 W28,22,16 RR1 F2,D,E,A,B,C,22 // i=40 : W20,W16,W12,W8,W4,W0,W28,W24 #undef K_XMM #define K_XMM 32 #if defined (__x86_64__) W_PRECALC_32_79_0 W20,W0,W28,W24 #else W_PRECALC_32_79_0_i386 W20,W0,W28,W24 #endif RR0 F2,B,C,D,E,A,24 W_PRECALC_32_79_1 W8,W24 RR1 F2,B,C,D,E,A,24 #if defined (__x86_64__) W_PRECALC_32_79_2 W24 #else W_PRECALC_32_79_2_i386 W24 #endif RR0 F2,E,A,B,C,D,26 W_PRECALC_32_79_3 W24,26,K_XMM RR1 F2,E,A,B,C,D,26 // i=44 : W16,W12,W8,W4,W0,W28,W24,W20 W_PRECALC_32_79_0 W16,W28,W24,W20 RR0 F2,C,D,E,A,B,28 W_PRECALC_32_79_1 W4,W20 RR1 F2,C,D,E,A,B,28 W_PRECALC_32_79_2 W20 RR0 F2,A,B,C,D,E,30 W_PRECALC_32_79_3 W20,30,K_XMM RR1 F2,A,B,C,D,E,30 // i=48 : W12,W8,W4,W0,W28,W24,W20,W16 W_PRECALC_32_79_0 W12,W24,W20,W16 RR0 F2,D,E,A,B,C,32 W_PRECALC_32_79_1 W0,W16 RR1 F2,D,E,A,B,C,32 W_PRECALC_32_79_2 W16 RR0 F2,B,C,D,E,A,34 W_PRECALC_32_79_3 W16,34,K_XMM RR1 F2,B,C,D,E,A,34 // i=52 : W8,W4,W0,W28,W24,W20,W16,W12 W_PRECALC_32_79_0 W8,W20,W16,W12 RR0 F2,E,A,B,C,D,36 W_PRECALC_32_79_1 W28,W12 RR1 F2,E,A,B,C,D,36 W_PRECALC_32_79_2 W12 RR0 F2,C,D,E,A,B,38 W_PRECALC_32_79_3 W12,38,K_XMM RR1 F2,C,D,E,A,B,38 // starting using F3 // i=56 : W4,W0,W28,W24,W20,W16,W12,W8 W_PRECALC_32_79_0 W4,W16,W12,W8 RR0 F3,A,B,C,D,E,40 W_PRECALC_32_79_1 W24,W8 RR1 F3,A,B,C,D,E,40 W_PRECALC_32_79_2 W8 RR0 F3,D,E,A,B,C,42 W_PRECALC_32_79_3 W8,42,K_XMM RR1 F3,D,E,A,B,C,42 // i=60 : W0,W28,W24,W20,W16,W12,W8,W4 #undef K_XMM #define K_XMM 48 W_PRECALC_32_79_0 W0,W12,W8,W4 RR0 F3,B,C,D,E,A,44 W_PRECALC_32_79_1 W20,W4 RR1 F3,B,C,D,E,A,44 W_PRECALC_32_79_2 W4 RR0 F3,E,A,B,C,D,46 W_PRECALC_32_79_3 W4,46,K_XMM RR1 F3,E,A,B,C,D,46 // i=64 : W28,W24,W20,W16,W12,W8,W4,W0 W_PRECALC_32_79_0 W28,W8,W4,W0 RR0 F3,C,D,E,A,B,48 W_PRECALC_32_79_1 W16,W0 RR1 F3,C,D,E,A,B,48 W_PRECALC_32_79_2 W0 RR0 F3,A,B,C,D,E,50 W_PRECALC_32_79_3 W0,50,K_XMM RR1 F3,A,B,C,D,E,50 // i=68 : W24,W20,W16,W12,W8,W4,W0,W28 #if defined (__x86_64__) W_PRECALC_32_79_0 W24,W4,W0,W28 #else W_PRECALC_32_79_0_i386 W24,W4,W0,W28 #endif RR0 F3,D,E,A,B,C,52 W_PRECALC_32_79_1 W12,W28 RR1 F3,D,E,A,B,C,52 #if defined (__x86_64__) W_PRECALC_32_79_2 W28 #else W_PRECALC_32_79_2_i386 W28 #endif RR0 F3,B,C,D,E,A,54 W_PRECALC_32_79_3 W28,54,K_XMM RR1 F3,B,C,D,E,A,54 // i=72 : W20,W16,W12,W8,W4,W0,W28,W24 #if defined (__x86_64__) W_PRECALC_32_79_0 W20,W0,W28,W24 #else W_PRECALC_32_79_0_i386 W20,W0,W28,W24 #endif RR0 F3,E,A,B,C,D,56 W_PRECALC_32_79_1 W8,W24 RR1 F3,E,A,B,C,D,56 #if defined (__x86_64__) W_PRECALC_32_79_2 W24 #else W_PRECALC_32_79_2_i386 W24 #endif RR0 F3,C,D,E,A,B,58 W_PRECALC_32_79_3 W24,58,K_XMM RR1 F3,C,D,E,A,B,58 // starting using F4 // i=76 : W16,W12,W8,W4,W0,W28,W24,W20 W_PRECALC_32_79_0 W16,W28,W24,W20 RR0 F4,A,B,C,D,E,60 W_PRECALC_32_79_1 W4,W20 RR1 F4,A,B,C,D,E,60 W_PRECALC_32_79_2 W20 RR0 F4,D,E,A,B,C,62 W_PRECALC_32_79_3 W20,62,K_XMM RR1 F4,D,E,A,B,C,62 .endm .macro SOFTWARE_PIPELINING_ssse3 // i=0 : W28,W24,W20,W16,W12,W8,W4,W0 W_PRECALC_00_15_0 0 // W_TMP = (BUFFER_PTR) RR0 F4,B,C,D,E,A,64 W_PRECALC_00_15_1 W0 // convert W_TMP to big-endian, and save W0 = W_TMP RR1 F4,B,C,D,E,A,64 W_PRECALC_00_15_2 // W_TMP = W0 + K RR0 F4,E,A,B,C,D,66 W_PRECALC_00_15_3 3 // (sp) = W_TMP = W0 + K RR1 F4,E,A,B,C,D,66 // i=4 : W24,W20,W16,W12,W8,W4,W0,W28 W_PRECALC_00_15_0 4 // W_TMP = 16(BUFFER_PTR) RR0 F4,C,D,E,A,B,68 W_PRECALC_00_15_1 W28 // convert W_TMP to big-endian, and save W28 = W_TMP RR1 F4,C,D,E,A,B,68 W_PRECALC_00_15_2 // W_TMP = W28 + K RR0 F4,A,B,C,D,E,70 W_PRECALC_00_15_3 7 // 16(sp) = W_TMP = W28 + K[0] RR1 F4,A,B,C,D,E,70 // i=8 : W20,W16,W12,W8,W4,W0,W28,W24 W_PRECALC_00_15_0 8 // W_TMP = 32(BUFFER_PTR) RR0 F4,D,E,A,B,C,72 W_PRECALC_00_15_1 W24 // convert W_TMP to big-endian, and save W24 = W_TMP RR1 F4,D,E,A,B,C,72 W_PRECALC_00_15_2 // W_TMP = W24 + K RR0 F4,B,C,D,E,A,74 W_PRECALC_00_15_3 11 // 32(sp) = W_TMP = W24 + K RR1 F4,B,C,D,E,A,74 // i=12 : W16,W12,W8,W4,W0,W28,W24,W20 W_PRECALC_00_15_0 12 // W_TMP = 48(BUFFER_PTR) RR0 F4,E,A,B,C,D,76 W_PRECALC_00_15_1 W20 // convert W_TMP to big-endian, and save W20 = W_TMP RR1 F4,E,A,B,C,D,76 W_PRECALC_00_15_2 // W_TMP = W20 + K RR0 F4,C,D,E,A,B,78 W_PRECALC_00_15_3 15 // 48(sp) = W_TMP = W20 + K RR1 F4,C,D,E,A,B,78 .endm #undef W_PRECALC_00_15_0 #undef W_PRECALC_00_15_1 #undef W_PRECALC_16_31_0 #undef W_PRECALC_32_79_0 #undef W_PRECALC_32_79_0_i386 /* The following are 3 macro definitions that are no-ssse3 variants of the previous 3 macro definitions. INITIAL_W_PRECALC_nossse3 INTERNAL_nossse3 SOFTWARE_PIPELINING_nossse3 They will be used in a sha1 code main body definition that will be used for system without ssse3 support. */ #define W_PRECALC_00_15_0 W_PRECALC_00_15_0_nossse3 #define W_PRECALC_00_15_1 W_PRECALC_00_15_1_nossse3 #define W_PRECALC_16_31_0 W_PRECALC_16_31_0_nossse3 #define W_PRECALC_32_79_0 W_PRECALC_32_79_0_nossse3 #define W_PRECALC_32_79_0_i386 W_PRECALC_32_79_0_i386_nossse3 .macro INITIAL_W_PRECALC_nossse3 // i=0 : W28,W24,W20,W16,W12,W8,W4,W0 W_PRECALC_00_15_0 0 // W_TMP = (BUFFER_PTR) W_PRECALC_00_15_1 W0 // convert W_TMP to big-endian, and save W0 = W_TMP W_PRECALC_00_15_2 // W_TMP = W0 + K W_PRECALC_00_15_3 3 // (sp) = W_TMP = W0 + K // i=4 : W24,W20,W16,W12,W8,W4,W0,W28 W_PRECALC_00_15_0 4 // W_TMP = 16(BUFFER_PTR) W_PRECALC_00_15_1 W28 // convert W_TMP to big-endian, and save W28 = W_TMP W_PRECALC_00_15_2 // W_TMP = W28 + K W_PRECALC_00_15_3 7 // 16(sp) = W_TMP = W28 + K // i=8 : W20,W16,W12,W8,W4,W0,W28,W24 W_PRECALC_00_15_0 8 // W_TMP = 32(BUFFER_PTR) W_PRECALC_00_15_1 W24 // convert W_TMP to big-endian, and save W24 = W_TMP W_PRECALC_00_15_2 // W_TMP = W24 + K W_PRECALC_00_15_3 11 // 32(sp) = W_TMP = W24 + K // i=12 : W16,W12,W8,W4,W0,W28,W24,W20 W_PRECALC_00_15_0 12 // W_TMP = 48(BUFFER_PTR) W_PRECALC_00_15_1 W20 // convert W_TMP to big-endian, and save W20 = W_TMP W_PRECALC_00_15_2 // W_TMP = W20 + K W_PRECALC_00_15_3 15 // 48(sp) = W_TMP = W20 + K .endm .macro INTERNAL_nossse3 // i=16 // circular buffer : W12,W8,W4,W0,W28,W24,W20,W16 W_PRECALC_16_31_0 W0,W28,W24,W20,W16 RR0 F1,A,B,C,D,E,0 W_PRECALC_16_31_1 W0,W16 RR1 F1,A,B,C,D,E,0 W_PRECALC_16_31_2 W16 RR0 F1,D,E,A,B,C,2 W_PRECALC_16_31_3 W16, 2, 0 RR1 F1,D,E,A,B,C,2 // i=20, // W8,W4,W0,W28,W24,W20,W16,W12 W_PRECALC_16_31_0 W28,W24,W20,W16,W12 RR0 F1,B,C,D,E,A,4 W_PRECALC_16_31_1 W28,W12 RR1 F1,B,C,D,E,A,4 W_PRECALC_16_31_2 W12 RR0 F1,E,A,B,C,D,6 W_PRECALC_16_31_3 W12, 6, 16 RR1 F1,E,A,B,C,D,6 // i=24, // W4,W0,W28,W24,W20,W16,W12,W8 W_PRECALC_16_31_0 W24,W20,W16,W12,W8 RR0 F1,C,D,E,A,B,8 W_PRECALC_16_31_1 W24,W8 RR1 F1,C,D,E,A,B,8 W_PRECALC_16_31_2 W8 RR0 F1,A,B,C,D,E,10 W_PRECALC_16_31_3 W8,10,16 RR1 F1,A,B,C,D,E,10 // i=28 // W0,W28,W24,W20,W16,W12,W8,W4 W_PRECALC_16_31_0 W20,W16,W12,W8,W4 RR0 F1,D,E,A,B,C,12 W_PRECALC_16_31_1 W20,W4 RR1 F1,D,E,A,B,C,12 W_PRECALC_16_31_2 W4 RR0 F1,B,C,D,E,A,14 W_PRECALC_16_31_3 W4,14,16 RR1 F1,B,C,D,E,A,14 //i=32 // W28,W24,W20,W16,W12,W8,W4,W0 W_PRECALC_32_79_0 W28,W8,W4,W0 RR0 F1,E,A,B,C,D,16 W_PRECALC_32_79_1 W16,W0 RR1 F1,E,A,B,C,D,16 W_PRECALC_32_79_2 W0 RR0 F1,C,D,E,A,B,18 W_PRECALC_32_79_3 W0,18,16 RR1 F1,C,D,E,A,B,18 //i=36 // W24,W20,W16,W12,W8,W4,W0,W28 #if defined (__x86_64__) W_PRECALC_32_79_0 W24,W4,W0,W28 #else W_PRECALC_32_79_0_i386 W24,W4,W0,W28 #endif RR0 F2,A,B,C,D,E,20 W_PRECALC_32_79_1 W12,W28 RR1 F2,A,B,C,D,E,20 #if defined (__x86_64__) W_PRECALC_32_79_2 W28 #else W_PRECALC_32_79_2_i386 W28 #endif RR0 F2,D,E,A,B,C,22 W_PRECALC_32_79_3 W28,22,16 RR1 F2,D,E,A,B,C,22 //i=40 #undef K_XMM #define K_XMM 32 // W20,W16,W12,W8,W4,W0,W28,W24 #if defined (__x86_64__) W_PRECALC_32_79_0 W20,W0,W28,W24 #else W_PRECALC_32_79_0_i386 W20,W0,W28,W24 #endif RR0 F2,B,C,D,E,A,24 W_PRECALC_32_79_1 W8,W24 RR1 F2,B,C,D,E,A,24 #if defined (__x86_64__) W_PRECALC_32_79_2 W24 #else W_PRECALC_32_79_2_i386 W24 #endif RR0 F2,E,A,B,C,D,26 W_PRECALC_32_79_3 W24,26,K_XMM RR1 F2,E,A,B,C,D,26 //i=44 // W16,W12,W8,W4,W0,W28,W24,W20 W_PRECALC_32_79_0 W16,W28,W24,W20 RR0 F2,C,D,E,A,B,28 W_PRECALC_32_79_1 W4,W20 RR1 F2,C,D,E,A,B,28 W_PRECALC_32_79_2 W20 RR0 F2,A,B,C,D,E,30 W_PRECALC_32_79_3 W20,30,K_XMM RR1 F2,A,B,C,D,E,30 //i=48 // W12,W8,W4,W0,W28,W24,W20,W16 W_PRECALC_32_79_0 W12,W24,W20,W16 RR0 F2,D,E,A,B,C,32 W_PRECALC_32_79_1 W0,W16 RR1 F2,D,E,A,B,C,32 W_PRECALC_32_79_2 W16 RR0 F2,B,C,D,E,A,34 W_PRECALC_32_79_3 W16,34,K_XMM RR1 F2,B,C,D,E,A,34 //i=52 // W8,W4,W0,W28,W24,W20,W16,W12 W_PRECALC_32_79_0 W8,W20,W16,W12 RR0 F2,E,A,B,C,D,36 W_PRECALC_32_79_1 W28,W12 RR1 F2,E,A,B,C,D,36 W_PRECALC_32_79_2 W12 RR0 F2,C,D,E,A,B,38 W_PRECALC_32_79_3 W12,38,K_XMM RR1 F2,C,D,E,A,B,38 //i=56 // W4,W0,W28,W24,W20,W16,W12,W8 W_PRECALC_32_79_0 W4,W16,W12,W8 RR0 F3,A,B,C,D,E,40 W_PRECALC_32_79_1 W24,W8 RR1 F3,A,B,C,D,E,40 W_PRECALC_32_79_2 W8 RR0 F3,D,E,A,B,C,42 W_PRECALC_32_79_3 W8,42,K_XMM RR1 F3,D,E,A,B,C,42 //i=60 #undef K_XMM #define K_XMM 48 // W0,W28,W24,W20,W16,W12,W8,W4 W_PRECALC_32_79_0 W0,W12,W8,W4 RR0 F3,B,C,D,E,A,44 W_PRECALC_32_79_1 W20,W4 RR1 F3,B,C,D,E,A,44 W_PRECALC_32_79_2 W4 RR0 F3,E,A,B,C,D,46 W_PRECALC_32_79_3 W4,46,K_XMM RR1 F3,E,A,B,C,D,46 //i=64 // W28,W24,W20,W16,W12,W8,W4,W0 W_PRECALC_32_79_0 W28,W8,W4,W0 RR0 F3,C,D,E,A,B,48 W_PRECALC_32_79_1 W16,W0 RR1 F3,C,D,E,A,B,48 W_PRECALC_32_79_2 W0 RR0 F3,A,B,C,D,E,50 W_PRECALC_32_79_3 W0,50,K_XMM RR1 F3,A,B,C,D,E,50 //i=68 // W24,W20,W16,W12,W8,W4,W0,W28 #if defined (__x86_64__) W_PRECALC_32_79_0 W24,W4,W0,W28 #else W_PRECALC_32_79_0_i386 W24,W4,W0,W28 #endif RR0 F3,D,E,A,B,C,52 W_PRECALC_32_79_1 W12,W28 RR1 F3,D,E,A,B,C,52 #if defined (__x86_64__) W_PRECALC_32_79_2 W28 #else W_PRECALC_32_79_2_i386 W28 #endif RR0 F3,B,C,D,E,A,54 W_PRECALC_32_79_3 W28,54,K_XMM RR1 F3,B,C,D,E,A,54 //i=72 // W20,W16,W12,W8,W4,W0,W28,W24 #if defined (__x86_64__) W_PRECALC_32_79_0 W20,W0,W28,W24 #else W_PRECALC_32_79_0_i386 W20,W0,W28,W24 #endif RR0 F3,E,A,B,C,D,56 W_PRECALC_32_79_1 W8,W24 RR1 F3,E,A,B,C,D,56 #if defined (__x86_64__) W_PRECALC_32_79_2 W24 #else W_PRECALC_32_79_2_i386 W24 #endif RR0 F3,C,D,E,A,B,58 W_PRECALC_32_79_3 W24,58,K_XMM RR1 F3,C,D,E,A,B,58 // starting using F4 //i=76 // W16,W12,W8,W4,W0,W28,W24,W20 W_PRECALC_32_79_0 W16,W28,W24,W20 RR0 F4,A,B,C,D,E,60 W_PRECALC_32_79_1 W4,W20 RR1 F4,A,B,C,D,E,60 W_PRECALC_32_79_2 W20 RR0 F4,D,E,A,B,C,62 W_PRECALC_32_79_3 W20,62,K_XMM RR1 F4,D,E,A,B,C,62 .endm .macro SOFTWARE_PIPELINING_nossse3 // i=0 : W28,W24,W20,W16,W12,W8,W4,W0 W_PRECALC_00_15_0 0 // W_TMP = (BUFFER_PTR) RR0 F4,B,C,D,E,A,64 W_PRECALC_00_15_1 W0 // convert W_TMP to big-endian, and save W0 = W_TMP RR1 F4,B,C,D,E,A,64 W_PRECALC_00_15_2 // W_TMP = W0 + K RR0 F4,E,A,B,C,D,66 W_PRECALC_00_15_3 3 // (sp) = W_TMP = W0 + K RR1 F4,E,A,B,C,D,66 // i=4 : W24,W20,W16,W12,W8,W4,W0,W28 W_PRECALC_00_15_0 4 // W_TMP = 16(BUFFER_PTR) RR0 F4,C,D,E,A,B,68 W_PRECALC_00_15_1 W28 // convert W_TMP to big-endian, and save W28 = W_TMP RR1 F4,C,D,E,A,B,68 W_PRECALC_00_15_2 // W_TMP = W28 + K RR0 F4,A,B,C,D,E,70 W_PRECALC_00_15_3 7 // 16(sp) = W_TMP = W28 + K[0] RR1 F4,A,B,C,D,E,70 // i=8 : W20,W16,W12,W8,W4,W0,W28,W24 W_PRECALC_00_15_0 8 // W_TMP = 32(BUFFER_PTR) RR0 F4,D,E,A,B,C,72 W_PRECALC_00_15_1 W24 // convert W_TMP to big-endian, and save W24 = W_TMP RR1 F4,D,E,A,B,C,72 W_PRECALC_00_15_2 // W_TMP = W24 + K RR0 F4,B,C,D,E,A,74 W_PRECALC_00_15_3 11 // 32(sp) = W_TMP = W24 + K RR1 F4,B,C,D,E,A,74 // i=12 : W16,W12,W8,W4,W0,W28,W24,W20 W_PRECALC_00_15_0 12 // W_TMP = 48(BUFFER_PTR) RR0 F4,E,A,B,C,D,76 W_PRECALC_00_15_1 W20 // convert W_TMP to big-endian, and save W20 = W_TMP RR1 F4,E,A,B,C,D,76 W_PRECALC_00_15_2 // W_TMP = W20 + K RR0 F4,C,D,E,A,B,78 W_PRECALC_00_15_3 15 // 48(sp) = W_TMP = W20 + K RR1 F4,C,D,E,A,B,78 .endm .macro ENDING // finish up updating hash digests (i=64:79) //i=80 RR0 F4,B,C,D,E,A,64 RR1 F4,B,C,D,E,A,64 RR0 F4,E,A,B,C,D,66 RR1 F4,E,A,B,C,D,66 //i=84 RR0 F4,C,D,E,A,B,68 RR1 F4,C,D,E,A,B,68 RR0 F4,A,B,C,D,E,70 RR1 F4,A,B,C,D,E,70 //i=88 RR0 F4,D,E,A,B,C,72 RR1 F4,D,E,A,B,C,72 RR0 F4,B,C,D,E,A,74 RR1 F4,B,C,D,E,A,74 //i=92 RR0 F4,E,A,B,C,D,76 RR1 F4,E,A,B,C,D,76 RR0 F4,C,D,E,A,B,78 RR1 F4,C,D,E,A,B,78 .endm // load hash digests A,B,C,D,E from memory into registers .macro LOAD_HASH #if defined (__x86_64__) mov (HASH_PTR), A mov 4(HASH_PTR), B mov 8(HASH_PTR), C mov 12(HASH_PTR), D mov 16(HASH_PTR), E #else mov HASH_PTR, T1 mov (T1), A mov 4(T1), B mov 8(T1), C mov 12(T1), D mov 16(T1), E #endif .endm .macro UPDATE_HASH add $0, $1 mov $1, $0 .endm .macro UPDATE_ALL_HASH #if defined (__x86_64__) UPDATE_HASH (HASH_PTR), A UPDATE_HASH 4(HASH_PTR), B UPDATE_HASH 8(HASH_PTR), C UPDATE_HASH 12(HASH_PTR), D UPDATE_HASH 16(HASH_PTR), E #else mov HASH_PTR, T1 UPDATE_HASH (T1), A UPDATE_HASH 4(T1), B UPDATE_HASH 8(T1), C UPDATE_HASH 12(T1), D UPDATE_HASH 16(T1), E #endif .endm /* main sha1 code for system without ssse3 support */ .macro SHA1_PIPELINED_MAIN_BODY_nossse3 LOAD_HASH // load initial hashes into A,B,C,D,E (registers) INITIAL_W_PRECALC_nossse3 // big_endian_load(W) and W+K (i=0:15) .align 4,0x90 0: INTERNAL_nossse3 // update W (i=16:79) and update ABCDE (i=0:63) #if Multiple_Blocks #if defined(__x86_64__) add $$64, BUFFER_PTR // BUFFER_PTR+=64; sub $$1, cnt // pre-decrement cnt by 1 #else addl $$64, BUFFER_PTR // BUFFER_PTR+=64; subl $$1, cnt // pre-decrement cnt by 1 #endif jbe 1f // if cnt <= 0, branch to finish off SOFTWARE_PIPELINING_nossse3 // update ABCDE (i=64:79) || big_endian_load(W) and W+K (i=0:15) UPDATE_ALL_HASH // update output hashes jmp 0b // repeat for next block .align 4,0x90 1: #endif ENDING // update ABCDE (i=64:79) UPDATE_ALL_HASH // update output hashes .endm /* main sha1 code for system with ssse3 support */ .macro SHA1_PIPELINED_MAIN_BODY_ssse3 LOAD_HASH // load initial hashes into A,B,C,D,E INITIAL_W_PRECALC_ssse3 // big_endian_load(W) and W+K (i=0:15) .align 4,0x90 0: INTERNAL_ssse3 // update W (i=16:79) and update ABCDE (i=0:63) #if Multiple_Blocks #if defined(__x86_64__) add $$64, BUFFER_PTR // BUFFER_PTR+=64; sub $$1, cnt // pre-decrement cnt by 1 #else addl $$64, BUFFER_PTR // BUFFER_PTR+=64; subl $$1, cnt // pre-decrement cnt by 1 #endif jbe 1f // if cnt <= 0, branch to finish off SOFTWARE_PIPELINING_ssse3 // update ABCDE (i=64:79) || big_endian_load(W) and W+K (i=0:15) UPDATE_ALL_HASH // update output hashes jmp 0b // repeat for next block .align 4,0x90 1: #endif ENDING // update ABCDE (i=64:79) UPDATE_ALL_HASH // update output hashes .endm #ifdef KERNEL #include <i386/cpu_capabilities.h> #else #include <System/i386/cpu_capabilities.h> #endif .text .globl _SHA1Transform //.private_extern _SHA1Transform _SHA1Transform: // detect SSSE3 and dispatch appropriate code branch #if defined __x86_64__ movq __cpu_capabilities@GOTPCREL(%rip), %rax // %rax -> __cpu_capabilities mov (%rax), %eax // %eax = __cpu_capabilities #else // i386 #if defined KERNEL leal __cpu_capabilities, %eax // %eax -> __cpu_capabilities mov (%eax), %eax // %eax = __cpu_capabilities #else mov _COMM_PAGE_CPU_CAPABILITIES, %eax #endif #endif test $(kHasSupplementalSSE3), %eax je _SHA1Transform_nossse3 // branch to no-ssse3 code // start the sha1 code with ssse3 support // save callee-save registers #if defined (__x86_64__) push %rbx push %rbp #else push %ebx push %ebp push %esi push %edi #endif sub $stack_size, sp // allocate stack memory for use // save used xmm register if this is for kernel #if KERNEL xmov %xmm0, 4*16(sp) xmov %xmm1, 5*16(sp) xmov %xmm2, 6*16(sp) xmov %xmm3, 7*16(sp) xmov %xmm4, 8*16(sp) xmov %xmm5, 9*16(sp) xmov %xmm6, 10*16(sp) xmov %xmm7, 11*16(sp) #if defined (__x86_64__) xmov %xmm8, 12*16(sp) xmov %xmm9, 13*16(sp) xmov %xmm10, 14*16(sp) #endif #endif #if defined (__x86_64__) // set up registers to free %edx/%edi/%esi for other use (ABCDE) mov ctx, HASH_PTR mov buf, BUFFER_PTR #if Multiple_Blocks mov %rdx, cnt #endif lea K_XMM_AR(%rip), K_BASE xmov 0x40(K_BASE), XMM_SHUFB_BSWAP #else // __i386__ #if KERNEL lea K_XMM_AR, %eax #else // Get address of 0 in R. call 0f // Push program counter onto stack. 0: pop %eax // Get program counter. lea K_XMM_AR-0b(%eax), %eax #endif mov %eax, K_BASE xmov 0x40(%eax), %xmm0 xmov %xmm0, XMM_SHUFB_BSWAP #endif SHA1_PIPELINED_MAIN_BODY_ssse3 // restore used xmm registers if this is for kernel #if KERNEL xmov 4*16(sp), %xmm0 xmov 5*16(sp), %xmm1 xmov 6*16(sp), %xmm2 xmov 7*16(sp), %xmm3 xmov 8*16(sp), %xmm4 xmov 9*16(sp), %xmm5 xmov 10*16(sp), %xmm6 xmov 11*16(sp), %xmm7 #if defined (__x86_64__) xmov 12*16(sp), %xmm8 xmov 13*16(sp), %xmm9 xmov 14*16(sp), %xmm10 #endif #endif add $stack_size, sp // deallocate stack memory // restore callee-save registers #if defined (__x86_64__) pop %rbp pop %rbx #else pop %edi pop %esi pop %ebp pop %ebx #endif ret // return // this is equivalent to the above function _SHA1Transform, but it does not use ssse3 instructions .globl _SHA1Transform_nossse3 .private_extern _SHA1Transform_nossse3 _SHA1Transform_nossse3: // push callee-save registers #if defined (__x86_64__) push %rbx push %rbp #else push %ebx push %ebp push %esi push %edi #endif sub $stack_size, sp // allocate stack memory for local use // save used xmm registers if this is for kernel #if KERNEL xmov %xmm0, 4*16(sp) xmov %xmm1, 5*16(sp) xmov %xmm2, 6*16(sp) xmov %xmm3, 7*16(sp) xmov %xmm4, 8*16(sp) xmov %xmm5, 9*16(sp) xmov %xmm6, 10*16(sp) xmov %xmm7, 11*16(sp) #if defined (__x86_64__) xmov %xmm8, 12*16(sp) xmov %xmm9, 13*16(sp) #endif #endif #if defined (__x86_64__) // set up registers to free %edx/%edi/%esi for other use (ABCDE) mov ctx, HASH_PTR mov buf, BUFFER_PTR #if Multiple_Blocks mov %rdx, cnt #endif lea K_XMM_AR(%rip), K_BASE #else // __i386__ #if KERNEL lea K_XMM_AR, %eax #else // Get address of 0 in R. call 0f // Push program counter onto stack. 0: pop %eax // Get program counter. lea K_XMM_AR-0b(%eax), %eax #endif mov %eax, K_BASE #endif SHA1_PIPELINED_MAIN_BODY_nossse3 // restore used xmm registers if this is for kernel #if KERNEL xmov 4*16(sp), %xmm0 xmov 5*16(sp), %xmm1 xmov 6*16(sp), %xmm2 xmov 7*16(sp), %xmm3 xmov 8*16(sp), %xmm4 xmov 9*16(sp), %xmm5 xmov 10*16(sp), %xmm6 xmov 11*16(sp), %xmm7 #if defined (__x86_64__) xmov 12*16(sp), %xmm8 xmov 13*16(sp), %xmm9 #endif #endif add $stack_size, sp // deallocate stack memory // restore callee-save registers #if defined (__x86_64__) pop %rbp pop %rbx #else pop %edi pop %esi pop %ebp pop %ebx #endif ret // return .const .align 4, 0x90 #define K1 0x5a827999 #define K2 0x6ed9eba1 #define K3 0x8f1bbcdc #define K4 0xca62c1d6 K_XMM_AR: .long K1 .long K1 .long K1 .long K1 .long K2 .long K2 .long K2 .long K2 .long K3 .long K3 .long K3 .long K3 .long K4 .long K4 .long K4 .long K4 // bswap_shufb_ctl: invoked thru 0x40(K_XMM_AR) .long 0x00010203 .long 0x04050607 .long 0x08090a0b .long 0x0c0d0e0f #endif // architecture x86_64 or i386