| ; This file is generated from a similarly-named Perl script in the BoringSSL |
| ; source tree. Do not edit by hand. |
| |
| %ifidn __OUTPUT_FORMAT__, win64 |
| default rel |
| %define XMMWORD |
| %define YMMWORD |
| %define ZMMWORD |
| %define _CET_ENDBR |
| |
| %ifdef BORINGSSL_PREFIX |
| %include "boringssl_prefix_symbols_internal_x86_64_win_asm.inc" |
| %endif |
| section .text code align=64 |
| |
| |
| ; gcm_gmult_ssse3 multiplies |Xi| by |Htable| and writes the result to |Xi|. |
| ; |Xi| is represented in GHASH's serialized byte representation. |Htable| is |
| ; formatted as described above. |
| ; void gcm_gmult_ssse3(uint64_t Xi[2], const u128 Htable[16]); |
| |
| global gcm_gmult_ssse3 |
| ALIGN 16 |
| gcm_gmult_ssse3: |
| |
| $L$SEH_begin_gcm_gmult_ssse3_1: |
| _CET_ENDBR |
| sub rsp,40 |
| $L$SEH_prologue_gcm_gmult_ssse3_2: |
| movdqa XMMWORD[rsp],xmm6 |
| $L$SEH_prologue_gcm_gmult_ssse3_3: |
| movdqa XMMWORD[16+rsp],xmm10 |
| $L$SEH_prologue_gcm_gmult_ssse3_4: |
| $L$SEH_endprologue_gcm_gmult_ssse3_5: |
| movdqu xmm0,XMMWORD[rcx] |
| movdqa xmm10,XMMWORD[$L$reverse_bytes] |
| movdqa xmm2,XMMWORD[$L$low4_mask] |
| |
| ; Reverse input bytes to deserialize. |
| pshufb xmm0,xmm10 |
| |
| ; Split each byte into low (%xmm0) and high (%xmm1) halves. |
| movdqa xmm1,xmm2 |
| pandn xmm1,xmm0 |
| psrld xmm1,4 |
| pand xmm0,xmm2 |
| |
| ; Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note |
| ; that, due to bit reversal, %xmm3 contains bits that fall off when |
| ; right-shifting, not left-shifting. |
| pxor xmm2,xmm2 |
| pxor xmm3,xmm3 |
| mov rax,5 |
| $L$oop_row_1: |
| movdqu xmm4,XMMWORD[rdx] |
| lea rdx,[16+rdx] |
| |
| ; Right-shift %xmm2 and %xmm3 by 8 bytes. |
| movdqa xmm6,xmm2 |
| palignr xmm6,xmm3,1 |
| movdqa xmm3,xmm6 |
| psrldq xmm2,1 |
| |
| ; Load the next table row and index the low and high bits of the input. |
| ; Note the low (respectively, high) half corresponds to more |
| ; (respectively, less) significant coefficients. |
| movdqa xmm5,xmm4 |
| pshufb xmm4,xmm0 |
| pshufb xmm5,xmm1 |
| |
| ; Add the high half (%xmm5) without shifting. |
| pxor xmm2,xmm5 |
| |
| ; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, |
| ; add into the carry register (%xmm3). |
| movdqa xmm5,xmm4 |
| psllq xmm5,60 |
| movdqa xmm6,xmm5 |
| pslldq xmm6,8 |
| pxor xmm3,xmm6 |
| |
| ; Next, add into %xmm2. |
| psrldq xmm5,8 |
| pxor xmm2,xmm5 |
| psrlq xmm4,4 |
| pxor xmm2,xmm4 |
| |
| sub rax,1 |
| jnz NEAR $L$oop_row_1 |
| |
| ; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + |
| ; x^7, so we shift and XOR four times. |
| pxor xmm2,xmm3 ; x^0 = 0 |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^1 = x |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^(1+1) = x^2 |
| psrlq xmm3,5 |
| pxor xmm2,xmm3 ; x^(1+1+5) = x^7 |
| pxor xmm3,xmm3 |
| mov rax,5 |
| $L$oop_row_2: |
| movdqu xmm4,XMMWORD[rdx] |
| lea rdx,[16+rdx] |
| |
| ; Right-shift %xmm2 and %xmm3 by 8 bytes. |
| movdqa xmm6,xmm2 |
| palignr xmm6,xmm3,1 |
| movdqa xmm3,xmm6 |
| psrldq xmm2,1 |
| |
| ; Load the next table row and index the low and high bits of the input. |
| ; Note the low (respectively, high) half corresponds to more |
| ; (respectively, less) significant coefficients. |
| movdqa xmm5,xmm4 |
| pshufb xmm4,xmm0 |
| pshufb xmm5,xmm1 |
| |
| ; Add the high half (%xmm5) without shifting. |
| pxor xmm2,xmm5 |
| |
| ; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, |
| ; add into the carry register (%xmm3). |
| movdqa xmm5,xmm4 |
| psllq xmm5,60 |
| movdqa xmm6,xmm5 |
| pslldq xmm6,8 |
| pxor xmm3,xmm6 |
| |
| ; Next, add into %xmm2. |
| psrldq xmm5,8 |
| pxor xmm2,xmm5 |
| psrlq xmm4,4 |
| pxor xmm2,xmm4 |
| |
| sub rax,1 |
| jnz NEAR $L$oop_row_2 |
| |
| ; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + |
| ; x^7, so we shift and XOR four times. |
| pxor xmm2,xmm3 ; x^0 = 0 |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^1 = x |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^(1+1) = x^2 |
| psrlq xmm3,5 |
| pxor xmm2,xmm3 ; x^(1+1+5) = x^7 |
| pxor xmm3,xmm3 |
| mov rax,6 |
| $L$oop_row_3: |
| movdqu xmm4,XMMWORD[rdx] |
| lea rdx,[16+rdx] |
| |
| ; Right-shift %xmm2 and %xmm3 by 8 bytes. |
| movdqa xmm6,xmm2 |
| palignr xmm6,xmm3,1 |
| movdqa xmm3,xmm6 |
| psrldq xmm2,1 |
| |
| ; Load the next table row and index the low and high bits of the input. |
| ; Note the low (respectively, high) half corresponds to more |
| ; (respectively, less) significant coefficients. |
| movdqa xmm5,xmm4 |
| pshufb xmm4,xmm0 |
| pshufb xmm5,xmm1 |
| |
| ; Add the high half (%xmm5) without shifting. |
| pxor xmm2,xmm5 |
| |
| ; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, |
| ; add into the carry register (%xmm3). |
| movdqa xmm5,xmm4 |
| psllq xmm5,60 |
| movdqa xmm6,xmm5 |
| pslldq xmm6,8 |
| pxor xmm3,xmm6 |
| |
| ; Next, add into %xmm2. |
| psrldq xmm5,8 |
| pxor xmm2,xmm5 |
| psrlq xmm4,4 |
| pxor xmm2,xmm4 |
| |
| sub rax,1 |
| jnz NEAR $L$oop_row_3 |
| |
| ; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + |
| ; x^7, so we shift and XOR four times. |
| pxor xmm2,xmm3 ; x^0 = 0 |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^1 = x |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^(1+1) = x^2 |
| psrlq xmm3,5 |
| pxor xmm2,xmm3 ; x^(1+1+5) = x^7 |
| pxor xmm3,xmm3 |
| ; Store the result. Reverse bytes to serialize. |
| pshufb xmm2,xmm10 |
| movdqu XMMWORD[rcx],xmm2 |
| |
| ; Zero any registers which contain secrets. |
| pxor xmm0,xmm0 |
| pxor xmm1,xmm1 |
| pxor xmm2,xmm2 |
| pxor xmm3,xmm3 |
| pxor xmm4,xmm4 |
| pxor xmm5,xmm5 |
| pxor xmm6,xmm6 |
| movdqa xmm6,XMMWORD[rsp] |
| movdqa xmm10,XMMWORD[16+rsp] |
| add rsp,40 |
| ret |
| |
| $L$SEH_end_gcm_gmult_ssse3_6: |
| |
| ; gcm_ghash_ssse3 incorporates |len| bytes from |in| to |Xi|, using |Htable| as |
| ; the key. It writes the result back to |Xi|. |Xi| is represented in GHASH's |
| ; serialized byte representation. |Htable| is formatted as described above. |
| ; void gcm_ghash_ssse3(uint64_t Xi[2], const u128 Htable[16], const uint8_t *in, |
| ; size_t len); |
| |
| global gcm_ghash_ssse3 |
| ALIGN 16 |
| gcm_ghash_ssse3: |
| |
| $L$SEH_begin_gcm_ghash_ssse3_1: |
| _CET_ENDBR |
| sub rsp,56 |
| $L$SEH_prologue_gcm_ghash_ssse3_2: |
| movdqa XMMWORD[rsp],xmm6 |
| $L$SEH_prologue_gcm_ghash_ssse3_3: |
| movdqa XMMWORD[16+rsp],xmm10 |
| $L$SEH_prologue_gcm_ghash_ssse3_4: |
| movdqa XMMWORD[32+rsp],xmm11 |
| $L$SEH_prologue_gcm_ghash_ssse3_5: |
| $L$SEH_endprologue_gcm_ghash_ssse3_6: |
| movdqu xmm0,XMMWORD[rcx] |
| movdqa xmm10,XMMWORD[$L$reverse_bytes] |
| movdqa xmm11,XMMWORD[$L$low4_mask] |
| |
| ; This function only processes whole blocks. |
| and r9,-16 |
| |
| ; Reverse input bytes to deserialize. We maintain the running |
| ; total in %xmm0. |
| pshufb xmm0,xmm10 |
| |
| ; Iterate over each block. On entry to each iteration, %xmm3 is zero. |
| pxor xmm3,xmm3 |
| $L$oop_ghash: |
| ; Incorporate the next block of input. |
| movdqu xmm1,XMMWORD[r8] |
| pshufb xmm1,xmm10 ; Reverse bytes. |
| pxor xmm0,xmm1 |
| |
| ; Split each byte into low (%xmm0) and high (%xmm1) halves. |
| movdqa xmm1,xmm11 |
| pandn xmm1,xmm0 |
| psrld xmm1,4 |
| pand xmm0,xmm11 |
| |
| ; Maintain the result in %xmm2 (the value) and %xmm3 (carry bits). Note |
| ; that, due to bit reversal, %xmm3 contains bits that fall off when |
| ; right-shifting, not left-shifting. |
| pxor xmm2,xmm2 |
| ; %xmm3 is already zero at this point. |
| mov rax,5 |
| $L$oop_row_4: |
| movdqu xmm4,XMMWORD[rdx] |
| lea rdx,[16+rdx] |
| |
| ; Right-shift %xmm2 and %xmm3 by 8 bytes. |
| movdqa xmm6,xmm2 |
| palignr xmm6,xmm3,1 |
| movdqa xmm3,xmm6 |
| psrldq xmm2,1 |
| |
| ; Load the next table row and index the low and high bits of the input. |
| ; Note the low (respectively, high) half corresponds to more |
| ; (respectively, less) significant coefficients. |
| movdqa xmm5,xmm4 |
| pshufb xmm4,xmm0 |
| pshufb xmm5,xmm1 |
| |
| ; Add the high half (%xmm5) without shifting. |
| pxor xmm2,xmm5 |
| |
| ; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, |
| ; add into the carry register (%xmm3). |
| movdqa xmm5,xmm4 |
| psllq xmm5,60 |
| movdqa xmm6,xmm5 |
| pslldq xmm6,8 |
| pxor xmm3,xmm6 |
| |
| ; Next, add into %xmm2. |
| psrldq xmm5,8 |
| pxor xmm2,xmm5 |
| psrlq xmm4,4 |
| pxor xmm2,xmm4 |
| |
| sub rax,1 |
| jnz NEAR $L$oop_row_4 |
| |
| ; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + |
| ; x^7, so we shift and XOR four times. |
| pxor xmm2,xmm3 ; x^0 = 0 |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^1 = x |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^(1+1) = x^2 |
| psrlq xmm3,5 |
| pxor xmm2,xmm3 ; x^(1+1+5) = x^7 |
| pxor xmm3,xmm3 |
| mov rax,5 |
| $L$oop_row_5: |
| movdqu xmm4,XMMWORD[rdx] |
| lea rdx,[16+rdx] |
| |
| ; Right-shift %xmm2 and %xmm3 by 8 bytes. |
| movdqa xmm6,xmm2 |
| palignr xmm6,xmm3,1 |
| movdqa xmm3,xmm6 |
| psrldq xmm2,1 |
| |
| ; Load the next table row and index the low and high bits of the input. |
| ; Note the low (respectively, high) half corresponds to more |
| ; (respectively, less) significant coefficients. |
| movdqa xmm5,xmm4 |
| pshufb xmm4,xmm0 |
| pshufb xmm5,xmm1 |
| |
| ; Add the high half (%xmm5) without shifting. |
| pxor xmm2,xmm5 |
| |
| ; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, |
| ; add into the carry register (%xmm3). |
| movdqa xmm5,xmm4 |
| psllq xmm5,60 |
| movdqa xmm6,xmm5 |
| pslldq xmm6,8 |
| pxor xmm3,xmm6 |
| |
| ; Next, add into %xmm2. |
| psrldq xmm5,8 |
| pxor xmm2,xmm5 |
| psrlq xmm4,4 |
| pxor xmm2,xmm4 |
| |
| sub rax,1 |
| jnz NEAR $L$oop_row_5 |
| |
| ; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + |
| ; x^7, so we shift and XOR four times. |
| pxor xmm2,xmm3 ; x^0 = 0 |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^1 = x |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^(1+1) = x^2 |
| psrlq xmm3,5 |
| pxor xmm2,xmm3 ; x^(1+1+5) = x^7 |
| pxor xmm3,xmm3 |
| mov rax,6 |
| $L$oop_row_6: |
| movdqu xmm4,XMMWORD[rdx] |
| lea rdx,[16+rdx] |
| |
| ; Right-shift %xmm2 and %xmm3 by 8 bytes. |
| movdqa xmm6,xmm2 |
| palignr xmm6,xmm3,1 |
| movdqa xmm3,xmm6 |
| psrldq xmm2,1 |
| |
| ; Load the next table row and index the low and high bits of the input. |
| ; Note the low (respectively, high) half corresponds to more |
| ; (respectively, less) significant coefficients. |
| movdqa xmm5,xmm4 |
| pshufb xmm4,xmm0 |
| pshufb xmm5,xmm1 |
| |
| ; Add the high half (%xmm5) without shifting. |
| pxor xmm2,xmm5 |
| |
| ; Add the low half (%xmm4). This must be right-shifted by 4 bits. First, |
| ; add into the carry register (%xmm3). |
| movdqa xmm5,xmm4 |
| psllq xmm5,60 |
| movdqa xmm6,xmm5 |
| pslldq xmm6,8 |
| pxor xmm3,xmm6 |
| |
| ; Next, add into %xmm2. |
| psrldq xmm5,8 |
| pxor xmm2,xmm5 |
| psrlq xmm4,4 |
| pxor xmm2,xmm4 |
| |
| sub rax,1 |
| jnz NEAR $L$oop_row_6 |
| |
| ; Reduce the carry register. The reduction polynomial is 1 + x + x^2 + |
| ; x^7, so we shift and XOR four times. |
| pxor xmm2,xmm3 ; x^0 = 0 |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^1 = x |
| psrlq xmm3,1 |
| pxor xmm2,xmm3 ; x^(1+1) = x^2 |
| psrlq xmm3,5 |
| pxor xmm2,xmm3 ; x^(1+1+5) = x^7 |
| pxor xmm3,xmm3 |
| movdqa xmm0,xmm2 |
| |
| ; Rewind %rdx for the next iteration. |
| lea rdx,[((-256))+rdx] |
| |
| ; Advance input and continue. |
| lea r8,[16+r8] |
| sub r9,16 |
| jnz NEAR $L$oop_ghash |
| |
| ; Reverse bytes and store the result. |
| pshufb xmm0,xmm10 |
| movdqu XMMWORD[rcx],xmm0 |
| |
| ; Zero any registers which contain secrets. |
| pxor xmm0,xmm0 |
| pxor xmm1,xmm1 |
| pxor xmm2,xmm2 |
| pxor xmm3,xmm3 |
| pxor xmm4,xmm4 |
| pxor xmm5,xmm5 |
| pxor xmm6,xmm6 |
| movdqa xmm6,XMMWORD[rsp] |
| movdqa xmm10,XMMWORD[16+rsp] |
| movdqa xmm11,XMMWORD[32+rsp] |
| add rsp,56 |
| ret |
| |
| $L$SEH_end_gcm_ghash_ssse3_7: |
| |
| |
| section .rdata rdata align=8 |
| |
| ALIGN 16 |
| ghash_ssse3_constants: |
| ; .Lreverse_bytes is a permutation which, if applied with pshufb, reverses the |
| ; bytes in an XMM register. |
| $L$reverse_bytes: |
| DB 15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0 |
| ; .Llow4_mask is an XMM mask which selects the low four bits of each byte. |
| $L$low4_mask: |
| DQ 0x0f0f0f0f0f0f0f0f,0x0f0f0f0f0f0f0f0f |
| section .text code align=64 |
| |
| section .pdata rdata align=4 |
| ALIGN 4 |
| DD $L$SEH_begin_gcm_gmult_ssse3_1 wrt ..imagebase |
| DD $L$SEH_end_gcm_gmult_ssse3_6 wrt ..imagebase |
| DD $L$SEH_info_gcm_gmult_ssse3_0 wrt ..imagebase |
| |
| DD $L$SEH_begin_gcm_ghash_ssse3_1 wrt ..imagebase |
| DD $L$SEH_end_gcm_ghash_ssse3_7 wrt ..imagebase |
| DD $L$SEH_info_gcm_ghash_ssse3_0 wrt ..imagebase |
| |
| |
| section .xdata rdata align=8 |
| ALIGN 4 |
| $L$SEH_info_gcm_gmult_ssse3_0: |
| DB 1 ; version 1, no flags |
| DB $L$SEH_endprologue_gcm_gmult_ssse3_5-$L$SEH_begin_gcm_gmult_ssse3_1 |
| DB 5 |
| DB 0 |
| DB $L$SEH_prologue_gcm_gmult_ssse3_4-$L$SEH_begin_gcm_gmult_ssse3_1 |
| DB 168 |
| DW 1 |
| DB $L$SEH_prologue_gcm_gmult_ssse3_3-$L$SEH_begin_gcm_gmult_ssse3_1 |
| DB 104 |
| DW 0 |
| DB $L$SEH_prologue_gcm_gmult_ssse3_2-$L$SEH_begin_gcm_gmult_ssse3_1 |
| DB 66 |
| |
| DW 0 |
| $L$SEH_info_gcm_ghash_ssse3_0: |
| DB 1 ; version 1, no flags |
| DB $L$SEH_endprologue_gcm_ghash_ssse3_6-$L$SEH_begin_gcm_ghash_ssse3_1 |
| DB 7 |
| DB 0 |
| DB $L$SEH_prologue_gcm_ghash_ssse3_5-$L$SEH_begin_gcm_ghash_ssse3_1 |
| DB 184 |
| DW 2 |
| DB $L$SEH_prologue_gcm_ghash_ssse3_4-$L$SEH_begin_gcm_ghash_ssse3_1 |
| DB 168 |
| DW 1 |
| DB $L$SEH_prologue_gcm_ghash_ssse3_3-$L$SEH_begin_gcm_ghash_ssse3_1 |
| DB 104 |
| DW 0 |
| DB $L$SEH_prologue_gcm_ghash_ssse3_2-$L$SEH_begin_gcm_ghash_ssse3_1 |
| DB 98 |
| |
| DW 0 |
| %else |
| ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 |
| ret |
| %endif |
