| // This file is generated from a similarly-named Perl script in the BoringSSL |
| // source tree. Do not edit by hand. |
| |
| #include <openssl/asm_base.h> |
| |
| #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && defined(__APPLE__) |
| .text |
| |
| // |
| // _aes_encrypt_core |
| // |
| // AES-encrypt %xmm0. |
| // |
| // Inputs: |
| // %xmm0 = input |
| // %xmm9-%xmm15 as in _vpaes_preheat |
| // (%rdx) = scheduled keys |
| // |
| // Output in %xmm0 |
| // Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax |
| // Preserves %xmm6 - %xmm8 so you get some local vectors |
| // |
| // |
| |
| .p2align 4 |
| _vpaes_encrypt_core: |
| |
| movq %rdx,%r9 |
| movq $16,%r11 |
| movl 240(%rdx),%eax |
| movdqa %xmm9,%xmm1 |
| movdqa L$k_ipt(%rip),%xmm2 // iptlo |
| pandn %xmm0,%xmm1 |
| movdqu (%r9),%xmm5 // round0 key |
| psrld $4,%xmm1 |
| pand %xmm9,%xmm0 |
| pshufb %xmm0,%xmm2 |
| movdqa L$k_ipt+16(%rip),%xmm0 // ipthi |
| pshufb %xmm1,%xmm0 |
| pxor %xmm5,%xmm2 |
| addq $16,%r9 |
| pxor %xmm2,%xmm0 |
| leaq L$k_mc_backward(%rip),%r10 |
| jmp L$enc_entry |
| |
| .p2align 4 |
| L$enc_loop: |
| // middle of middle round |
| movdqa %xmm13,%xmm4 // 4 : sb1u |
| movdqa %xmm12,%xmm0 // 0 : sb1t |
| pshufb %xmm2,%xmm4 // 4 = sb1u |
| pshufb %xmm3,%xmm0 // 0 = sb1t |
| pxor %xmm5,%xmm4 // 4 = sb1u + k |
| movdqa %xmm15,%xmm5 // 4 : sb2u |
| pxor %xmm4,%xmm0 // 0 = A |
| movdqa -64(%r11,%r10,1),%xmm1 // .Lk_mc_forward[] |
| pshufb %xmm2,%xmm5 // 4 = sb2u |
| movdqa (%r11,%r10,1),%xmm4 // .Lk_mc_backward[] |
| movdqa %xmm14,%xmm2 // 2 : sb2t |
| pshufb %xmm3,%xmm2 // 2 = sb2t |
| movdqa %xmm0,%xmm3 // 3 = A |
| pxor %xmm5,%xmm2 // 2 = 2A |
| pshufb %xmm1,%xmm0 // 0 = B |
| addq $16,%r9 // next key |
| pxor %xmm2,%xmm0 // 0 = 2A+B |
| pshufb %xmm4,%xmm3 // 3 = D |
| addq $16,%r11 // next mc |
| pxor %xmm0,%xmm3 // 3 = 2A+B+D |
| pshufb %xmm1,%xmm0 // 0 = 2B+C |
| andq $0x30,%r11 // ... mod 4 |
| subq $1,%rax // nr-- |
| pxor %xmm3,%xmm0 // 0 = 2A+3B+C+D |
| |
| L$enc_entry: |
| // top of round |
| movdqa %xmm9,%xmm1 // 1 : i |
| movdqa %xmm11,%xmm5 // 2 : a/k |
| pandn %xmm0,%xmm1 // 1 = i<<4 |
| psrld $4,%xmm1 // 1 = i |
| pand %xmm9,%xmm0 // 0 = k |
| pshufb %xmm0,%xmm5 // 2 = a/k |
| movdqa %xmm10,%xmm3 // 3 : 1/i |
| pxor %xmm1,%xmm0 // 0 = j |
| pshufb %xmm1,%xmm3 // 3 = 1/i |
| movdqa %xmm10,%xmm4 // 4 : 1/j |
| pxor %xmm5,%xmm3 // 3 = iak = 1/i + a/k |
| pshufb %xmm0,%xmm4 // 4 = 1/j |
| movdqa %xmm10,%xmm2 // 2 : 1/iak |
| pxor %xmm5,%xmm4 // 4 = jak = 1/j + a/k |
| pshufb %xmm3,%xmm2 // 2 = 1/iak |
| movdqa %xmm10,%xmm3 // 3 : 1/jak |
| pxor %xmm0,%xmm2 // 2 = io |
| pshufb %xmm4,%xmm3 // 3 = 1/jak |
| movdqu (%r9),%xmm5 |
| pxor %xmm1,%xmm3 // 3 = jo |
| jnz L$enc_loop |
| |
| // middle of last round |
| movdqa -96(%r10),%xmm4 // 3 : sbou .Lk_sbo |
| movdqa -80(%r10),%xmm0 // 0 : sbot .Lk_sbo+16 |
| pshufb %xmm2,%xmm4 // 4 = sbou |
| pxor %xmm5,%xmm4 // 4 = sb1u + k |
| pshufb %xmm3,%xmm0 // 0 = sb1t |
| movdqa 64(%r11,%r10,1),%xmm1 // .Lk_sr[] |
| pxor %xmm4,%xmm0 // 0 = A |
| pshufb %xmm1,%xmm0 |
| ret |
| |
| |
| |
| // |
| // _aes_encrypt_core_2x |
| // |
| // AES-encrypt %xmm0 and %xmm6 in parallel. |
| // |
| // Inputs: |
| // %xmm0 and %xmm6 = input |
| // %xmm9 and %xmm10 as in _vpaes_preheat |
| // (%rdx) = scheduled keys |
| // |
| // Output in %xmm0 and %xmm6 |
| // Clobbers %xmm1-%xmm5, %xmm7, %xmm8, %xmm11-%xmm13, %r9, %r10, %r11, %rax |
| // Preserves %xmm14 and %xmm15 |
| // |
| // This function stitches two parallel instances of _vpaes_encrypt_core. x86_64 |
| // provides 16 XMM registers. _vpaes_encrypt_core computes over six registers |
| // (%xmm0-%xmm5) and additionally uses seven registers with preloaded constants |
| // from _vpaes_preheat (%xmm9-%xmm15). This does not quite fit two instances, |
| // so we spill some of %xmm9 through %xmm15 back to memory. We keep %xmm9 and |
| // %xmm10 in registers as these values are used several times in a row. The |
| // remainder are read once per round and are spilled to memory. This leaves two |
| // registers preserved for the caller. |
| // |
| // Thus, of the two _vpaes_encrypt_core instances, the first uses (%xmm0-%xmm5) |
| // as before. The second uses %xmm6-%xmm8,%xmm11-%xmm13. (Add 6 to %xmm2 and |
| // below. Add 8 to %xmm3 and up.) Instructions in the second instance are |
| // indented by one space. |
| // |
| // |
| |
| .p2align 4 |
| .alt_entry _vpaes_encrypt_core_2x |
| _vpaes_encrypt_core_2x: |
| |
| movq %rdx,%r9 |
| movq $16,%r11 |
| movl 240(%rdx),%eax |
| movdqa %xmm9,%xmm1 |
| movdqa %xmm9,%xmm7 |
| movdqa L$k_ipt(%rip),%xmm2 // iptlo |
| movdqa %xmm2,%xmm8 |
| pandn %xmm0,%xmm1 |
| pandn %xmm6,%xmm7 |
| movdqu (%r9),%xmm5 // round0 key |
| // Also use %xmm5 in the second instance. |
| psrld $4,%xmm1 |
| psrld $4,%xmm7 |
| pand %xmm9,%xmm0 |
| pand %xmm9,%xmm6 |
| pshufb %xmm0,%xmm2 |
| pshufb %xmm6,%xmm8 |
| movdqa L$k_ipt+16(%rip),%xmm0 // ipthi |
| movdqa %xmm0,%xmm6 |
| pshufb %xmm1,%xmm0 |
| pshufb %xmm7,%xmm6 |
| pxor %xmm5,%xmm2 |
| pxor %xmm5,%xmm8 |
| addq $16,%r9 |
| pxor %xmm2,%xmm0 |
| pxor %xmm8,%xmm6 |
| leaq L$k_mc_backward(%rip),%r10 |
| jmp L$enc2x_entry |
| |
| .p2align 4 |
| L$enc2x_loop: |
| // middle of middle round |
| movdqa L$k_sb1(%rip),%xmm4 // 4 : sb1u |
| movdqa L$k_sb1+16(%rip),%xmm0 // 0 : sb1t |
| movdqa %xmm4,%xmm12 |
| movdqa %xmm0,%xmm6 |
| pshufb %xmm2,%xmm4 // 4 = sb1u |
| pshufb %xmm8,%xmm12 |
| pshufb %xmm3,%xmm0 // 0 = sb1t |
| pshufb %xmm11,%xmm6 |
| pxor %xmm5,%xmm4 // 4 = sb1u + k |
| pxor %xmm5,%xmm12 |
| movdqa L$k_sb2(%rip),%xmm5 // 4 : sb2u |
| movdqa %xmm5,%xmm13 |
| pxor %xmm4,%xmm0 // 0 = A |
| pxor %xmm12,%xmm6 |
| movdqa -64(%r11,%r10,1),%xmm1 // .Lk_mc_forward[] |
| // Also use %xmm1 in the second instance. |
| pshufb %xmm2,%xmm5 // 4 = sb2u |
| pshufb %xmm8,%xmm13 |
| movdqa (%r11,%r10,1),%xmm4 // .Lk_mc_backward[] |
| // Also use %xmm4 in the second instance. |
| movdqa L$k_sb2+16(%rip),%xmm2 // 2 : sb2t |
| movdqa %xmm2,%xmm8 |
| pshufb %xmm3,%xmm2 // 2 = sb2t |
| pshufb %xmm11,%xmm8 |
| movdqa %xmm0,%xmm3 // 3 = A |
| movdqa %xmm6,%xmm11 |
| pxor %xmm5,%xmm2 // 2 = 2A |
| pxor %xmm13,%xmm8 |
| pshufb %xmm1,%xmm0 // 0 = B |
| pshufb %xmm1,%xmm6 |
| addq $16,%r9 // next key |
| pxor %xmm2,%xmm0 // 0 = 2A+B |
| pxor %xmm8,%xmm6 |
| pshufb %xmm4,%xmm3 // 3 = D |
| pshufb %xmm4,%xmm11 |
| addq $16,%r11 // next mc |
| pxor %xmm0,%xmm3 // 3 = 2A+B+D |
| pxor %xmm6,%xmm11 |
| pshufb %xmm1,%xmm0 // 0 = 2B+C |
| pshufb %xmm1,%xmm6 |
| andq $0x30,%r11 // ... mod 4 |
| subq $1,%rax // nr-- |
| pxor %xmm3,%xmm0 // 0 = 2A+3B+C+D |
| pxor %xmm11,%xmm6 |
| |
| L$enc2x_entry: |
| // top of round |
| movdqa %xmm9,%xmm1 // 1 : i |
| movdqa %xmm9,%xmm7 |
| movdqa L$k_inv+16(%rip),%xmm5 // 2 : a/k |
| movdqa %xmm5,%xmm13 |
| pandn %xmm0,%xmm1 // 1 = i<<4 |
| pandn %xmm6,%xmm7 |
| psrld $4,%xmm1 // 1 = i |
| psrld $4,%xmm7 |
| pand %xmm9,%xmm0 // 0 = k |
| pand %xmm9,%xmm6 |
| pshufb %xmm0,%xmm5 // 2 = a/k |
| pshufb %xmm6,%xmm13 |
| movdqa %xmm10,%xmm3 // 3 : 1/i |
| movdqa %xmm10,%xmm11 |
| pxor %xmm1,%xmm0 // 0 = j |
| pxor %xmm7,%xmm6 |
| pshufb %xmm1,%xmm3 // 3 = 1/i |
| pshufb %xmm7,%xmm11 |
| movdqa %xmm10,%xmm4 // 4 : 1/j |
| movdqa %xmm10,%xmm12 |
| pxor %xmm5,%xmm3 // 3 = iak = 1/i + a/k |
| pxor %xmm13,%xmm11 |
| pshufb %xmm0,%xmm4 // 4 = 1/j |
| pshufb %xmm6,%xmm12 |
| movdqa %xmm10,%xmm2 // 2 : 1/iak |
| movdqa %xmm10,%xmm8 |
| pxor %xmm5,%xmm4 // 4 = jak = 1/j + a/k |
| pxor %xmm13,%xmm12 |
| pshufb %xmm3,%xmm2 // 2 = 1/iak |
| pshufb %xmm11,%xmm8 |
| movdqa %xmm10,%xmm3 // 3 : 1/jak |
| movdqa %xmm10,%xmm11 |
| pxor %xmm0,%xmm2 // 2 = io |
| pxor %xmm6,%xmm8 |
| pshufb %xmm4,%xmm3 // 3 = 1/jak |
| pshufb %xmm12,%xmm11 |
| movdqu (%r9),%xmm5 |
| // Also use %xmm5 in the second instance. |
| pxor %xmm1,%xmm3 // 3 = jo |
| pxor %xmm7,%xmm11 |
| jnz L$enc2x_loop |
| |
| // middle of last round |
| movdqa -96(%r10),%xmm4 // 3 : sbou .Lk_sbo |
| movdqa -80(%r10),%xmm0 // 0 : sbot .Lk_sbo+16 |
| movdqa %xmm4,%xmm12 |
| movdqa %xmm0,%xmm6 |
| pshufb %xmm2,%xmm4 // 4 = sbou |
| pshufb %xmm8,%xmm12 |
| pxor %xmm5,%xmm4 // 4 = sb1u + k |
| pxor %xmm5,%xmm12 |
| pshufb %xmm3,%xmm0 // 0 = sb1t |
| pshufb %xmm11,%xmm6 |
| movdqa 64(%r11,%r10,1),%xmm1 // .Lk_sr[] |
| // Also use %xmm1 in the second instance. |
| pxor %xmm4,%xmm0 // 0 = A |
| pxor %xmm12,%xmm6 |
| pshufb %xmm1,%xmm0 |
| pshufb %xmm1,%xmm6 |
| ret |
| |
| |
| |
| // |
| // Decryption core |
| // |
| // Same API as encryption core. |
| // |
| |
| .p2align 4 |
| .alt_entry _vpaes_decrypt_core |
| _vpaes_decrypt_core: |
| |
| movq %rdx,%r9 // load key |
| movl 240(%rdx),%eax |
| movdqa %xmm9,%xmm1 |
| movdqa L$k_dipt(%rip),%xmm2 // iptlo |
| pandn %xmm0,%xmm1 |
| movq %rax,%r11 |
| psrld $4,%xmm1 |
| movdqu (%r9),%xmm5 // round0 key |
| shlq $4,%r11 |
| pand %xmm9,%xmm0 |
| pshufb %xmm0,%xmm2 |
| movdqa L$k_dipt+16(%rip),%xmm0 // ipthi |
| xorq $0x30,%r11 |
| leaq L$k_dsbd(%rip),%r10 |
| pshufb %xmm1,%xmm0 |
| andq $0x30,%r11 |
| pxor %xmm5,%xmm2 |
| movdqa L$k_mc_forward+48(%rip),%xmm5 |
| pxor %xmm2,%xmm0 |
| addq $16,%r9 |
| addq %r10,%r11 |
| jmp L$dec_entry |
| |
| .p2align 4 |
| L$dec_loop: |
| // |
| // Inverse mix columns |
| // |
| movdqa -32(%r10),%xmm4 // 4 : sb9u |
| movdqa -16(%r10),%xmm1 // 0 : sb9t |
| pshufb %xmm2,%xmm4 // 4 = sb9u |
| pshufb %xmm3,%xmm1 // 0 = sb9t |
| pxor %xmm4,%xmm0 |
| movdqa 0(%r10),%xmm4 // 4 : sbdu |
| pxor %xmm1,%xmm0 // 0 = ch |
| movdqa 16(%r10),%xmm1 // 0 : sbdt |
| |
| pshufb %xmm2,%xmm4 // 4 = sbdu |
| pshufb %xmm5,%xmm0 // MC ch |
| pshufb %xmm3,%xmm1 // 0 = sbdt |
| pxor %xmm4,%xmm0 // 4 = ch |
| movdqa 32(%r10),%xmm4 // 4 : sbbu |
| pxor %xmm1,%xmm0 // 0 = ch |
| movdqa 48(%r10),%xmm1 // 0 : sbbt |
| |
| pshufb %xmm2,%xmm4 // 4 = sbbu |
| pshufb %xmm5,%xmm0 // MC ch |
| pshufb %xmm3,%xmm1 // 0 = sbbt |
| pxor %xmm4,%xmm0 // 4 = ch |
| movdqa 64(%r10),%xmm4 // 4 : sbeu |
| pxor %xmm1,%xmm0 // 0 = ch |
| movdqa 80(%r10),%xmm1 // 0 : sbet |
| |
| pshufb %xmm2,%xmm4 // 4 = sbeu |
| pshufb %xmm5,%xmm0 // MC ch |
| pshufb %xmm3,%xmm1 // 0 = sbet |
| pxor %xmm4,%xmm0 // 4 = ch |
| addq $16,%r9 // next round key |
| palignr $12,%xmm5,%xmm5 |
| pxor %xmm1,%xmm0 // 0 = ch |
| subq $1,%rax // nr-- |
| |
| L$dec_entry: |
| // top of round |
| movdqa %xmm9,%xmm1 // 1 : i |
| pandn %xmm0,%xmm1 // 1 = i<<4 |
| movdqa %xmm11,%xmm2 // 2 : a/k |
| psrld $4,%xmm1 // 1 = i |
| pand %xmm9,%xmm0 // 0 = k |
| pshufb %xmm0,%xmm2 // 2 = a/k |
| movdqa %xmm10,%xmm3 // 3 : 1/i |
| pxor %xmm1,%xmm0 // 0 = j |
| pshufb %xmm1,%xmm3 // 3 = 1/i |
| movdqa %xmm10,%xmm4 // 4 : 1/j |
| pxor %xmm2,%xmm3 // 3 = iak = 1/i + a/k |
| pshufb %xmm0,%xmm4 // 4 = 1/j |
| pxor %xmm2,%xmm4 // 4 = jak = 1/j + a/k |
| movdqa %xmm10,%xmm2 // 2 : 1/iak |
| pshufb %xmm3,%xmm2 // 2 = 1/iak |
| movdqa %xmm10,%xmm3 // 3 : 1/jak |
| pxor %xmm0,%xmm2 // 2 = io |
| pshufb %xmm4,%xmm3 // 3 = 1/jak |
| movdqu (%r9),%xmm0 |
| pxor %xmm1,%xmm3 // 3 = jo |
| jnz L$dec_loop |
| |
| // middle of last round |
| movdqa 96(%r10),%xmm4 // 3 : sbou |
| pshufb %xmm2,%xmm4 // 4 = sbou |
| pxor %xmm0,%xmm4 // 4 = sb1u + k |
| movdqa 112(%r10),%xmm0 // 0 : sbot |
| movdqa -352(%r11),%xmm2 // .Lk_sr-.Lk_dsbd=-0x160 |
| pshufb %xmm3,%xmm0 // 0 = sb1t |
| pxor %xmm4,%xmm0 // 0 = A |
| pshufb %xmm2,%xmm0 |
| ret |
| |
| |
| |
| // ###################################################### |
| // ## |
| // AES key schedule ## |
| // ## |
| // ###################################################### |
| |
| .p2align 4 |
| .alt_entry _vpaes_schedule_core |
| _vpaes_schedule_core: |
| |
| // rdi = key |
| // rsi = size in bits |
| // rdx = buffer |
| // rcx = direction. 0=encrypt, 1=decrypt |
| |
| call _vpaes_preheat // load the tables |
| movdqa L$k_rcon(%rip),%xmm8 // load rcon |
| movdqu (%rdi),%xmm0 // load key (unaligned) |
| |
| // input transform |
| movdqa %xmm0,%xmm3 |
| leaq L$k_ipt(%rip),%r11 |
| call _vpaes_schedule_transform |
| movdqa %xmm0,%xmm7 |
| |
| leaq L$k_sr(%rip),%r10 |
| testq %rcx,%rcx |
| jnz L$schedule_am_decrypting |
| |
| // encrypting, output zeroth round key after transform |
| movdqu %xmm0,(%rdx) |
| jmp L$schedule_go |
| |
| L$schedule_am_decrypting: |
| // decrypting, output zeroth round key after shiftrows |
| movdqa (%r8,%r10,1),%xmm1 |
| pshufb %xmm1,%xmm3 |
| movdqu %xmm3,(%rdx) |
| xorq $0x30,%r8 |
| |
| L$schedule_go: |
| cmpl $192,%esi |
| ja L$schedule_256 |
| je L$schedule_192 |
| // 128: fall though |
| |
| // |
| // .schedule_128 |
| // |
| // 128-bit specific part of key schedule. |
| // |
| // This schedule is really simple, because all its parts |
| // are accomplished by the subroutines. |
| // |
| L$schedule_128: |
| movl $10,%esi |
| |
| L$oop_schedule_128: |
| call _vpaes_schedule_round |
| decq %rsi |
| jz L$schedule_mangle_last |
| call _vpaes_schedule_mangle // write output |
| jmp L$oop_schedule_128 |
| |
| // |
| // .aes_schedule_192 |
| // |
| // 192-bit specific part of key schedule. |
| // |
| // The main body of this schedule is the same as the 128-bit |
| // schedule, but with more smearing. The long, high side is |
| // stored in %xmm7 as before, and the short, low side is in |
| // the high bits of %xmm6. |
| // |
| // This schedule is somewhat nastier, however, because each |
| // round produces 192 bits of key material, or 1.5 round keys. |
| // Therefore, on each cycle we do 2 rounds and produce 3 round |
| // keys. |
| // |
| .p2align 4 |
| L$schedule_192: |
| movdqu 8(%rdi),%xmm0 // load key part 2 (very unaligned) |
| call _vpaes_schedule_transform // input transform |
| movdqa %xmm0,%xmm6 // save short part |
| pxor %xmm4,%xmm4 // clear 4 |
| movhlps %xmm4,%xmm6 // clobber low side with zeros |
| movl $4,%esi |
| |
| L$oop_schedule_192: |
| call _vpaes_schedule_round |
| palignr $8,%xmm6,%xmm0 |
| call _vpaes_schedule_mangle // save key n |
| call _vpaes_schedule_192_smear |
| call _vpaes_schedule_mangle // save key n+1 |
| call _vpaes_schedule_round |
| decq %rsi |
| jz L$schedule_mangle_last |
| call _vpaes_schedule_mangle // save key n+2 |
| call _vpaes_schedule_192_smear |
| jmp L$oop_schedule_192 |
| |
| // |
| // .aes_schedule_256 |
| // |
| // 256-bit specific part of key schedule. |
| // |
| // The structure here is very similar to the 128-bit |
| // schedule, but with an additional "low side" in |
| // %xmm6. The low side's rounds are the same as the |
| // high side's, except no rcon and no rotation. |
| // |
| .p2align 4 |
| L$schedule_256: |
| movdqu 16(%rdi),%xmm0 // load key part 2 (unaligned) |
| call _vpaes_schedule_transform // input transform |
| movl $7,%esi |
| |
| L$oop_schedule_256: |
| call _vpaes_schedule_mangle // output low result |
| movdqa %xmm0,%xmm6 // save cur_lo in xmm6 |
| |
| // high round |
| call _vpaes_schedule_round |
| decq %rsi |
| jz L$schedule_mangle_last |
| call _vpaes_schedule_mangle |
| |
| // low round. swap xmm7 and xmm6 |
| pshufd $0xFF,%xmm0,%xmm0 |
| movdqa %xmm7,%xmm5 |
| movdqa %xmm6,%xmm7 |
| call _vpaes_schedule_low_round |
| movdqa %xmm5,%xmm7 |
| |
| jmp L$oop_schedule_256 |
| |
| |
| // |
| // .aes_schedule_mangle_last |
| // |
| // Mangler for last round of key schedule |
| // Mangles %xmm0 |
| // when encrypting, outputs out(%xmm0) ^ 63 |
| // when decrypting, outputs unskew(%xmm0) |
| // |
| // Always called right before return... jumps to cleanup and exits |
| // |
| .p2align 4 |
| L$schedule_mangle_last: |
| // schedule last round key from xmm0 |
| leaq L$k_deskew(%rip),%r11 // prepare to deskew |
| testq %rcx,%rcx |
| jnz L$schedule_mangle_last_dec |
| |
| // encrypting |
| movdqa (%r8,%r10,1),%xmm1 |
| pshufb %xmm1,%xmm0 // output permute |
| leaq L$k_opt(%rip),%r11 // prepare to output transform |
| addq $32,%rdx |
| |
| L$schedule_mangle_last_dec: |
| addq $-16,%rdx |
| pxor L$k_s63(%rip),%xmm0 |
| call _vpaes_schedule_transform // output transform |
| movdqu %xmm0,(%rdx) // save last key |
| |
| // cleanup |
| pxor %xmm0,%xmm0 |
| pxor %xmm1,%xmm1 |
| pxor %xmm2,%xmm2 |
| pxor %xmm3,%xmm3 |
| pxor %xmm4,%xmm4 |
| pxor %xmm5,%xmm5 |
| pxor %xmm6,%xmm6 |
| pxor %xmm7,%xmm7 |
| ret |
| |
| |
| |
| // |
| // .aes_schedule_192_smear |
| // |
| // Smear the short, low side in the 192-bit key schedule. |
| // |
| // Inputs: |
| // %xmm7: high side, b a x y |
| // %xmm6: low side, d c 0 0 |
| // %xmm13: 0 |
| // |
| // Outputs: |
| // %xmm6: b+c+d b+c 0 0 |
| // %xmm0: b+c+d b+c b a |
| // |
| |
| .p2align 4 |
| .alt_entry _vpaes_schedule_192_smear |
| _vpaes_schedule_192_smear: |
| |
| pshufd $0x80,%xmm6,%xmm1 // d c 0 0 -> c 0 0 0 |
| pshufd $0xFE,%xmm7,%xmm0 // b a _ _ -> b b b a |
| pxor %xmm1,%xmm6 // -> c+d c 0 0 |
| pxor %xmm1,%xmm1 |
| pxor %xmm0,%xmm6 // -> b+c+d b+c b a |
| movdqa %xmm6,%xmm0 |
| movhlps %xmm1,%xmm6 // clobber low side with zeros |
| ret |
| |
| |
| |
| // |
| // .aes_schedule_round |
| // |
| // Runs one main round of the key schedule on %xmm0, %xmm7 |
| // |
| // Specifically, runs subbytes on the high dword of %xmm0 |
| // then rotates it by one byte and xors into the low dword of |
| // %xmm7. |
| // |
| // Adds rcon from low byte of %xmm8, then rotates %xmm8 for |
| // next rcon. |
| // |
| // Smears the dwords of %xmm7 by xoring the low into the |
| // second low, result into third, result into highest. |
| // |
| // Returns results in %xmm7 = %xmm0. |
| // Clobbers %xmm1-%xmm4, %r11. |
| // |
| |
| .p2align 4 |
| .alt_entry _vpaes_schedule_round |
| _vpaes_schedule_round: |
| |
| // extract rcon from xmm8 |
| pxor %xmm1,%xmm1 |
| palignr $15,%xmm8,%xmm1 |
| palignr $15,%xmm8,%xmm8 |
| pxor %xmm1,%xmm7 |
| |
| // rotate |
| pshufd $0xFF,%xmm0,%xmm0 |
| palignr $1,%xmm0,%xmm0 |
| |
| // fall through... |
| |
| // low round: same as high round, but no rotation and no rcon. |
| .alt_entry _vpaes_schedule_low_round |
| _vpaes_schedule_low_round: |
| // smear xmm7 |
| movdqa %xmm7,%xmm1 |
| pslldq $4,%xmm7 |
| pxor %xmm1,%xmm7 |
| movdqa %xmm7,%xmm1 |
| pslldq $8,%xmm7 |
| pxor %xmm1,%xmm7 |
| pxor L$k_s63(%rip),%xmm7 |
| |
| // subbytes |
| movdqa %xmm9,%xmm1 |
| pandn %xmm0,%xmm1 |
| psrld $4,%xmm1 // 1 = i |
| pand %xmm9,%xmm0 // 0 = k |
| movdqa %xmm11,%xmm2 // 2 : a/k |
| pshufb %xmm0,%xmm2 // 2 = a/k |
| pxor %xmm1,%xmm0 // 0 = j |
| movdqa %xmm10,%xmm3 // 3 : 1/i |
| pshufb %xmm1,%xmm3 // 3 = 1/i |
| pxor %xmm2,%xmm3 // 3 = iak = 1/i + a/k |
| movdqa %xmm10,%xmm4 // 4 : 1/j |
| pshufb %xmm0,%xmm4 // 4 = 1/j |
| pxor %xmm2,%xmm4 // 4 = jak = 1/j + a/k |
| movdqa %xmm10,%xmm2 // 2 : 1/iak |
| pshufb %xmm3,%xmm2 // 2 = 1/iak |
| pxor %xmm0,%xmm2 // 2 = io |
| movdqa %xmm10,%xmm3 // 3 : 1/jak |
| pshufb %xmm4,%xmm3 // 3 = 1/jak |
| pxor %xmm1,%xmm3 // 3 = jo |
| movdqa %xmm13,%xmm4 // 4 : sbou |
| pshufb %xmm2,%xmm4 // 4 = sbou |
| movdqa %xmm12,%xmm0 // 0 : sbot |
| pshufb %xmm3,%xmm0 // 0 = sb1t |
| pxor %xmm4,%xmm0 // 0 = sbox output |
| |
| // add in smeared stuff |
| pxor %xmm7,%xmm0 |
| movdqa %xmm0,%xmm7 |
| ret |
| |
| |
| |
| // |
| // .aes_schedule_transform |
| // |
| // Linear-transform %xmm0 according to tables at (%r11) |
| // |
| // Requires that %xmm9 = 0x0F0F... as in preheat |
| // Output in %xmm0 |
| // Clobbers %xmm1, %xmm2 |
| // |
| |
| .p2align 4 |
| .alt_entry _vpaes_schedule_transform |
| _vpaes_schedule_transform: |
| |
| movdqa %xmm9,%xmm1 |
| pandn %xmm0,%xmm1 |
| psrld $4,%xmm1 |
| pand %xmm9,%xmm0 |
| movdqa (%r11),%xmm2 // lo |
| pshufb %xmm0,%xmm2 |
| movdqa 16(%r11),%xmm0 // hi |
| pshufb %xmm1,%xmm0 |
| pxor %xmm2,%xmm0 |
| ret |
| |
| |
| |
| // |
| // .aes_schedule_mangle |
| // |
| // Mangle xmm0 from (basis-transformed) standard version |
| // to our version. |
| // |
| // On encrypt, |
| // xor with 0x63 |
| // multiply by circulant 0,1,1,1 |
| // apply shiftrows transform |
| // |
| // On decrypt, |
| // xor with 0x63 |
| // multiply by "inverse mixcolumns" circulant E,B,D,9 |
| // deskew |
| // apply shiftrows transform |
| // |
| // |
| // Writes out to (%rdx), and increments or decrements it |
| // Keeps track of round number mod 4 in %r8 |
| // Preserves xmm0 |
| // Clobbers xmm1-xmm5 |
| // |
| |
| .p2align 4 |
| .alt_entry _vpaes_schedule_mangle |
| _vpaes_schedule_mangle: |
| |
| movdqa %xmm0,%xmm4 // save xmm0 for later |
| movdqa L$k_mc_forward(%rip),%xmm5 |
| testq %rcx,%rcx |
| jnz L$schedule_mangle_dec |
| |
| // encrypting |
| addq $16,%rdx |
| pxor L$k_s63(%rip),%xmm4 |
| pshufb %xmm5,%xmm4 |
| movdqa %xmm4,%xmm3 |
| pshufb %xmm5,%xmm4 |
| pxor %xmm4,%xmm3 |
| pshufb %xmm5,%xmm4 |
| pxor %xmm4,%xmm3 |
| |
| jmp L$schedule_mangle_both |
| .p2align 4 |
| L$schedule_mangle_dec: |
| // inverse mix columns |
| leaq L$k_dksd(%rip),%r11 |
| movdqa %xmm9,%xmm1 |
| pandn %xmm4,%xmm1 |
| psrld $4,%xmm1 // 1 = hi |
| pand %xmm9,%xmm4 // 4 = lo |
| |
| movdqa 0(%r11),%xmm2 |
| pshufb %xmm4,%xmm2 |
| movdqa 16(%r11),%xmm3 |
| pshufb %xmm1,%xmm3 |
| pxor %xmm2,%xmm3 |
| pshufb %xmm5,%xmm3 |
| |
| movdqa 32(%r11),%xmm2 |
| pshufb %xmm4,%xmm2 |
| pxor %xmm3,%xmm2 |
| movdqa 48(%r11),%xmm3 |
| pshufb %xmm1,%xmm3 |
| pxor %xmm2,%xmm3 |
| pshufb %xmm5,%xmm3 |
| |
| movdqa 64(%r11),%xmm2 |
| pshufb %xmm4,%xmm2 |
| pxor %xmm3,%xmm2 |
| movdqa 80(%r11),%xmm3 |
| pshufb %xmm1,%xmm3 |
| pxor %xmm2,%xmm3 |
| pshufb %xmm5,%xmm3 |
| |
| movdqa 96(%r11),%xmm2 |
| pshufb %xmm4,%xmm2 |
| pxor %xmm3,%xmm2 |
| movdqa 112(%r11),%xmm3 |
| pshufb %xmm1,%xmm3 |
| pxor %xmm2,%xmm3 |
| |
| addq $-16,%rdx |
| |
| L$schedule_mangle_both: |
| movdqa (%r8,%r10,1),%xmm1 |
| pshufb %xmm1,%xmm3 |
| addq $-16,%r8 |
| andq $0x30,%r8 |
| movdqu %xmm3,(%rdx) |
| ret |
| |
| |
| |
| // |
| // Interface to OpenSSL |
| // |
| .globl _vpaes_set_encrypt_key |
| .private_extern _vpaes_set_encrypt_key |
| |
| .p2align 4 |
| .alt_entry _vpaes_set_encrypt_key |
| _vpaes_set_encrypt_key: |
| |
| _CET_ENDBR |
| #ifdef BORINGSSL_DISPATCH_TEST |
| |
| movb $1,_BORINGSSL_function_hit+5(%rip) |
| #endif |
| |
| movl %esi,%eax |
| shrl $5,%eax |
| addl $5,%eax |
| movl %eax,240(%rdx) // AES_KEY->rounds = nbits/32+5; |
| |
| movl $0,%ecx |
| movl $0x30,%r8d |
| call _vpaes_schedule_core |
| xorl %eax,%eax |
| ret |
| |
| |
| |
| .globl _vpaes_set_decrypt_key |
| .private_extern _vpaes_set_decrypt_key |
| |
| .p2align 4 |
| .alt_entry _vpaes_set_decrypt_key |
| _vpaes_set_decrypt_key: |
| |
| _CET_ENDBR |
| movl %esi,%eax |
| shrl $5,%eax |
| addl $5,%eax |
| movl %eax,240(%rdx) // AES_KEY->rounds = nbits/32+5; |
| shll $4,%eax |
| leaq 16(%rdx,%rax,1),%rdx |
| |
| movl $1,%ecx |
| movl %esi,%r8d |
| shrl $1,%r8d |
| andl $32,%r8d |
| xorl $32,%r8d // nbits==192?0:32 |
| call _vpaes_schedule_core |
| xorl %eax,%eax |
| ret |
| |
| |
| |
| .globl _vpaes_encrypt |
| .private_extern _vpaes_encrypt |
| |
| .p2align 4 |
| .alt_entry _vpaes_encrypt |
| _vpaes_encrypt: |
| |
| _CET_ENDBR |
| #ifdef BORINGSSL_DISPATCH_TEST |
| |
| movb $1,_BORINGSSL_function_hit+4(%rip) |
| #endif |
| movdqu (%rdi),%xmm0 |
| call _vpaes_preheat |
| call _vpaes_encrypt_core |
| movdqu %xmm0,(%rsi) |
| ret |
| |
| |
| |
| .globl _vpaes_decrypt |
| .private_extern _vpaes_decrypt |
| |
| .p2align 4 |
| .alt_entry _vpaes_decrypt |
| _vpaes_decrypt: |
| |
| _CET_ENDBR |
| movdqu (%rdi),%xmm0 |
| call _vpaes_preheat |
| call _vpaes_decrypt_core |
| movdqu %xmm0,(%rsi) |
| ret |
| |
| |
| .globl _vpaes_cbc_encrypt |
| .private_extern _vpaes_cbc_encrypt |
| |
| .p2align 4 |
| .alt_entry _vpaes_cbc_encrypt |
| _vpaes_cbc_encrypt: |
| |
| _CET_ENDBR |
| xchgq %rcx,%rdx |
| subq $16,%rcx |
| jc L$cbc_abort |
| movdqu (%r8),%xmm6 // load IV |
| subq %rdi,%rsi |
| call _vpaes_preheat |
| cmpl $0,%r9d |
| je L$cbc_dec_loop |
| jmp L$cbc_enc_loop |
| .p2align 4 |
| L$cbc_enc_loop: |
| movdqu (%rdi),%xmm0 |
| pxor %xmm6,%xmm0 |
| call _vpaes_encrypt_core |
| movdqa %xmm0,%xmm6 |
| movdqu %xmm0,(%rsi,%rdi,1) |
| leaq 16(%rdi),%rdi |
| subq $16,%rcx |
| jnc L$cbc_enc_loop |
| jmp L$cbc_done |
| .p2align 4 |
| L$cbc_dec_loop: |
| movdqu (%rdi),%xmm0 |
| movdqa %xmm0,%xmm7 |
| call _vpaes_decrypt_core |
| pxor %xmm6,%xmm0 |
| movdqa %xmm7,%xmm6 |
| movdqu %xmm0,(%rsi,%rdi,1) |
| leaq 16(%rdi),%rdi |
| subq $16,%rcx |
| jnc L$cbc_dec_loop |
| L$cbc_done: |
| movdqu %xmm6,(%r8) // save IV |
| L$cbc_abort: |
| ret |
| |
| |
| .globl _vpaes_ctr32_encrypt_blocks |
| .private_extern _vpaes_ctr32_encrypt_blocks |
| |
| .p2align 4 |
| .alt_entry _vpaes_ctr32_encrypt_blocks |
| _vpaes_ctr32_encrypt_blocks: |
| |
| _CET_ENDBR |
| // _vpaes_encrypt_core and _vpaes_encrypt_core_2x expect the key in %rdx. |
| xchgq %rcx,%rdx |
| testq %rcx,%rcx |
| jz L$ctr32_abort |
| movdqu (%r8),%xmm0 // Load IV. |
| movdqa L$ctr_add_one(%rip),%xmm8 |
| subq %rdi,%rsi // This allows only incrementing %rdi. |
| call _vpaes_preheat |
| movdqa %xmm0,%xmm6 |
| pshufb L$rev_ctr(%rip),%xmm6 |
| |
| testq $1,%rcx |
| jz L$ctr32_prep_loop |
| |
| // Handle one block so the remaining block count is even for |
| // _vpaes_encrypt_core_2x. |
| movdqu (%rdi),%xmm7 // Load input. |
| call _vpaes_encrypt_core |
| pxor %xmm7,%xmm0 |
| paddd %xmm8,%xmm6 |
| movdqu %xmm0,(%rsi,%rdi,1) |
| subq $1,%rcx |
| leaq 16(%rdi),%rdi |
| jz L$ctr32_done |
| |
| L$ctr32_prep_loop: |
| // _vpaes_encrypt_core_2x leaves only %xmm14 and %xmm15 as spare |
| // registers. We maintain two byte-swapped counters in them. |
| movdqa %xmm6,%xmm14 |
| movdqa %xmm6,%xmm15 |
| paddd %xmm8,%xmm15 |
| |
| L$ctr32_loop: |
| movdqa L$rev_ctr(%rip),%xmm1 // Set up counters. |
| movdqa %xmm14,%xmm0 |
| movdqa %xmm15,%xmm6 |
| pshufb %xmm1,%xmm0 |
| pshufb %xmm1,%xmm6 |
| call _vpaes_encrypt_core_2x |
| movdqu (%rdi),%xmm1 // Load input. |
| movdqu 16(%rdi),%xmm2 |
| movdqa L$ctr_add_two(%rip),%xmm3 |
| pxor %xmm1,%xmm0 // XOR input. |
| pxor %xmm2,%xmm6 |
| paddd %xmm3,%xmm14 // Increment counters. |
| paddd %xmm3,%xmm15 |
| movdqu %xmm0,(%rsi,%rdi,1) // Write output. |
| movdqu %xmm6,16(%rsi,%rdi,1) |
| subq $2,%rcx // Advance loop. |
| leaq 32(%rdi),%rdi |
| jnz L$ctr32_loop |
| |
| L$ctr32_done: |
| L$ctr32_abort: |
| ret |
| |
| |
| // |
| // _aes_preheat |
| // |
| // Fills register %r10 -> .aes_consts (so you can -fPIC) |
| // and %xmm9-%xmm15 as specified below. |
| // |
| |
| .p2align 4 |
| .alt_entry _vpaes_preheat |
| _vpaes_preheat: |
| |
| leaq L$k_s0F(%rip),%r10 |
| movdqa -32(%r10),%xmm10 // .Lk_inv |
| movdqa -16(%r10),%xmm11 // .Lk_inv+16 |
| movdqa 0(%r10),%xmm9 // .Lk_s0F |
| movdqa 48(%r10),%xmm13 // .Lk_sb1 |
| movdqa 64(%r10),%xmm12 // .Lk_sb1+16 |
| movdqa 80(%r10),%xmm15 // .Lk_sb2 |
| movdqa 96(%r10),%xmm14 // .Lk_sb2+16 |
| ret |
| |
| |
| // ###################################################### |
| // ## |
| // Constants ## |
| // ## |
| // ###################################################### |
| |
| .section __DATA,__const |
| .p2align 6 |
| _vpaes_consts: |
| L$k_inv: // inv, inva |
| .quad 0x0E05060F0D080180, 0x040703090A0B0C02 |
| .quad 0x01040A060F0B0780, 0x030D0E0C02050809 |
| |
| L$k_s0F: // s0F |
| .quad 0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F0F0F0F0F |
| |
| L$k_ipt: // input transform (lo, hi) |
| .quad 0xC2B2E8985A2A7000, 0xCABAE09052227808 |
| .quad 0x4C01307D317C4D00, 0xCD80B1FCB0FDCC81 |
| |
| L$k_sb1: // sb1u, sb1t |
| .quad 0xB19BE18FCB503E00, 0xA5DF7A6E142AF544 |
| .quad 0x3618D415FAE22300, 0x3BF7CCC10D2ED9EF |
| L$k_sb2: // sb2u, sb2t |
| .quad 0xE27A93C60B712400, 0x5EB7E955BC982FCD |
| .quad 0x69EB88400AE12900, 0xC2A163C8AB82234A |
| L$k_sbo: // sbou, sbot |
| .quad 0xD0D26D176FBDC700, 0x15AABF7AC502A878 |
| .quad 0xCFE474A55FBB6A00, 0x8E1E90D1412B35FA |
| |
| L$k_mc_forward: // mc_forward |
| .quad 0x0407060500030201, 0x0C0F0E0D080B0A09 |
| .quad 0x080B0A0904070605, 0x000302010C0F0E0D |
| .quad 0x0C0F0E0D080B0A09, 0x0407060500030201 |
| .quad 0x000302010C0F0E0D, 0x080B0A0904070605 |
| |
| L$k_mc_backward: // mc_backward |
| .quad 0x0605040702010003, 0x0E0D0C0F0A09080B |
| .quad 0x020100030E0D0C0F, 0x0A09080B06050407 |
| .quad 0x0E0D0C0F0A09080B, 0x0605040702010003 |
| .quad 0x0A09080B06050407, 0x020100030E0D0C0F |
| |
| L$k_sr: // sr |
| .quad 0x0706050403020100, 0x0F0E0D0C0B0A0908 |
| .quad 0x030E09040F0A0500, 0x0B06010C07020D08 |
| .quad 0x0F060D040B020900, 0x070E050C030A0108 |
| .quad 0x0B0E0104070A0D00, 0x0306090C0F020508 |
| |
| L$k_rcon: // rcon |
| .quad 0x1F8391B9AF9DEEB6, 0x702A98084D7C7D81 |
| |
| L$k_s63: // s63: all equal to 0x63 transformed |
| .quad 0x5B5B5B5B5B5B5B5B, 0x5B5B5B5B5B5B5B5B |
| |
| L$k_opt: // output transform |
| .quad 0xFF9F4929D6B66000, 0xF7974121DEBE6808 |
| .quad 0x01EDBD5150BCEC00, 0xE10D5DB1B05C0CE0 |
| |
| L$k_deskew: // deskew tables: inverts the sbox's "skew" |
| .quad 0x07E4A34047A4E300, 0x1DFEB95A5DBEF91A |
| .quad 0x5F36B5DC83EA6900, 0x2841C2ABF49D1E77 |
| |
| // |
| // Decryption stuff |
| // Key schedule constants |
| // |
| L$k_dksd: // decryption key schedule: invskew x*D |
| .quad 0xFEB91A5DA3E44700, 0x0740E3A45A1DBEF9 |
| .quad 0x41C277F4B5368300, 0x5FDC69EAAB289D1E |
| L$k_dksb: // decryption key schedule: invskew x*B |
| .quad 0x9A4FCA1F8550D500, 0x03D653861CC94C99 |
| .quad 0x115BEDA7B6FC4A00, 0xD993256F7E3482C8 |
| L$k_dkse: // decryption key schedule: invskew x*E + 0x63 |
| .quad 0xD5031CCA1FC9D600, 0x53859A4C994F5086 |
| .quad 0xA23196054FDC7BE8, 0xCD5EF96A20B31487 |
| L$k_dks9: // decryption key schedule: invskew x*9 |
| .quad 0xB6116FC87ED9A700, 0x4AED933482255BFC |
| .quad 0x4576516227143300, 0x8BB89FACE9DAFDCE |
| |
| // |
| // Decryption stuff |
| // Round function constants |
| // |
| L$k_dipt: // decryption input transform |
| .quad 0x0F505B040B545F00, 0x154A411E114E451A |
| .quad 0x86E383E660056500, 0x12771772F491F194 |
| |
| L$k_dsb9: // decryption sbox output *9*u, *9*t |
| .quad 0x851C03539A86D600, 0xCAD51F504F994CC9 |
| .quad 0xC03B1789ECD74900, 0x725E2C9EB2FBA565 |
| L$k_dsbd: // decryption sbox output *D*u, *D*t |
| .quad 0x7D57CCDFE6B1A200, 0xF56E9B13882A4439 |
| .quad 0x3CE2FAF724C6CB00, 0x2931180D15DEEFD3 |
| L$k_dsbb: // decryption sbox output *B*u, *B*t |
| .quad 0xD022649296B44200, 0x602646F6B0F2D404 |
| .quad 0xC19498A6CD596700, 0xF3FF0C3E3255AA6B |
| L$k_dsbe: // decryption sbox output *E*u, *E*t |
| .quad 0x46F2929626D4D000, 0x2242600464B4F6B0 |
| .quad 0x0C55A6CDFFAAC100, 0x9467F36B98593E32 |
| L$k_dsbo: // decryption sbox final output |
| .quad 0x1387EA537EF94000, 0xC7AA6DB9D4943E2D |
| .quad 0x12D7560F93441D00, 0xCA4B8159D8C58E9C |
| |
| // .Lrev_ctr is a permutation which byte-swaps the counter portion of the IV. |
| L$rev_ctr: |
| .quad 0x0706050403020100, 0x0c0d0e0f0b0a0908 |
| // .Lctr_add_* may be added to a byte-swapped xmm register to increment the |
| // counter. The register must be byte-swapped again to form the actual input. |
| L$ctr_add_one: |
| .quad 0x0000000000000000, 0x0000000100000000 |
| L$ctr_add_two: |
| .quad 0x0000000000000000, 0x0000000200000000 |
| |
| .byte 86,101,99,116,111,114,32,80,101,114,109,117,116,97,116,105,111,110,32,65,69,83,32,102,111,114,32,120,56,54,95,54,52,47,83,83,83,69,51,44,32,77,105,107,101,32,72,97,109,98,117,114,103,32,40,83,116,97,110,102,111,114,100,32,85,110,105,118,101,114,115,105,116,121,41,0 |
| .p2align 6 |
| |
| .text |
| #endif |
