Google Git
Sign in
boringssl / boringssl / b92fcfdc17f3ad794c220a86f4ae6695d0a0fb61 / . / crypto / cipher_extra / asm / chacha20_poly1305_armv8.pl
blob: 9bd7f49d60a8ccdbe13c4645e04e4c41e92a3403 [file] [log] [blame]
#!/usr/bin/env perl
# Copyright (c) 2020, CloudFlare Ltd.
#
# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
##############################################################################
# #
# Author: Vlad Krasnov #
# #
##############################################################################
$flavour = shift;
while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {}
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or
( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or
die "can't locate arm-xlate.pl";
open OUT,"| \"$^X\" $xlate $flavour $output";
*STDOUT=*OUT;
my ($oup,$inp,$inl,$adp,$adl,$keyp,$itr1,$itr2) = ("x0","x1","x2","x3","x4","x5","x6","x7");
my ($acc0,$acc1,$acc2) = map("x$_",(8..10));
my ($t0,$t1,$t2,$t3) = map("x$_",(11..14));
my ($one, $r0, $r1) = ("x15","x16","x17");
my ($t0w) = $t0 =~ s/x/w/r;
my ($A0,$A1,$A2,$A3,$A4,$B0,$B1,$B2,$B3,$B4,$C0,$C1,$C2,$C3,$C4,$D0,$D1,$D2,$D3,$D4) = map("v$_",(0..19));
my ($T0,$T1,$T2,$T3) = map("v$_",(20..23));
my $CONSTS = "v24";
my $INC = "v25";
my $ROL8 = "v26";
my $CLAMP = "v27";
my ($B_STORE, $C_STORE, $D_STORE) = map("v$_",(28..30));
my $S_STORE = $CLAMP;
my $LEN_STORE = "v31";
sub chacha_qr {
my ($a,$b,$c,$d,$t,$dir)=@_;
my ($shift_b,$shift_d) = $dir =~ /left/ ? ("#4","#12") : ("#12","#4");
$code.=<<___;
add $a.4s, $a.4s, $b.4s
eor $d.16b, $d.16b, $a.16b
rev32 $d.8h, $d.8h
add $c.4s, $c.4s, $d.4s
eor $b.16b, $b.16b, $c.16b
ushr $t.4s, $b.4s, #20
sli $t.4s, $b.4s, #12
___
($t,$b) = ($b,$t);
$code.=<<___;
add $a.4s, $a.4s, $b.4s
eor $d.16b, $d.16b, $a.16b
tbl $d.16b, {$d.16b}, $ROL8.16b
add $c.4s, $c.4s, $d.4s
eor $b.16b, $b.16b, $c.16b
ushr $t.4s, $b.4s, #25
sli $t.4s, $b.4s, #7
___
($t,$b) = ($b,$t);
$code.=<<___;
ext $b.16b, $b.16b, $b.16b, $shift_b
ext $c.16b, $c.16b, $c.16b, #8
ext $d.16b, $d.16b, $d.16b, $shift_d
___
}
sub poly_add {
my ($src)=@_;
$code.="ldp $t0, $t1, [$src], 16
adds $acc0, $acc0, $t0
adcs $acc1, $acc1, $t1
adc $acc2, $acc2, $one\n";
}
sub poly_add_vec {
my ($src)=@_;
$code.="mov $t0, $src.d[0]
mov $t1, $src.d[1]
adds $acc0, $acc0, $t0
adcs $acc1, $acc1, $t1
adc $acc2, $acc2, $one\n";
}
sub poly_stage1 {
$code.="mul $t0, $acc0, $r0 // [t2:t1:t0] = [acc2:acc1:acc0] * r0
umulh $t1, $acc0, $r0
mul $t2, $acc1, $r0
umulh $t3, $acc1, $r0
adds $t1, $t1, $t2
mul $t2, $acc2, $r0
adc $t2, $t2, $t3\n";
}
sub poly_stage2 {
$code.="mul $t3, $acc0, $r1 // [t3:t2:t1:t0] = [acc2:acc1:acc0] * [r1:r0]
umulh $acc0, $acc0, $r1
adds $t1, $t1, $t3
mul $t3, $acc1, $r1
umulh $acc1, $acc1, $r1
adcs $t3, $t3, $acc0
mul $acc2, $acc2, $r1
adc $acc2, $acc2, $acc1
adds $t2, $t2, $t3
adc $t3, $acc2, xzr\n";
}
# At the beginning of the reduce stage t = [t3:t2:t1:t0] is a product of
# r = [r1:r0] and acc = [acc2:acc1:acc0]
# r is 124 bits at most (due to clamping) and acc is 131 bits at most
# (acc2 is at most 4 before the addition and can be at most 6 when we add in
# the next block) therefore t is at most 255 bits big, and t3 is 63 bits.
sub poly_reduce_stage {
$code.="and $acc2, $t2, #3 // At this point acc2 is 2 bits at most (value of 3)
and $acc0, $t2, #-4
extr $t2, $t3, $t2, #2
adds $acc0, $acc0, $t0
lsr $t0, $t3, #2
adc $acc1, $t3, $t0 // No carry out since t0 is 61 bits and t3 is 63 bits
adds $acc0, $acc0, $t2
adcs $acc1, $acc1, $t1
adc $acc2, $acc2, xzr // At this point acc2 has the value of 4 at most \n";
}
sub poly_mul {
&poly_stage1();
&poly_stage2();
&poly_reduce_stage();
}
sub chacha_qr_x3 {
my ($dir)=@_;
my ($shift_b,$shift_d) = $dir =~ /left/ ? ("#4","#12") : ("#12","#4");
$code.=<<___;
add $A0.4s, $A0.4s, $B0.4s
add $A1.4s, $A1.4s, $B1.4s
add $A2.4s, $A2.4s, $B2.4s
eor $D0.16b, $D0.16b, $A0.16b
eor $D1.16b, $D1.16b, $A1.16b
eor $D2.16b, $D2.16b, $A2.16b
rev32 $D0.8h, $D0.8h
rev32 $D1.8h, $D1.8h
rev32 $D2.8h, $D2.8h
add $C0.4s, $C0.4s, $D0.4s
add $C1.4s, $C1.4s, $D1.4s
add $C2.4s, $C2.4s, $D2.4s
eor $B0.16b, $B0.16b, $C0.16b
eor $B1.16b, $B1.16b, $C1.16b
eor $B2.16b, $B2.16b, $C2.16b
ushr $T0.4s, $B0.4s, #20
sli $T0.4s, $B0.4s, #12
ushr $B0.4s, $B1.4s, #20
sli $B0.4s, $B1.4s, #12
ushr $B1.4s, $B2.4s, #20
sli $B1.4s, $B2.4s, #12
add $A0.4s, $A0.4s, $T0.4s
add $A1.4s, $A1.4s, $B0.4s
add $A2.4s, $A2.4s, $B1.4s
eor $D0.16b, $D0.16b, $A0.16b
eor $D1.16b, $D1.16b, $A1.16b
eor $D2.16b, $D2.16b, $A2.16b
tbl $D0.16b, {$D0.16b}, $ROL8.16b
tbl $D1.16b, {$D1.16b}, $ROL8.16b
tbl $D2.16b, {$D2.16b}, $ROL8.16b
add $C0.4s, $C0.4s, $D0.4s
add $C1.4s, $C1.4s, $D1.4s
add $C2.4s, $C2.4s, $D2.4s
eor $T0.16b, $T0.16b, $C0.16b
eor $B0.16b, $B0.16b, $C1.16b
eor $B1.16b, $B1.16b, $C2.16b
ushr $B2.4s, $B1.4s, #25
sli $B2.4s, $B1.4s, #7
ushr $B1.4s, $B0.4s, #25
sli $B1.4s, $B0.4s, #7
ushr $B0.4s, $T0.4s, #25
sli $B0.4s, $T0.4s, #7
ext $B0.16b, $B0.16b, $B0.16b, $shift_b
ext $B1.16b, $B1.16b, $B1.16b, $shift_b
ext $B2.16b, $B2.16b, $B2.16b, $shift_b
ext $C0.16b, $C0.16b, $C0.16b, #8
ext $C1.16b, $C1.16b, $C1.16b, #8
ext $C2.16b, $C2.16b, $C2.16b, #8
ext $D0.16b, $D0.16b, $D0.16b, $shift_d
ext $D1.16b, $D1.16b, $D1.16b, $shift_d
ext $D2.16b, $D2.16b, $D2.16b, $shift_d
___
}
# When preparing 5 ChaCha20 blocks in parallel, we operate on 4 blocks vertically as introduced by Andrew Moon
# the fifth block is done horizontally
sub chacha_qr_x5 {
my ($dir)=@_;
my ($a0,$a1,$a2,$a3) = $dir =~ /left/ ? ($A0,$A1,$A2,$A3) : ($A0,$A1,$A2,$A3);
my ($b0,$b1,$b2,$b3) = $dir =~ /left/ ? ($B0,$B1,$B2,$B3) : ($B1,$B2,$B3,$B0);
my ($c0,$c1,$c2,$c3) = $dir =~ /left/ ? ($C0,$C1,$C2,$C3) : ($C2,$C3,$C0,$C1);
my ($d0,$d1,$d2,$d3) = $dir =~ /left/ ? ($D0,$D1,$D2,$D3) : ($D3,$D0,$D1,$D2);
my ($shift_b,$shift_d) = $dir =~ /left/ ? ("#4","#12") : ("#12","#4");
$code.=<<___;
add $a0.4s, $a0.4s, $b0.4s
add $a1.4s, $a1.4s, $b1.4s
add $a2.4s, $a2.4s, $b2.4s
add $a3.4s, $a3.4s, $b3.4s
add $A4.4s, $A4.4s, $B4.4s
eor $d0.16b, $d0.16b, $a0.16b
eor $d1.16b, $d1.16b, $a1.16b
eor $d2.16b, $d2.16b, $a2.16b
eor $d3.16b, $d3.16b, $a3.16b
eor $D4.16b, $D4.16b, $A4.16b
rev32 $d0.8h, $d0.8h
rev32 $d1.8h, $d1.8h
rev32 $d2.8h, $d2.8h
rev32 $d3.8h, $d3.8h
rev32 $D4.8h, $D4.8h
add $c0.4s, $c0.4s, $d0.4s
add $c1.4s, $c1.4s, $d1.4s
add $c2.4s, $c2.4s, $d2.4s
add $c3.4s, $c3.4s, $d3.4s
add $C4.4s, $C4.4s, $D4.4s
eor $b0.16b, $b0.16b, $c0.16b
eor $b1.16b, $b1.16b, $c1.16b
eor $b2.16b, $b2.16b, $c2.16b
eor $b3.16b, $b3.16b, $c3.16b
eor $B4.16b, $B4.16b, $C4.16b
ushr $T0.4s, $b0.4s, #20
sli $T0.4s, $b0.4s, #12
ushr $b0.4s, $b1.4s, #20
sli $b0.4s, $b1.4s, #12
ushr $b1.4s, $b2.4s, #20
sli $b1.4s, $b2.4s, #12
ushr $b2.4s, $b3.4s, #20
sli $b2.4s, $b3.4s, #12
ushr $b3.4s, $B4.4s, #20
sli $b3.4s, $B4.4s, #12
add $a0.4s, $a0.4s, $T0.4s
add $a1.4s, $a1.4s, $b0.4s
add $a2.4s, $a2.4s, $b1.4s
add $a3.4s, $a3.4s, $b2.4s
add $A4.4s, $A4.4s, $b3.4s
eor $d0.16b, $d0.16b, $a0.16b
eor $d1.16b, $d1.16b, $a1.16b
eor $d2.16b, $d2.16b, $a2.16b
eor $d3.16b, $d3.16b, $a3.16b
eor $D4.16b, $D4.16b, $A4.16b
tbl $d0.16b, {$d0.16b}, $ROL8.16b
tbl $d1.16b, {$d1.16b}, $ROL8.16b
tbl $d2.16b, {$d2.16b}, $ROL8.16b
tbl $d3.16b, {$d3.16b}, $ROL8.16b
tbl $D4.16b, {$D4.16b}, $ROL8.16b
add $c0.4s, $c0.4s, $d0.4s
add $c1.4s, $c1.4s, $d1.4s
add $c2.4s, $c2.4s, $d2.4s
add $c3.4s, $c3.4s, $d3.4s
add $C4.4s, $C4.4s, $D4.4s
eor $T0.16b, $T0.16b, $c0.16b
eor $b0.16b, $b0.16b, $c1.16b
eor $b1.16b, $b1.16b, $c2.16b
eor $b2.16b, $b2.16b, $c3.16b
eor $b3.16b, $b3.16b, $C4.16b
ushr $B4.4s, $b3.4s, #25
sli $B4.4s, $b3.4s, #7
ushr $b3.4s, $b2.4s, #25
sli $b3.4s, $b2.4s, #7
ushr $b2.4s, $b1.4s, #25
sli $b2.4s, $b1.4s, #7
ushr $b1.4s, $b0.4s, #25
sli $b1.4s, $b0.4s, #7
ushr $b0.4s, $T0.4s, #25
sli $b0.4s, $T0.4s, #7
ext $B4.16b, $B4.16b, $B4.16b, $shift_b
ext $C4.16b, $C4.16b, $C4.16b, #8
ext $D4.16b, $D4.16b, $D4.16b, $shift_d
___
}
{
$code.=<<___;
#include <openssl/arm_arch.h>
.section .rodata
.align 7
.Lchacha20_consts:
.byte 'e','x','p','a','n','d',' ','3','2','-','b','y','t','e',' ','k'
.Linc:
.long 1,2,3,4
.Lrol8:
.byte 3,0,1,2, 7,4,5,6, 11,8,9,10, 15,12,13,14
.Lclamp:
.quad 0x0FFFFFFC0FFFFFFF, 0x0FFFFFFC0FFFFFFC
.text
.type .Lpoly_hash_ad_internal,%function
.align 6
.Lpoly_hash_ad_internal:
.cfi_startproc
cbnz $adl, .Lpoly_hash_intro
ret
.Lpoly_hash_intro:
cmp $adl, #16
b.lt .Lpoly_hash_ad_tail
___
&poly_add($adp);
&poly_mul();
$code.=<<___;
sub $adl, $adl, #16
b .Lpoly_hash_ad_internal
.Lpoly_hash_ad_tail:
cbz $adl, .Lpoly_hash_ad_ret
eor $T0.16b, $T0.16b, $T0.16b // Use T0 to load the AAD
sub $adl, $adl, #1
.Lpoly_hash_tail_16_compose:
ext $T0.16b, $T0.16b, $T0.16b, #15
ldrb $t0w, [$adp, $adl]
mov $T0.b[0], $t0w
subs $adl, $adl, #1
b.ge .Lpoly_hash_tail_16_compose
___
&poly_add_vec($T0);
&poly_mul();
$code.=<<___;
.Lpoly_hash_ad_ret:
ret
.cfi_endproc
.size .Lpoly_hash_ad_internal, .-.Lpoly_hash_ad_internal
/////////////////////////////////
//
// void chacha20_poly1305_seal(uint8_t *pt, uint8_t *ct, size_t len_in, uint8_t *ad, size_t len_ad, union open_data *seal_data);
//
.globl chacha20_poly1305_seal
.type chacha20_poly1305_seal,%function
.align 6
chacha20_poly1305_seal:
AARCH64_SIGN_LINK_REGISTER
.cfi_startproc
stp x29, x30, [sp, #-80]!
.cfi_def_cfa_offset 80
.cfi_offset w30, -72
.cfi_offset w29, -80
mov x29, sp
// We probably could do .cfi_def_cfa w29, 80 at this point, but since
// we don't actually use the frame pointer like that, it's probably not
// worth bothering.
stp d8, d9, [sp, #16]
stp d10, d11, [sp, #32]
stp d12, d13, [sp, #48]
stp d14, d15, [sp, #64]
.cfi_offset b15, -8
.cfi_offset b14, -16
.cfi_offset b13, -24
.cfi_offset b12, -32
.cfi_offset b11, -40
.cfi_offset b10, -48
.cfi_offset b9, -56
.cfi_offset b8, -64
adrp $t0, :pg_hi21:.Lchacha20_consts
add $t0, $t0, :lo12:.Lchacha20_consts
ld1 {$CONSTS.16b - $CLAMP.16b}, [$t0] // Load the CONSTS, INC, ROL8 and CLAMP values
ld1 {$B_STORE.16b - $D_STORE.16b}, [$keyp]
mov $one, #1 // Prepare the Poly1305 state
mov $acc0, #0
mov $acc1, #0
mov $acc2, #0
ldr $t1, [$keyp, #56] // The total cipher text length includes extra_in_len
add $t1, $t1, $inl
mov $LEN_STORE.d[0], $adl // Store the input and aad lengths
mov $LEN_STORE.d[1], $t1
cmp $inl, #128
b.le .Lseal_128 // Optimization for smaller buffers
// Initially we prepare 5 ChaCha20 blocks. Four to encrypt up to 4 blocks (256 bytes) of plaintext,
// and one for the Poly1305 R and S keys. The first four blocks (A0-A3..D0-D3) are computed vertically,
// the fifth block (A4-D4) horizontally.
ld4r {$A0.4s-$A3.4s}, [$t0]
mov $A4.16b, $CONSTS.16b
ld4r {$B0.4s-$B3.4s}, [$keyp], #16
mov $B4.16b, $B_STORE.16b
ld4r {$C0.4s-$C3.4s}, [$keyp], #16
mov $C4.16b, $C_STORE.16b
ld4r {$D0.4s-$D3.4s}, [$keyp]
add $D0.4s, $D0.4s, $INC.4s
mov $D4.16b, $D_STORE.16b
sub $keyp, $keyp, #32
mov $itr1, #10
.align 5
.Lseal_init_rounds:
___
&chacha_qr_x5("left");
&chacha_qr_x5("right");
$code.=<<___;
subs $itr1, $itr1, #1
b.hi .Lseal_init_rounds
add $D0.4s, $D0.4s, $INC.4s
mov $t0, #4
dup $T0.4s, $t0w
add $INC.4s, $INC.4s, $T0.4s
zip1 $T0.4s, $A0.4s, $A1.4s
zip2 $T1.4s, $A0.4s, $A1.4s
zip1 $T2.4s, $A2.4s, $A3.4s
zip2 $T3.4s, $A2.4s, $A3.4s
zip1 $A0.2d, $T0.2d, $T2.2d
zip2 $A1.2d, $T0.2d, $T2.2d
zip1 $A2.2d, $T1.2d, $T3.2d
zip2 $A3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $B0.4s, $B1.4s
zip2 $T1.4s, $B0.4s, $B1.4s
zip1 $T2.4s, $B2.4s, $B3.4s
zip2 $T3.4s, $B2.4s, $B3.4s
zip1 $B0.2d, $T0.2d, $T2.2d
zip2 $B1.2d, $T0.2d, $T2.2d
zip1 $B2.2d, $T1.2d, $T3.2d
zip2 $B3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $C0.4s, $C1.4s
zip2 $T1.4s, $C0.4s, $C1.4s
zip1 $T2.4s, $C2.4s, $C3.4s
zip2 $T3.4s, $C2.4s, $C3.4s
zip1 $C0.2d, $T0.2d, $T2.2d
zip2 $C1.2d, $T0.2d, $T2.2d
zip1 $C2.2d, $T1.2d, $T3.2d
zip2 $C3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $D0.4s, $D1.4s
zip2 $T1.4s, $D0.4s, $D1.4s
zip1 $T2.4s, $D2.4s, $D3.4s
zip2 $T3.4s, $D2.4s, $D3.4s
zip1 $D0.2d, $T0.2d, $T2.2d
zip2 $D1.2d, $T0.2d, $T2.2d
zip1 $D2.2d, $T1.2d, $T3.2d
zip2 $D3.2d, $T1.2d, $T3.2d
add $A4.4s, $A4.4s, $CONSTS.4s
add $B4.4s, $B4.4s, $B_STORE.4s
and $A4.16b, $A4.16b, $CLAMP.16b
add $A0.4s, $A0.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
add $C0.4s, $C0.4s, $C_STORE.4s
add $D0.4s, $D0.4s, $D_STORE.4s
add $A1.4s, $A1.4s, $CONSTS.4s
add $B1.4s, $B1.4s, $B_STORE.4s
add $C1.4s, $C1.4s, $C_STORE.4s
add $D1.4s, $D1.4s, $D_STORE.4s
add $A2.4s, $A2.4s, $CONSTS.4s
add $B2.4s, $B2.4s, $B_STORE.4s
add $C2.4s, $C2.4s, $C_STORE.4s
add $D2.4s, $D2.4s, $D_STORE.4s
add $A3.4s, $A3.4s, $CONSTS.4s
add $B3.4s, $B3.4s, $B_STORE.4s
add $C3.4s, $C3.4s, $C_STORE.4s
add $D3.4s, $D3.4s, $D_STORE.4s
mov $r0, $A4.d[0] // Move the R key to GPRs
mov $r1, $A4.d[1]
mov $S_STORE.16b, $B4.16b // Store the S key
bl .Lpoly_hash_ad_internal
mov $adp, $oup
cmp $inl, #256
b.le .Lseal_tail
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A0.16b
eor $T1.16b, $T1.16b, $B0.16b
eor $T2.16b, $T2.16b, $C0.16b
eor $T3.16b, $T3.16b, $D0.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A1.16b
eor $T1.16b, $T1.16b, $B1.16b
eor $T2.16b, $T2.16b, $C1.16b
eor $T3.16b, $T3.16b, $D1.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A2.16b
eor $T1.16b, $T1.16b, $B2.16b
eor $T2.16b, $T2.16b, $C2.16b
eor $T3.16b, $T3.16b, $D2.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A3.16b
eor $T1.16b, $T1.16b, $B3.16b
eor $T2.16b, $T2.16b, $C3.16b
eor $T3.16b, $T3.16b, $D3.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #256
mov $itr1, #4 // In the first run of the loop we need to hash 256 bytes, therefore we hash one block for the first 4 rounds
mov $itr2, #6 // and two blocks for the remaining 6, for a total of (1 * 4 + 2 * 6) * 16 = 256
.Lseal_main_loop:
adrp $t0, :pg_hi21:.Lchacha20_consts
add $t0, $t0, :lo12:.Lchacha20_consts
ld4r {$A0.4s-$A3.4s}, [$t0]
mov $A4.16b, $CONSTS.16b
ld4r {$B0.4s-$B3.4s}, [$keyp], #16
mov $B4.16b, $B_STORE.16b
ld4r {$C0.4s-$C3.4s}, [$keyp], #16
mov $C4.16b, $C_STORE.16b
ld4r {$D0.4s-$D3.4s}, [$keyp]
add $D0.4s, $D0.4s, $INC.4s
mov $D4.16b, $D_STORE.16b
eor $T0.16b, $T0.16b, $T0.16b //zero
not $T1.16b, $T0.16b // -1
sub $T1.4s, $INC.4s, $T1.4s // Add +1
ext $T0.16b, $T1.16b, $T0.16b, #12 // Get the last element (counter)
add $D4.4s, $D4.4s, $T0.4s
sub $keyp, $keyp, #32
.align 5
.Lseal_main_loop_rounds:
___
&chacha_qr_x5("left");
&poly_add($adp);
&poly_mul();
&chacha_qr_x5("right");
$code.=<<___;
subs $itr1, $itr1, #1
b.ge .Lseal_main_loop_rounds
___
&poly_add($adp);
&poly_mul();
$code.=<<___;
subs $itr2, $itr2, #1
b.gt .Lseal_main_loop_rounds
eor $T0.16b, $T0.16b, $T0.16b //zero
not $T1.16b, $T0.16b // -1
sub $T1.4s, $INC.4s, $T1.4s // Add +1
ext $T0.16b, $T1.16b, $T0.16b, #12 // Get the last element (counter)
add $D4.4s, $D4.4s, $T0.4s
add $D0.4s, $D0.4s, $INC.4s
mov $t0, #5
dup $T0.4s, $t0w
add $INC.4s, $INC.4s, $T0.4s
zip1 $T0.4s, $A0.4s, $A1.4s
zip2 $T1.4s, $A0.4s, $A1.4s
zip1 $T2.4s, $A2.4s, $A3.4s
zip2 $T3.4s, $A2.4s, $A3.4s
zip1 $A0.2d, $T0.2d, $T2.2d
zip2 $A1.2d, $T0.2d, $T2.2d
zip1 $A2.2d, $T1.2d, $T3.2d
zip2 $A3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $B0.4s, $B1.4s
zip2 $T1.4s, $B0.4s, $B1.4s
zip1 $T2.4s, $B2.4s, $B3.4s
zip2 $T3.4s, $B2.4s, $B3.4s
zip1 $B0.2d, $T0.2d, $T2.2d
zip2 $B1.2d, $T0.2d, $T2.2d
zip1 $B2.2d, $T1.2d, $T3.2d
zip2 $B3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $C0.4s, $C1.4s
zip2 $T1.4s, $C0.4s, $C1.4s
zip1 $T2.4s, $C2.4s, $C3.4s
zip2 $T3.4s, $C2.4s, $C3.4s
zip1 $C0.2d, $T0.2d, $T2.2d
zip2 $C1.2d, $T0.2d, $T2.2d
zip1 $C2.2d, $T1.2d, $T3.2d
zip2 $C3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $D0.4s, $D1.4s
zip2 $T1.4s, $D0.4s, $D1.4s
zip1 $T2.4s, $D2.4s, $D3.4s
zip2 $T3.4s, $D2.4s, $D3.4s
zip1 $D0.2d, $T0.2d, $T2.2d
zip2 $D1.2d, $T0.2d, $T2.2d
zip1 $D2.2d, $T1.2d, $T3.2d
zip2 $D3.2d, $T1.2d, $T3.2d
add $A0.4s, $A0.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
add $C0.4s, $C0.4s, $C_STORE.4s
add $D0.4s, $D0.4s, $D_STORE.4s
add $A1.4s, $A1.4s, $CONSTS.4s
add $B1.4s, $B1.4s, $B_STORE.4s
add $C1.4s, $C1.4s, $C_STORE.4s
add $D1.4s, $D1.4s, $D_STORE.4s
add $A2.4s, $A2.4s, $CONSTS.4s
add $B2.4s, $B2.4s, $B_STORE.4s
add $C2.4s, $C2.4s, $C_STORE.4s
add $D2.4s, $D2.4s, $D_STORE.4s
add $A3.4s, $A3.4s, $CONSTS.4s
add $B3.4s, $B3.4s, $B_STORE.4s
add $C3.4s, $C3.4s, $C_STORE.4s
add $D3.4s, $D3.4s, $D_STORE.4s
add $A4.4s, $A4.4s, $CONSTS.4s
add $B4.4s, $B4.4s, $B_STORE.4s
add $C4.4s, $C4.4s, $C_STORE.4s
add $D4.4s, $D4.4s, $D_STORE.4s
cmp $inl, #320
b.le .Lseal_tail
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A0.16b
eor $T1.16b, $T1.16b, $B0.16b
eor $T2.16b, $T2.16b, $C0.16b
eor $T3.16b, $T3.16b, $D0.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A1.16b
eor $T1.16b, $T1.16b, $B1.16b
eor $T2.16b, $T2.16b, $C1.16b
eor $T3.16b, $T3.16b, $D1.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A2.16b
eor $T1.16b, $T1.16b, $B2.16b
eor $T2.16b, $T2.16b, $C2.16b
eor $T3.16b, $T3.16b, $D2.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A3.16b
eor $T1.16b, $T1.16b, $B3.16b
eor $T2.16b, $T2.16b, $C3.16b
eor $T3.16b, $T3.16b, $D3.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A4.16b
eor $T1.16b, $T1.16b, $B4.16b
eor $T2.16b, $T2.16b, $C4.16b
eor $T3.16b, $T3.16b, $D4.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #320
mov $itr1, #0
mov $itr2, #10 // For the remainder of the loop we always hash and encrypt 320 bytes per iteration
b .Lseal_main_loop
.Lseal_tail:
// This part of the function handles the storage and authentication of the last [0,320) bytes
// We assume A0-A4 ... D0-D4 hold at least inl (320 max) bytes of the stream data.
cmp $inl, #64
b.lt .Lseal_tail_64
// Store and authenticate 64B blocks per iteration
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A0.16b
eor $T1.16b, $T1.16b, $B0.16b
eor $T2.16b, $T2.16b, $C0.16b
eor $T3.16b, $T3.16b, $D0.16b
___
&poly_add_vec($T0);
&poly_mul();
&poly_add_vec($T1);
&poly_mul();
&poly_add_vec($T2);
&poly_mul();
&poly_add_vec($T3);
&poly_mul();
$code.=<<___;
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #64
// Shift the state left by 64 bytes for the next iteration of the loop
mov $A0.16b, $A1.16b
mov $B0.16b, $B1.16b
mov $C0.16b, $C1.16b
mov $D0.16b, $D1.16b
mov $A1.16b, $A2.16b
mov $B1.16b, $B2.16b
mov $C1.16b, $C2.16b
mov $D1.16b, $D2.16b
mov $A2.16b, $A3.16b
mov $B2.16b, $B3.16b
mov $C2.16b, $C3.16b
mov $D2.16b, $D3.16b
mov $A3.16b, $A4.16b
mov $B3.16b, $B4.16b
mov $C3.16b, $C4.16b
mov $D3.16b, $D4.16b
b .Lseal_tail
.Lseal_tail_64:
ldp $adp, $adl, [$keyp, #48] // extra_in_len and extra_in_ptr
// Here we handle the last [0,64) bytes of plaintext
cmp $inl, #16
b.lt .Lseal_tail_16
// Each iteration encrypt and authenticate a 16B block
ld1 {$T0.16b}, [$inp], #16
eor $T0.16b, $T0.16b, $A0.16b
___
&poly_add_vec($T0);
&poly_mul();
$code.=<<___;
st1 {$T0.16b}, [$oup], #16
sub $inl, $inl, #16
// Shift the state left by 16 bytes for the next iteration of the loop
mov $A0.16b, $B0.16b
mov $B0.16b, $C0.16b
mov $C0.16b, $D0.16b
b .Lseal_tail_64
.Lseal_tail_16:
// Here we handle the last [0,16) bytes of ciphertext that require a padded block
cbz $inl, .Lseal_hash_extra
eor $T0.16b, $T0.16b, $T0.16b // Use T0 to load the plaintext/extra in
eor $T1.16b, $T1.16b, $T1.16b // Use T1 to generate an AND mask that will only mask the ciphertext bytes
not $T2.16b, $T0.16b
mov $itr1, $inl
add $inp, $inp, $inl
cbz $adl, .Lseal_tail_16_compose // No extra data to pad with, zero padding
mov $itr2, #16 // We need to load some extra_in first for padding
sub $itr2, $itr2, $inl
cmp $adl, $itr2
csel $itr2, $adl, $itr2, lt // Load the minimum of extra_in_len and the amount needed to fill the register
mov $t1, $itr2
add $adp, $adp, $itr2
sub $adl, $adl, $itr2
.Lseal_tail16_compose_extra_in:
ext $T0.16b, $T0.16b, $T0.16b, #15
ldrb $t0w, [$adp, #-1]!
mov $T0.b[0], $t0w
subs $itr2, $itr2, #1
b.gt .Lseal_tail16_compose_extra_in
add $adp, $adp, $t1
.Lseal_tail_16_compose:
ext $T0.16b, $T0.16b, $T0.16b, #15
ldrb $t0w, [$inp, #-1]!
mov $T0.b[0], $t0w
ext $T1.16b, $T2.16b, $T1.16b, #15
subs $inl, $inl, #1
b.gt .Lseal_tail_16_compose
and $A0.16b, $A0.16b, $T1.16b
eor $T0.16b, $T0.16b, $A0.16b
mov $T1.16b, $T0.16b
.Lseal_tail_16_store:
umov $t0w, $T0.b[0]
strb $t0w, [$oup], #1
ext $T0.16b, $T0.16b, $T0.16b, #1
subs $itr1, $itr1, #1
b.gt .Lseal_tail_16_store
// Hash in the final ct block concatenated with extra_in
___
&poly_add_vec($T1);
&poly_mul();
$code.=<<___;
.Lseal_hash_extra:
cbz $adl, .Lseal_finalize
.Lseal_hash_extra_loop:
cmp $adl, #16
b.lt .Lseal_hash_extra_tail
ld1 {$T0.16b}, [$adp], #16
___
&poly_add_vec($T0);
&poly_mul();
$code.=<<___;
sub $adl, $adl, #16
b .Lseal_hash_extra_loop
.Lseal_hash_extra_tail:
cbz $adl, .Lseal_finalize
eor $T0.16b, $T0.16b, $T0.16b // Use T0 to load the remaining extra ciphertext
add $adp, $adp, $adl
.Lseal_hash_extra_load:
ext $T0.16b, $T0.16b, $T0.16b, #15
ldrb $t0w, [$adp, #-1]!
mov $T0.b[0], $t0w
subs $adl, $adl, #1
b.gt .Lseal_hash_extra_load
// Hash in the final padded extra_in blcok
___
&poly_add_vec($T0);
&poly_mul();
$code.=<<___;
.Lseal_finalize:
___
&poly_add_vec($LEN_STORE);
&poly_mul();
$code.=<<___;
// Final reduction step
sub $t1, xzr, $one
orr $t2, xzr, #3
subs $t0, $acc0, #-5
sbcs $t1, $acc1, $t1
sbcs $t2, $acc2, $t2
csel $acc0, $t0, $acc0, cs
csel $acc1, $t1, $acc1, cs
csel $acc2, $t2, $acc2, cs
___
&poly_add_vec($S_STORE);
$code.=<<___;
stp $acc0, $acc1, [$keyp]
ldp d8, d9, [sp, #16]
ldp d10, d11, [sp, #32]
ldp d12, d13, [sp, #48]
ldp d14, d15, [sp, #64]
.cfi_restore b15
.cfi_restore b14
.cfi_restore b13
.cfi_restore b12
.cfi_restore b11
.cfi_restore b10
.cfi_restore b9
.cfi_restore b8
ldp x29, x30, [sp], 80
.cfi_restore w29
.cfi_restore w30
.cfi_def_cfa_offset 0
AARCH64_VALIDATE_LINK_REGISTER
ret
.Lseal_128:
// On some architectures preparing 5 blocks for small buffers is wasteful
eor $INC.16b, $INC.16b, $INC.16b
mov $t0, #1
mov $INC.s[0], $t0w
mov $A0.16b, $CONSTS.16b
mov $A1.16b, $CONSTS.16b
mov $A2.16b, $CONSTS.16b
mov $B0.16b, $B_STORE.16b
mov $B1.16b, $B_STORE.16b
mov $B2.16b, $B_STORE.16b
mov $C0.16b, $C_STORE.16b
mov $C1.16b, $C_STORE.16b
mov $C2.16b, $C_STORE.16b
mov $D2.16b, $D_STORE.16b
add $D0.4s, $D2.4s, $INC.4s
add $D1.4s, $D0.4s, $INC.4s
mov $itr1, #10
.Lseal_128_rounds:
___
&chacha_qr_x3("left");
&chacha_qr_x3("right");
$code.=<<___;
subs $itr1, $itr1, #1
b.hi .Lseal_128_rounds
add $A0.4s, $A0.4s, $CONSTS.4s
add $A1.4s, $A1.4s, $CONSTS.4s
add $A2.4s, $A2.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
add $B1.4s, $B1.4s, $B_STORE.4s
add $B2.4s, $B2.4s, $B_STORE.4s
// Only the first 32 bytes of the third block (counter = 0) are needed,
// so skip updating $C2 and $D2.
add $C0.4s, $C0.4s, $C_STORE.4s
add $C1.4s, $C1.4s, $C_STORE.4s
add $D_STORE.4s, $D_STORE.4s, $INC.4s
add $D0.4s, $D0.4s, $D_STORE.4s
add $D_STORE.4s, $D_STORE.4s, $INC.4s
add $D1.4s, $D1.4s, $D_STORE.4s
and $A2.16b, $A2.16b, $CLAMP.16b
mov $r0, $A2.d[0] // Move the R key to GPRs
mov $r1, $A2.d[1]
mov $S_STORE.16b, $B2.16b // Store the S key
bl .Lpoly_hash_ad_internal
b .Lseal_tail
.cfi_endproc
.size chacha20_poly1305_seal,.-chacha20_poly1305_seal
/////////////////////////////////
//
// void chacha20_poly1305_open(uint8_t *pt, uint8_t *ct, size_t len_in, uint8_t *ad, size_t len_ad, union open_data *aead_data);
//
.globl chacha20_poly1305_open
.type chacha20_poly1305_open,%function
.align 6
chacha20_poly1305_open:
AARCH64_SIGN_LINK_REGISTER
.cfi_startproc
stp x29, x30, [sp, #-80]!
.cfi_def_cfa_offset 80
.cfi_offset w30, -72
.cfi_offset w29, -80
mov x29, sp
// We probably could do .cfi_def_cfa w29, 80 at this point, but since
// we don't actually use the frame pointer like that, it's probably not
// worth bothering.
stp d8, d9, [sp, #16]
stp d10, d11, [sp, #32]
stp d12, d13, [sp, #48]
stp d14, d15, [sp, #64]
.cfi_offset b15, -8
.cfi_offset b14, -16
.cfi_offset b13, -24
.cfi_offset b12, -32
.cfi_offset b11, -40
.cfi_offset b10, -48
.cfi_offset b9, -56
.cfi_offset b8, -64
adrp $t0, :pg_hi21:.Lchacha20_consts
add $t0, $t0, :lo12:.Lchacha20_consts
ld1 {$CONSTS.16b - $CLAMP.16b}, [$t0] // Load the CONSTS, INC, ROL8 and CLAMP values
ld1 {$B_STORE.16b - $D_STORE.16b}, [$keyp]
mov $one, #1 // Prepare the Poly1305 state
mov $acc0, #0
mov $acc1, #0
mov $acc2, #0
mov $LEN_STORE.d[0], $adl // Store the input and aad lengths
mov $LEN_STORE.d[1], $inl
cmp $inl, #128
b.le .Lopen_128 // Optimization for smaller buffers
// Initially we prepare a single ChaCha20 block for the Poly1305 R and S keys
mov $A0.16b, $CONSTS.16b
mov $B0.16b, $B_STORE.16b
mov $C0.16b, $C_STORE.16b
mov $D0.16b, $D_STORE.16b
mov $itr1, #10
.align 5
.Lopen_init_rounds:
___
&chacha_qr($A0, $B0, $C0, $D0, $T0, "left");
&chacha_qr($A0, $B0, $C0, $D0, $T0, "right");
$code.=<<___;
subs $itr1, $itr1, #1
b.hi .Lopen_init_rounds
add $A0.4s, $A0.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
and $A0.16b, $A0.16b, $CLAMP.16b
mov $r0, $A0.d[0] // Move the R key to GPRs
mov $r1, $A0.d[1]
mov $S_STORE.16b, $B0.16b // Store the S key
bl .Lpoly_hash_ad_internal
.Lopen_ad_done:
mov $adp, $inp
// Each iteration of the loop hash 320 bytes, and prepare stream for 320 bytes
.Lopen_main_loop:
cmp $inl, #192
b.lt .Lopen_tail
adrp $t0, :pg_hi21:.Lchacha20_consts
add $t0, $t0, :lo12:.Lchacha20_consts
ld4r {$A0.4s-$A3.4s}, [$t0]
mov $A4.16b, $CONSTS.16b
ld4r {$B0.4s-$B3.4s}, [$keyp], #16
mov $B4.16b, $B_STORE.16b
ld4r {$C0.4s-$C3.4s}, [$keyp], #16
mov $C4.16b, $C_STORE.16b
ld4r {$D0.4s-$D3.4s}, [$keyp]
sub $keyp, $keyp, #32
add $D0.4s, $D0.4s, $INC.4s
mov $D4.16b, $D_STORE.16b
eor $T0.16b, $T0.16b, $T0.16b //zero
not $T1.16b, $T0.16b // -1
sub $T1.4s, $INC.4s, $T1.4s // Add +1
ext $T0.16b, $T1.16b, $T0.16b, #12 // Get the last element (counter)
add $D4.4s, $D4.4s, $T0.4s
lsr $adl, $inl, #4 // How many whole blocks we have to hash, will always be at least 12
sub $adl, $adl, #10
mov $itr2, #10
subs $itr1, $itr2, $adl
subs $itr1, $itr2, $adl // itr1 can be negative if we have more than 320 bytes to hash
csel $itr2, $itr2, $adl, le // if itr1 is zero or less, itr2 should be 10 to indicate all 10 rounds are full
cbz $itr2, .Lopen_main_loop_rounds_short
.align 5
.Lopen_main_loop_rounds:
___
&poly_add($adp);
&poly_mul();
$code.=<<___;
.Lopen_main_loop_rounds_short:
___
&chacha_qr_x5("left");
&poly_add($adp);
&poly_mul();
&chacha_qr_x5("right");
$code.=<<___;
subs $itr2, $itr2, #1
b.gt .Lopen_main_loop_rounds
subs $itr1, $itr1, #1
b.ge .Lopen_main_loop_rounds_short
___
$code.=<<___;
eor $T0.16b, $T0.16b, $T0.16b //zero
not $T1.16b, $T0.16b // -1
sub $T1.4s, $INC.4s, $T1.4s // Add +1
ext $T0.16b, $T1.16b, $T0.16b, #12 // Get the last element (counter)
add $D4.4s, $D4.4s, $T0.4s
add $D0.4s, $D0.4s, $INC.4s
mov $t0, #5
dup $T0.4s, $t0w
add $INC.4s, $INC.4s, $T0.4s
zip1 $T0.4s, $A0.4s, $A1.4s
zip2 $T1.4s, $A0.4s, $A1.4s
zip1 $T2.4s, $A2.4s, $A3.4s
zip2 $T3.4s, $A2.4s, $A3.4s
zip1 $A0.2d, $T0.2d, $T2.2d
zip2 $A1.2d, $T0.2d, $T2.2d
zip1 $A2.2d, $T1.2d, $T3.2d
zip2 $A3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $B0.4s, $B1.4s
zip2 $T1.4s, $B0.4s, $B1.4s
zip1 $T2.4s, $B2.4s, $B3.4s
zip2 $T3.4s, $B2.4s, $B3.4s
zip1 $B0.2d, $T0.2d, $T2.2d
zip2 $B1.2d, $T0.2d, $T2.2d
zip1 $B2.2d, $T1.2d, $T3.2d
zip2 $B3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $C0.4s, $C1.4s
zip2 $T1.4s, $C0.4s, $C1.4s
zip1 $T2.4s, $C2.4s, $C3.4s
zip2 $T3.4s, $C2.4s, $C3.4s
zip1 $C0.2d, $T0.2d, $T2.2d
zip2 $C1.2d, $T0.2d, $T2.2d
zip1 $C2.2d, $T1.2d, $T3.2d
zip2 $C3.2d, $T1.2d, $T3.2d
zip1 $T0.4s, $D0.4s, $D1.4s
zip2 $T1.4s, $D0.4s, $D1.4s
zip1 $T2.4s, $D2.4s, $D3.4s
zip2 $T3.4s, $D2.4s, $D3.4s
zip1 $D0.2d, $T0.2d, $T2.2d
zip2 $D1.2d, $T0.2d, $T2.2d
zip1 $D2.2d, $T1.2d, $T3.2d
zip2 $D3.2d, $T1.2d, $T3.2d
add $A0.4s, $A0.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
add $C0.4s, $C0.4s, $C_STORE.4s
add $D0.4s, $D0.4s, $D_STORE.4s
add $A1.4s, $A1.4s, $CONSTS.4s
add $B1.4s, $B1.4s, $B_STORE.4s
add $C1.4s, $C1.4s, $C_STORE.4s
add $D1.4s, $D1.4s, $D_STORE.4s
add $A2.4s, $A2.4s, $CONSTS.4s
add $B2.4s, $B2.4s, $B_STORE.4s
add $C2.4s, $C2.4s, $C_STORE.4s
add $D2.4s, $D2.4s, $D_STORE.4s
add $A3.4s, $A3.4s, $CONSTS.4s
add $B3.4s, $B3.4s, $B_STORE.4s
add $C3.4s, $C3.4s, $C_STORE.4s
add $D3.4s, $D3.4s, $D_STORE.4s
add $A4.4s, $A4.4s, $CONSTS.4s
add $B4.4s, $B4.4s, $B_STORE.4s
add $C4.4s, $C4.4s, $C_STORE.4s
add $D4.4s, $D4.4s, $D_STORE.4s
// We can always safely store 192 bytes
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A0.16b
eor $T1.16b, $T1.16b, $B0.16b
eor $T2.16b, $T2.16b, $C0.16b
eor $T3.16b, $T3.16b, $D0.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A1.16b
eor $T1.16b, $T1.16b, $B1.16b
eor $T2.16b, $T2.16b, $C1.16b
eor $T3.16b, $T3.16b, $D1.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A2.16b
eor $T1.16b, $T1.16b, $B2.16b
eor $T2.16b, $T2.16b, $C2.16b
eor $T3.16b, $T3.16b, $D2.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #192
mov $A0.16b, $A3.16b
mov $B0.16b, $B3.16b
mov $C0.16b, $C3.16b
mov $D0.16b, $D3.16b
cmp $inl, #64
b.lt .Lopen_tail_64_store
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A3.16b
eor $T1.16b, $T1.16b, $B3.16b
eor $T2.16b, $T2.16b, $C3.16b
eor $T3.16b, $T3.16b, $D3.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #64
mov $A0.16b, $A4.16b
mov $B0.16b, $B4.16b
mov $C0.16b, $C4.16b
mov $D0.16b, $D4.16b
cmp $inl, #64
b.lt .Lopen_tail_64_store
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A4.16b
eor $T1.16b, $T1.16b, $B4.16b
eor $T2.16b, $T2.16b, $C4.16b
eor $T3.16b, $T3.16b, $D4.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #64
b .Lopen_main_loop
.Lopen_tail:
cbz $inl, .Lopen_finalize
lsr $adl, $inl, #4 // How many whole blocks we have to hash
cmp $inl, #64
b.le .Lopen_tail_64
cmp $inl, #128
b.le .Lopen_tail_128
.Lopen_tail_192:
// We need three more blocks
mov $A0.16b, $CONSTS.16b
mov $A1.16b, $CONSTS.16b
mov $A2.16b, $CONSTS.16b
mov $B0.16b, $B_STORE.16b
mov $B1.16b, $B_STORE.16b
mov $B2.16b, $B_STORE.16b
mov $C0.16b, $C_STORE.16b
mov $C1.16b, $C_STORE.16b
mov $C2.16b, $C_STORE.16b
mov $D0.16b, $D_STORE.16b
mov $D1.16b, $D_STORE.16b
mov $D2.16b, $D_STORE.16b
eor $T3.16b, $T3.16b, $T3.16b
eor $T1.16b, $T1.16b, $T1.16b
ins $T3.s[0], $INC.s[0]
ins $T1.d[0], $one
add $T2.4s, $T3.4s, $T1.4s
add $T1.4s, $T2.4s, $T1.4s
add $D0.4s, $D0.4s, $T1.4s
add $D1.4s, $D1.4s, $T3.4s
add $D2.4s, $D2.4s, $T2.4s
mov $itr2, #10
subs $itr1, $itr2, $adl // itr1 can be negative if we have more than 160 bytes to hash
csel $itr2, $itr2, $adl, le // if itr1 is zero or less, itr2 should be 10 to indicate all 10 rounds are hashing
sub $adl, $adl, $itr2
cbz $itr2, .Lopen_tail_192_rounds_no_hash
.Lopen_tail_192_rounds:
___
&poly_add($adp);
&poly_mul();
$code.=<<___;
.Lopen_tail_192_rounds_no_hash:
___
&chacha_qr_x3("left");
&chacha_qr_x3("right");
$code.=<<___;
subs $itr2, $itr2, #1
b.gt .Lopen_tail_192_rounds
subs $itr1, $itr1, #1
b.ge .Lopen_tail_192_rounds_no_hash
// We hashed 160 bytes at most, may still have 32 bytes left
.Lopen_tail_192_hash:
cbz $adl, .Lopen_tail_192_hash_done
___
&poly_add($adp);
&poly_mul();
$code.=<<___;
sub $adl, $adl, #1
b .Lopen_tail_192_hash
.Lopen_tail_192_hash_done:
add $A0.4s, $A0.4s, $CONSTS.4s
add $A1.4s, $A1.4s, $CONSTS.4s
add $A2.4s, $A2.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
add $B1.4s, $B1.4s, $B_STORE.4s
add $B2.4s, $B2.4s, $B_STORE.4s
add $C0.4s, $C0.4s, $C_STORE.4s
add $C1.4s, $C1.4s, $C_STORE.4s
add $C2.4s, $C2.4s, $C_STORE.4s
add $D0.4s, $D0.4s, $D_STORE.4s
add $D1.4s, $D1.4s, $D_STORE.4s
add $D2.4s, $D2.4s, $D_STORE.4s
add $D0.4s, $D0.4s, $T1.4s
add $D1.4s, $D1.4s, $T3.4s
add $D2.4s, $D2.4s, $T2.4s
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A1.16b
eor $T1.16b, $T1.16b, $B1.16b
eor $T2.16b, $T2.16b, $C1.16b
eor $T3.16b, $T3.16b, $D1.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A2.16b
eor $T1.16b, $T1.16b, $B2.16b
eor $T2.16b, $T2.16b, $C2.16b
eor $T3.16b, $T3.16b, $D2.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #128
b .Lopen_tail_64_store
.Lopen_tail_128:
// We need two more blocks
mov $A0.16b, $CONSTS.16b
mov $A1.16b, $CONSTS.16b
mov $B0.16b, $B_STORE.16b
mov $B1.16b, $B_STORE.16b
mov $C0.16b, $C_STORE.16b
mov $C1.16b, $C_STORE.16b
mov $D0.16b, $D_STORE.16b
mov $D1.16b, $D_STORE.16b
eor $T3.16b, $T3.16b, $T3.16b
eor $T2.16b, $T2.16b, $T2.16b
ins $T3.s[0], $INC.s[0]
ins $T2.d[0], $one
add $T2.4s, $T2.4s, $T3.4s
add $D0.4s, $D0.4s, $T2.4s
add $D1.4s, $D1.4s, $T3.4s
mov $itr1, #10
sub $itr1, $itr1, $adl
.Lopen_tail_128_rounds:
___
&chacha_qr($A0, $B0, $C0, $D0, $T0, "left");
&chacha_qr($A1, $B1, $C1, $D1, $T0, "left");
&chacha_qr($A0, $B0, $C0, $D0, $T0, "right");
&chacha_qr($A1, $B1, $C1, $D1, $T0, "right");
$code.=<<___;
subs $itr1, $itr1, #1
b.gt .Lopen_tail_128_rounds
cbz $adl, .Lopen_tail_128_rounds_done
subs $adl, $adl, #1
___
&poly_add($adp);
&poly_mul();
$code.=<<___;
b .Lopen_tail_128_rounds
.Lopen_tail_128_rounds_done:
add $A0.4s, $A0.4s, $CONSTS.4s
add $A1.4s, $A1.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
add $B1.4s, $B1.4s, $B_STORE.4s
add $C0.4s, $C0.4s, $C_STORE.4s
add $C1.4s, $C1.4s, $C_STORE.4s
add $D0.4s, $D0.4s, $D_STORE.4s
add $D1.4s, $D1.4s, $D_STORE.4s
add $D0.4s, $D0.4s, $T2.4s
add $D1.4s, $D1.4s, $T3.4s
ld1 {$T0.16b - $T3.16b}, [$inp], #64
eor $T0.16b, $T0.16b, $A1.16b
eor $T1.16b, $T1.16b, $B1.16b
eor $T2.16b, $T2.16b, $C1.16b
eor $T3.16b, $T3.16b, $D1.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #64
b .Lopen_tail_64_store
.Lopen_tail_64:
// We just need a single block
mov $A0.16b, $CONSTS.16b
mov $B0.16b, $B_STORE.16b
mov $C0.16b, $C_STORE.16b
mov $D0.16b, $D_STORE.16b
eor $T3.16b, $T3.16b, $T3.16b
ins $T3.s[0], $INC.s[0]
add $D0.4s, $D0.4s, $T3.4s
mov $itr1, #10
sub $itr1, $itr1, $adl
.Lopen_tail_64_rounds:
___
&chacha_qr($A0, $B0, $C0, $D0, $T0, "left");
&chacha_qr($A0, $B0, $C0, $D0, $T0, "right");
$code.=<<___;
subs $itr1, $itr1, #1
b.gt .Lopen_tail_64_rounds
cbz $adl, .Lopen_tail_64_rounds_done
subs $adl, $adl, #1
___
&poly_add($adp);
&poly_mul();
$code.=<<___;
b .Lopen_tail_64_rounds
.Lopen_tail_64_rounds_done:
add $A0.4s, $A0.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
add $C0.4s, $C0.4s, $C_STORE.4s
add $D0.4s, $D0.4s, $D_STORE.4s
add $D0.4s, $D0.4s, $T3.4s
.Lopen_tail_64_store:
cmp $inl, #16
b.lt .Lopen_tail_16
ld1 {$T0.16b}, [$inp], #16
eor $T0.16b, $T0.16b, $A0.16b
st1 {$T0.16b}, [$oup], #16
mov $A0.16b, $B0.16b
mov $B0.16b, $C0.16b
mov $C0.16b, $D0.16b
sub $inl, $inl, #16
b .Lopen_tail_64_store
.Lopen_tail_16:
// Here we handle the last [0,16) bytes that require a padded block
cbz $inl, .Lopen_finalize
eor $T0.16b, $T0.16b, $T0.16b // Use T0 to load the ciphertext
eor $T1.16b, $T1.16b, $T1.16b // Use T1 to generate an AND mask
not $T2.16b, $T0.16b
add $itr2, $inp, $inl
mov $itr1, $inl
.Lopen_tail_16_compose:
ext $T0.16b, $T0.16b, $T0.16b, #15
ldrb $t0w, [$itr2, #-1]!
mov $T0.b[0], $t0w
ext $T1.16b, $T2.16b, $T1.16b, #15
subs $inl, $inl, #1
b.gt .Lopen_tail_16_compose
and $T0.16b, $T0.16b, $T1.16b
// Hash in the final padded block
___
&poly_add_vec($T0);
&poly_mul();
$code.=<<___;
eor $T0.16b, $T0.16b, $A0.16b
.Lopen_tail_16_store:
umov $t0w, $T0.b[0]
strb $t0w, [$oup], #1
ext $T0.16b, $T0.16b, $T0.16b, #1
subs $itr1, $itr1, #1
b.gt .Lopen_tail_16_store
.Lopen_finalize:
___
&poly_add_vec($LEN_STORE);
&poly_mul();
$code.=<<___;
// Final reduction step
sub $t1, xzr, $one
orr $t2, xzr, #3
subs $t0, $acc0, #-5
sbcs $t1, $acc1, $t1
sbcs $t2, $acc2, $t2
csel $acc0, $t0, $acc0, cs
csel $acc1, $t1, $acc1, cs
csel $acc2, $t2, $acc2, cs
___
&poly_add_vec($S_STORE);
$code.=<<___;
stp $acc0, $acc1, [$keyp]
ldp d8, d9, [sp, #16]
ldp d10, d11, [sp, #32]
ldp d12, d13, [sp, #48]
ldp d14, d15, [sp, #64]
.cfi_restore b15
.cfi_restore b14
.cfi_restore b13
.cfi_restore b12
.cfi_restore b11
.cfi_restore b10
.cfi_restore b9
.cfi_restore b8
ldp x29, x30, [sp], 80
.cfi_restore w29
.cfi_restore w30
.cfi_def_cfa_offset 0
AARCH64_VALIDATE_LINK_REGISTER
ret
.Lopen_128:
// On some architectures preparing 5 blocks for small buffers is wasteful
eor $INC.16b, $INC.16b, $INC.16b
mov $t0, #1
mov $INC.s[0], $t0w
mov $A0.16b, $CONSTS.16b
mov $A1.16b, $CONSTS.16b
mov $A2.16b, $CONSTS.16b
mov $B0.16b, $B_STORE.16b
mov $B1.16b, $B_STORE.16b
mov $B2.16b, $B_STORE.16b
mov $C0.16b, $C_STORE.16b
mov $C1.16b, $C_STORE.16b
mov $C2.16b, $C_STORE.16b
mov $D2.16b, $D_STORE.16b
add $D0.4s, $D2.4s, $INC.4s
add $D1.4s, $D0.4s, $INC.4s
mov $itr1, #10
.Lopen_128_rounds:
___
&chacha_qr_x3("left");
&chacha_qr_x3("right");
$code.=<<___;
subs $itr1, $itr1, #1
b.hi .Lopen_128_rounds
add $A0.4s, $A0.4s, $CONSTS.4s
add $A1.4s, $A1.4s, $CONSTS.4s
add $A2.4s, $A2.4s, $CONSTS.4s
add $B0.4s, $B0.4s, $B_STORE.4s
add $B1.4s, $B1.4s, $B_STORE.4s
add $B2.4s, $B2.4s, $B_STORE.4s
add $C0.4s, $C0.4s, $C_STORE.4s
add $C1.4s, $C1.4s, $C_STORE.4s
add $D_STORE.4s, $D_STORE.4s, $INC.4s
add $D0.4s, $D0.4s, $D_STORE.4s
add $D_STORE.4s, $D_STORE.4s, $INC.4s
add $D1.4s, $D1.4s, $D_STORE.4s
and $A2.16b, $A2.16b, $CLAMP.16b
mov $r0, $A2.d[0] // Move the R key to GPRs
mov $r1, $A2.d[1]
mov $S_STORE.16b, $B2.16b // Store the S key
bl .Lpoly_hash_ad_internal
.Lopen_128_store:
cmp $inl, #64
b.lt .Lopen_128_store_64
ld1 {$T0.16b - $T3.16b}, [$inp], #64
___
&poly_add_vec($T0);
&poly_mul();
&poly_add_vec($T1);
&poly_mul();
&poly_add_vec($T2);
&poly_mul();
&poly_add_vec($T3);
&poly_mul();
$code.=<<___;
eor $T0.16b, $T0.16b, $A0.16b
eor $T1.16b, $T1.16b, $B0.16b
eor $T2.16b, $T2.16b, $C0.16b
eor $T3.16b, $T3.16b, $D0.16b
st1 {$T0.16b - $T3.16b}, [$oup], #64
sub $inl, $inl, #64
mov $A0.16b, $A1.16b
mov $B0.16b, $B1.16b
mov $C0.16b, $C1.16b
mov $D0.16b, $D1.16b
.Lopen_128_store_64:
lsr $adl, $inl, #4
mov $adp, $inp
.Lopen_128_hash_64:
cbz $adl, .Lopen_tail_64_store
___
&poly_add($adp);
&poly_mul();
$code.=<<___;
sub $adl, $adl, #1
b .Lopen_128_hash_64
.cfi_endproc
.size chacha20_poly1305_open,.-chacha20_poly1305_open
___
}
foreach (split("\n",$code)) {
s/\`([^\`]*)\`/eval $1/ge;
print $_,"\n";
}
close STDOUT or die "error closing STDOUT";