Revert "Replace most of P-256 assembly code with C" This reverts commit ba76be9c75d35798a397a8f6d741cc360a3026ae. This seems to have caused significant performance regressions in some configurations. See b/525050645 (internal). It was not a clean revert due to other changes that went in in the meantime. I had to revert the following other CLs: * 85a291c0ef3cd5774d961a39b286d253efe401b2 * 28950bf42fdaa937e31ee0be8e02e97bbc6a17dc * 6bec274e2a2d7dfc6ac3919d55264597695cb1c7 * 66370b6b9301fdf4d316d049581832cba46a4e99 Then I resolved merge conflicts with the changes to put a symbol at the start of all the .rodata sections, and then reapplied 28950bf42fdaa937e31ee0be8e02e97bbc6a17dc and 6bec274e2a2d7dfc6ac3919d55264597695cb1c7, but to where the code used to be. I also kept the changes to: - crypto/ecdh/ecdh_test.cc - crypto/internal.h - third_party/fiat/bedrock_unverified_platform.c.inc Bug: 525050645 Change-Id: I1a016785dfa3da57ac07d26dd034a67d0bc14566 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/97607 Reviewed-by: Adam Langley <agl@google.com> Presubmit-BoringSSL-Verified: boringssl-scoped@luci-project-accounts.iam.gserviceaccount.com <boringssl-scoped@luci-project-accounts.iam.gserviceaccount.com> Commit-Queue: David Benjamin <davidben@google.com>
diff --git a/build.json b/build.json index fe101b3..f9e3a7e 100644 --- a/build.json +++ b/build.json
@@ -62,6 +62,7 @@ "crypto/fipsmodule/ec/ec_montgomery.cc.inc", "crypto/fipsmodule/ec/felem.cc.inc", "crypto/fipsmodule/ec/oct.cc.inc", + "crypto/fipsmodule/ec/p256-nistz.cc.inc", "crypto/fipsmodule/ec/p256.cc.inc", "crypto/fipsmodule/ec/scalar.cc.inc", "crypto/fipsmodule/ec/simple.cc.inc", @@ -524,8 +525,8 @@ "crypto/fipsmodule/digest/md32_common.h", "crypto/fipsmodule/ec/builtin_curves.h", "crypto/fipsmodule/ec/internal.h", - "crypto/fipsmodule/ec/p256_internal.h", "crypto/fipsmodule/ec/p256-nistz-table.h", + "crypto/fipsmodule/ec/p256-nistz.h", "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/entropy/internal.h", @@ -567,6 +568,7 @@ "third_party/fiat/p256_32.h", "third_party/fiat/p256_64.h", "third_party/fiat/p256_64_msvc.h", + "third_party/fiat/p256_field.c.inc", "third_party/fiat/p256_field_32.br.c.inc", "third_party/fiat/p256_field_64.br.c.inc", "third_party/fiat/p256_point.br.c.inc", @@ -852,6 +854,7 @@ "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", + "crypto/fipsmodule/ec/p256-nistz_test.cc", "crypto/fipsmodule/ec/p256_test.cc", "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/entropy/jitter_test.cc", @@ -910,7 +913,7 @@ "crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt", "crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt", "crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt", - "crypto/fipsmodule/ec/p256-tests.txt", + "crypto/fipsmodule/ec/p256-nistz_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt", "crypto/fipsmodule/keccak/keccak_tests.txt",
diff --git a/crypto/fipsmodule/bcm.cc b/crypto/fipsmodule/bcm.cc index 7cb8410..b0eae67 100644 --- a/crypto/fipsmodule/bcm.cc +++ b/crypto/fipsmodule/bcm.cc
@@ -87,6 +87,7 @@ #include "ec/ec_montgomery.cc.inc" #include "ec/felem.cc.inc" #include "ec/oct.cc.inc" +#include "ec/p256-nistz.cc.inc" #include "ec/p256.cc.inc" #include "ec/scalar.cc.inc" #include "ec/simple.cc.inc"
diff --git a/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl b/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl index 00d4d9c..97707a1 100644 --- a/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl +++ b/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl
@@ -15,9 +15,26 @@ # ==================================================================== -# Originally written by Andy Polyakov <appro@openssl.org> for the OpenSSL -# project. Modified for BoringSSL. +# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL +# project. # ==================================================================== +# +# ECP_NISTZ256 module for ARMv8. +# +# February 2015. +# +# Original ECP_NISTZ256 submission targeting x86_64 is detailed in +# http://eprint.iacr.org/2013/816. +# +# with/without -DECP_NISTZ256_ASM +# Apple A7 +190-360% +# Cortex-A53 +190-400% +# Cortex-A57 +190-350% +# Denver +230-400% +# +# Ranges denote minimum and maximum improvement coefficients depending +# on benchmark. Lower coefficients are for ECDSA sign, server-side +# operation. Keep in mind that +400% means 5x improvement. # The first two arguments should always be the flavour and output file path. if ($#ARGV < 1) { die "Not enough arguments provided. @@ -35,13 +52,33 @@ *STDOUT=*OUT; { -my ($rp,$bp,$bi,$a0,$a1,$a2,$a3,$poly1,$t0,$t1,$t2,$t3,$poly3, - $acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6) = +my ($rp,$ap,$bp,$bi,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3,$poly1,$poly3, + $acc0,$acc1,$acc2,$acc3,$acc4,$acc5) = map("x$_",(0..17,19,20)); -my $acc7=$bi; # used in ecp_nistz256_sqr_mont -my $poly1w = $poly1 =~ s/x/w/r; + +my ($acc6,$acc7)=($ap,$bp); # used in __ecp_nistz256_sqr_mont $code.=<<___; +.section .rodata +.align 5 +p256_constants: +.Lpoly: +.quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 +.LRR: // 2^512 mod P precomputed for NIST P256 polynomial +.quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd +.Lone_mont: +.quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe +.Lone: +.quad 1,0,0,0 +.Lord: +.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 +.LordK: +.quad 0xccd1c8aaee00bc4f +.asciz "ECP_NISTZ256 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>" +.text + +// void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4], +// const BN_ULONG x2[4]); .globl ecp_nistz256_mul_mont .type ecp_nistz256_mul_mont,%function .align 4 @@ -51,6 +88,190 @@ add x29,sp,#0 stp x19,x20,[sp,#16] + ldr $bi,[$bp] // bp[0] + ldp $a0,$a1,[$ap] + ldp $a2,$a3,[$ap,#16] + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + + bl __ecp_nistz256_mul_mont + + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont + +// void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_sqr_mont +.type ecp_nistz256_sqr_mont,%function +.align 4 +ecp_nistz256_sqr_mont: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-32]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + + ldp $a0,$a1,[$ap] + ldp $a2,$a3,[$ap,#16] + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + + bl __ecp_nistz256_sqr_mont + + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont + +// void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_div_by_2 +.type ecp_nistz256_div_by_2,%function +.align 4 +ecp_nistz256_div_by_2: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp $acc0,$acc1,[$ap] + ldp $acc2,$acc3,[$ap,#16] + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + + bl __ecp_nistz256_div_by_2 + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2 + +// void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_mul_by_2 +.type ecp_nistz256_mul_by_2,%function +.align 4 +ecp_nistz256_mul_by_2: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp $acc0,$acc1,[$ap] + ldp $acc2,$acc3,[$ap,#16] + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + mov $t0,$acc0 + mov $t1,$acc1 + mov $t2,$acc2 + mov $t3,$acc3 + + bl __ecp_nistz256_add_to // ret = a+a // 2*a + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2 + +// void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_mul_by_3 +.type ecp_nistz256_mul_by_3,%function +.align 4 +ecp_nistz256_mul_by_3: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp $acc0,$acc1,[$ap] + ldp $acc2,$acc3,[$ap,#16] + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + mov $t0,$acc0 + mov $t1,$acc1 + mov $t2,$acc2 + mov $t3,$acc3 + mov $a0,$acc0 + mov $a1,$acc1 + mov $a2,$acc2 + mov $a3,$acc3 + + bl __ecp_nistz256_add_to // ret = a+a // 2*a + + mov $t0,$a0 + mov $t1,$a1 + mov $t2,$a2 + mov $t3,$a3 + + bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3 + +// void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4], +// const BN_ULONG x2[4]); +.globl ecp_nistz256_sub +.type ecp_nistz256_sub,%function +.align 4 +ecp_nistz256_sub: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp $acc0,$acc1,[$ap] + ldp $acc2,$acc3,[$ap,#16] + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + + bl __ecp_nistz256_sub_from + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_sub,.-ecp_nistz256_sub + +// void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_neg +.type ecp_nistz256_neg,%function +.align 4 +ecp_nistz256_neg: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov $bp,$ap + mov $acc0,xzr // a = 0 + mov $acc1,xzr + mov $acc2,xzr + mov $acc3,xzr + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + + bl __ecp_nistz256_sub_from + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_neg,.-ecp_nistz256_neg + +// note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded +// to $a0-$a3 and b[0] - to $bi +.type __ecp_nistz256_mul_mont,%function +.align 4 +__ecp_nistz256_mul_mont: mul $acc0,$a0,$bi // a[0]*b[0] umulh $t0,$a0,$bi @@ -134,7 +355,6 @@ ___ } $code.=<<___; - mov $poly1w, #-1 // poly1 = 0x00000000ffffffff // last reduction subs $t2,$acc0,$t0 // "*0xffff0001" sbc $t3,$acc0,$t1 @@ -144,7 +364,6 @@ adcs $acc3,$acc4,$t3 adc $acc4,$acc5,xzr - neg $poly3,$poly1 // poly3 = 0xffffffff00000001 adds $t0,$acc0,#1 // subs $t0,$acc0,#-1 // tmp = ret-modulus sbcs $t1,$acc1,$poly1 sbcs $t2,$acc2,xzr @@ -158,21 +377,14 @@ csel $acc3,$acc3,$t3,lo stp $acc2,$acc3,[$rp,#16] - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER ret -.size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont +.size __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont -.globl ecp_nistz256_sqr_mont -.type ecp_nistz256_sqr_mont,%function +// note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded +// to $a0-$a3 +.type __ecp_nistz256_sqr_mont,%function .align 4 -ecp_nistz256_sqr_mont: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-32]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - +__ecp_nistz256_sqr_mont: // | | | | | |a1*a0| | // | | | | |a2*a0| | | // | |a3*a2|a3*a0| | | | @@ -260,14 +472,12 @@ adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001 adc $acc3,$t3,xzr // can't overflow - mov $poly1w, #-1 // poly1 = 0x00000000ffffffff adds $acc0,$acc0,$acc4 // accumulate upper half adcs $acc1,$acc1,$acc5 adcs $acc2,$acc2,$acc6 adcs $acc3,$acc3,$acc7 adc $acc4,xzr,xzr - neg $poly3,$poly1 // poly3 = 0xffffffff00000001 adds $t0,$acc0,#1 // subs $t0,$acc0,#-1 // tmp = ret-modulus sbcs $t1,$acc1,$poly1 sbcs $t2,$acc2,xzr @@ -281,36 +491,762 @@ csel $acc3,$acc3,$t3,lo stp $acc2,$acc3,[$rp,#16] - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 + ret +.size __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont + +// Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to +// $a0-$a3 and $t0-$t3. This is done because it's used in multiple +// contexts, e.g. in multiplication by 2 and 3... +.type __ecp_nistz256_add_to,%function +.align 4 +__ecp_nistz256_add_to: + adds $acc0,$acc0,$t0 // ret = a+b + adcs $acc1,$acc1,$t1 + adcs $acc2,$acc2,$t2 + adcs $acc3,$acc3,$t3 + adc $ap,xzr,xzr // zap $ap + + adds $t0,$acc0,#1 // subs $t0,$a0,#-1 // tmp = ret-modulus + sbcs $t1,$acc1,$poly1 + sbcs $t2,$acc2,xzr + sbcs $t3,$acc3,$poly3 + sbcs xzr,$ap,xzr // did subtraction borrow? + + csel $acc0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus + csel $acc1,$acc1,$t1,lo + csel $acc2,$acc2,$t2,lo + stp $acc0,$acc1,[$rp] + csel $acc3,$acc3,$t3,lo + stp $acc2,$acc3,[$rp,#16] + + ret +.size __ecp_nistz256_add_to,.-__ecp_nistz256_add_to + +.type __ecp_nistz256_sub_from,%function +.align 4 +__ecp_nistz256_sub_from: + ldp $t0,$t1,[$bp] + ldp $t2,$t3,[$bp,#16] + subs $acc0,$acc0,$t0 // ret = a-b + sbcs $acc1,$acc1,$t1 + sbcs $acc2,$acc2,$t2 + sbcs $acc3,$acc3,$t3 + sbc $ap,xzr,xzr // zap $ap + + subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = ret+modulus + adcs $t1,$acc1,$poly1 + adcs $t2,$acc2,xzr + adc $t3,$acc3,$poly3 + cmp $ap,xzr // did subtraction borrow? + + csel $acc0,$acc0,$t0,eq // ret = borrow ? ret+modulus : ret + csel $acc1,$acc1,$t1,eq + csel $acc2,$acc2,$t2,eq + stp $acc0,$acc1,[$rp] + csel $acc3,$acc3,$t3,eq + stp $acc2,$acc3,[$rp,#16] + + ret +.size __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from + +.type __ecp_nistz256_sub_morf,%function +.align 4 +__ecp_nistz256_sub_morf: + ldp $t0,$t1,[$bp] + ldp $t2,$t3,[$bp,#16] + subs $acc0,$t0,$acc0 // ret = b-a + sbcs $acc1,$t1,$acc1 + sbcs $acc2,$t2,$acc2 + sbcs $acc3,$t3,$acc3 + sbc $ap,xzr,xzr // zap $ap + + subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = ret+modulus + adcs $t1,$acc1,$poly1 + adcs $t2,$acc2,xzr + adc $t3,$acc3,$poly3 + cmp $ap,xzr // did subtraction borrow? + + csel $acc0,$acc0,$t0,eq // ret = borrow ? ret+modulus : ret + csel $acc1,$acc1,$t1,eq + csel $acc2,$acc2,$t2,eq + stp $acc0,$acc1,[$rp] + csel $acc3,$acc3,$t3,eq + stp $acc2,$acc3,[$rp,#16] + + ret +.size __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf + +.type __ecp_nistz256_div_by_2,%function +.align 4 +__ecp_nistz256_div_by_2: + subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = a+modulus + adcs $t1,$acc1,$poly1 + adcs $t2,$acc2,xzr + adcs $t3,$acc3,$poly3 + adc $ap,xzr,xzr // zap $ap + tst $acc0,#1 // is a even? + + csel $acc0,$acc0,$t0,eq // ret = even ? a : a+modulus + csel $acc1,$acc1,$t1,eq + csel $acc2,$acc2,$t2,eq + csel $acc3,$acc3,$t3,eq + csel $ap,xzr,$ap,eq + + lsr $acc0,$acc0,#1 // ret >>= 1 + orr $acc0,$acc0,$acc1,lsl#63 + lsr $acc1,$acc1,#1 + orr $acc1,$acc1,$acc2,lsl#63 + lsr $acc2,$acc2,#1 + orr $acc2,$acc2,$acc3,lsl#63 + lsr $acc3,$acc3,#1 + stp $acc0,$acc1,[$rp] + orr $acc3,$acc3,$ap,lsl#63 + stp $acc2,$acc3,[$rp,#16] + + ret +.size __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2 +___ +######################################################################## +# following subroutines are "literal" implementation of those found in +# ecp_nistz256.c +# +######################################################################## +# void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp); +# +{ +my ($S,$M,$Zsqr,$tmp0)=map(32*$_,(0..3)); +# above map() describes stack layout with 4 temporary +# 256-bit vectors on top. +my ($rp_real,$ap_real) = map("x$_",(21,22)); + +$code.=<<___; +.globl ecp_nistz256_point_double +.type ecp_nistz256_point_double,%function +.align 5 +ecp_nistz256_point_double: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + sub sp,sp,#32*4 + +.Ldouble_shortcut: + ldp $acc0,$acc1,[$ap,#32] + mov $rp_real,$rp + ldp $acc2,$acc3,[$ap,#48] + mov $ap_real,$ap + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + mov $t0,$acc0 + ldr $poly3,[$poly3,#24] + mov $t1,$acc1 + ldp $a0,$a1,[$ap_real,#64] // forward load for p256_sqr_mont + mov $t2,$acc2 + mov $t3,$acc3 + ldp $a2,$a3,[$ap_real,#64+16] + add $rp,sp,#$S + bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y); + + add $rp,sp,#$Zsqr + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z); + + ldp $t0,$t1,[$ap_real] + ldp $t2,$t3,[$ap_real,#16] + mov $a0,$acc0 // put Zsqr aside for p256_sub + mov $a1,$acc1 + mov $a2,$acc2 + mov $a3,$acc3 + add $rp,sp,#$M + bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x); + + add $bp,$ap_real,#0 + mov $acc0,$a0 // restore Zsqr + mov $acc1,$a1 + ldp $a0,$a1,[sp,#$S] // forward load for p256_sqr_mont + mov $acc2,$a2 + mov $acc3,$a3 + ldp $a2,$a3,[sp,#$S+16] + add $rp,sp,#$Zsqr + bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr); + + add $rp,sp,#$S + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S); + + ldr $bi,[$ap_real,#32] + ldp $a0,$a1,[$ap_real,#64] + ldp $a2,$a3,[$ap_real,#64+16] + add $bp,$ap_real,#32 + add $rp,sp,#$tmp0 + bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y); + + mov $t0,$acc0 + mov $t1,$acc1 + ldp $a0,$a1,[sp,#$S] // forward load for p256_sqr_mont + mov $t2,$acc2 + mov $t3,$acc3 + ldp $a2,$a3,[sp,#$S+16] + add $rp,$rp_real,#64 + bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0); + + add $rp,sp,#$tmp0 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S); + + ldr $bi,[sp,#$Zsqr] // forward load for p256_mul_mont + ldp $a0,$a1,[sp,#$M] + ldp $a2,$a3,[sp,#$M+16] + add $rp,$rp_real,#32 + bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0); + + add $bp,sp,#$Zsqr + add $rp,sp,#$M + bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr); + + mov $t0,$acc0 // duplicate M + mov $t1,$acc1 + mov $t2,$acc2 + mov $t3,$acc3 + mov $a0,$acc0 // put M aside + mov $a1,$acc1 + mov $a2,$acc2 + mov $a3,$acc3 + add $rp,sp,#$M + bl __ecp_nistz256_add_to + mov $t0,$a0 // restore M + mov $t1,$a1 + ldr $bi,[$ap_real] // forward load for p256_mul_mont + mov $t2,$a2 + ldp $a0,$a1,[sp,#$S] + mov $t3,$a3 + ldp $a2,$a3,[sp,#$S+16] + bl __ecp_nistz256_add_to // p256_mul_by_3(M, M); + + add $bp,$ap_real,#0 + add $rp,sp,#$S + bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x); + + mov $t0,$acc0 + mov $t1,$acc1 + ldp $a0,$a1,[sp,#$M] // forward load for p256_sqr_mont + mov $t2,$acc2 + mov $t3,$acc3 + ldp $a2,$a3,[sp,#$M+16] + add $rp,sp,#$tmp0 + bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S); + + add $rp,$rp_real,#0 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M); + + add $bp,sp,#$tmp0 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0); + + add $bp,sp,#$S + add $rp,sp,#$S + bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x); + + ldr $bi,[sp,#$M] + mov $a0,$acc0 // copy S + mov $a1,$acc1 + mov $a2,$acc2 + mov $a3,$acc3 + add $bp,sp,#$M + bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M); + + add $bp,$rp_real,#32 + add $rp,$rp_real,#32 + bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y); + + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x29,x30,[sp],#96 AARCH64_VALIDATE_LINK_REGISTER ret -.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont +.size ecp_nistz256_point_double,.-ecp_nistz256_point_double ___ +} +######################################################################## +# void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1, +# const P256_POINT *in2); +{ +my ($res_x,$res_y,$res_z, + $H,$Hsqr,$R,$Rsqr,$Hcub, + $U1,$U2,$S1,$S2)=map(32*$_,(0..11)); +my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr); +# above map() describes stack layout with 12 temporary +# 256-bit vectors on top. +my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp0,$temp1,$temp2)=map("x$_",(21..28)); + +$code.=<<___; +.globl ecp_nistz256_point_add +.type ecp_nistz256_point_add,%function +.align 5 +ecp_nistz256_point_add: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#32*12 + + ldp $a0,$a1,[$bp,#64] // in2_z + ldp $a2,$a3,[$bp,#64+16] + mov $rp_real,$rp + mov $ap_real,$ap + mov $bp_real,$bp + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + orr $t0,$a0,$a1 + orr $t2,$a2,$a3 + orr $in2infty,$t0,$t2 + cmp $in2infty,#0 + csetm $in2infty,ne // ~in2infty + add $rp,sp,#$Z2sqr + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z); + + ldp $a0,$a1,[$ap_real,#64] // in1_z + ldp $a2,$a3,[$ap_real,#64+16] + orr $t0,$a0,$a1 + orr $t2,$a2,$a3 + orr $in1infty,$t0,$t2 + cmp $in1infty,#0 + csetm $in1infty,ne // ~in1infty + add $rp,sp,#$Z1sqr + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); + + ldr $bi,[$bp_real,#64] + ldp $a0,$a1,[sp,#$Z2sqr] + ldp $a2,$a3,[sp,#$Z2sqr+16] + add $bp,$bp_real,#64 + add $rp,sp,#$S1 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z); + + ldr $bi,[$ap_real,#64] + ldp $a0,$a1,[sp,#$Z1sqr] + ldp $a2,$a3,[sp,#$Z1sqr+16] + add $bp,$ap_real,#64 + add $rp,sp,#$S2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); + + ldr $bi,[$ap_real,#32] + ldp $a0,$a1,[sp,#$S1] + ldp $a2,$a3,[sp,#$S1+16] + add $bp,$ap_real,#32 + add $rp,sp,#$S1 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y); + + ldr $bi,[$bp_real,#32] + ldp $a0,$a1,[sp,#$S2] + ldp $a2,$a3,[sp,#$S2+16] + add $bp,$bp_real,#32 + add $rp,sp,#$S2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); + + add $bp,sp,#$S1 + ldr $bi,[sp,#$Z2sqr] // forward load for p256_mul_mont + ldp $a0,$a1,[$ap_real] + ldp $a2,$a3,[$ap_real,#16] + add $rp,sp,#$R + bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1); + + orr $acc0,$acc0,$acc1 // see if result is zero + orr $acc2,$acc2,$acc3 + orr $temp0,$acc0,$acc2 // ~is_equal(S1,S2) + + add $bp,sp,#$Z2sqr + add $rp,sp,#$U1 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr); + + ldr $bi,[sp,#$Z1sqr] + ldp $a0,$a1,[$bp_real] + ldp $a2,$a3,[$bp_real,#16] + add $bp,sp,#$Z1sqr + add $rp,sp,#$U2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr); + + add $bp,sp,#$U1 + ldp $a0,$a1,[sp,#$R] // forward load for p256_sqr_mont + ldp $a2,$a3,[sp,#$R+16] + add $rp,sp,#$H + bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1); + + orr $acc0,$acc0,$acc1 // see if result is zero + orr $acc2,$acc2,$acc3 + orr $acc0,$acc0,$acc2 // ~is_equal(U1,U2) + + mvn $temp1,$in1infty // -1/0 -> 0/-1 + mvn $temp2,$in2infty // -1/0 -> 0/-1 + orr $acc0,$acc0,$temp1 + orr $acc0,$acc0,$temp2 + orr $acc0,$acc0,$temp0 + cbnz $acc0,.Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2)) + +.Ladd_double: + mov $ap,$ap_real + mov $rp,$rp_real + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames + b .Ldouble_shortcut + +.align 4 +.Ladd_proceed: + add $rp,sp,#$Rsqr + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); + + ldr $bi,[$ap_real,#64] + ldp $a0,$a1,[sp,#$H] + ldp $a2,$a3,[sp,#$H+16] + add $bp,$ap_real,#64 + add $rp,sp,#$res_z + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); + + ldp $a0,$a1,[sp,#$H] + ldp $a2,$a3,[sp,#$H+16] + add $rp,sp,#$Hsqr + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); + + ldr $bi,[$bp_real,#64] + ldp $a0,$a1,[sp,#$res_z] + ldp $a2,$a3,[sp,#$res_z+16] + add $bp,$bp_real,#64 + add $rp,sp,#$res_z + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z); + + ldr $bi,[sp,#$H] + ldp $a0,$a1,[sp,#$Hsqr] + ldp $a2,$a3,[sp,#$Hsqr+16] + add $bp,sp,#$H + add $rp,sp,#$Hcub + bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); + + ldr $bi,[sp,#$Hsqr] + ldp $a0,$a1,[sp,#$U1] + ldp $a2,$a3,[sp,#$U1+16] + add $bp,sp,#$Hsqr + add $rp,sp,#$U2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr); + + mov $t0,$acc0 + mov $t1,$acc1 + mov $t2,$acc2 + mov $t3,$acc3 + add $rp,sp,#$Hsqr + bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); + + add $bp,sp,#$Rsqr + add $rp,sp,#$res_x + bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); + + add $bp,sp,#$Hcub + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); + + add $bp,sp,#$U2 + ldr $bi,[sp,#$Hcub] // forward load for p256_mul_mont + ldp $a0,$a1,[sp,#$S1] + ldp $a2,$a3,[sp,#$S1+16] + add $rp,sp,#$res_y + bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); + + add $bp,sp,#$Hcub + add $rp,sp,#$S2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub); + + ldr $bi,[sp,#$R] + ldp $a0,$a1,[sp,#$res_y] + ldp $a2,$a3,[sp,#$res_y+16] + add $bp,sp,#$R + add $rp,sp,#$res_y + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); + + add $bp,sp,#$S2 + bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); + + ldp $a0,$a1,[sp,#$res_x] // res + ldp $a2,$a3,[sp,#$res_x+16] + ldp $t0,$t1,[$bp_real] // in2 + ldp $t2,$t3,[$bp_real,#16] +___ +for($i=0;$i<64;$i+=32) { # conditional moves +$code.=<<___; + ldp $acc0,$acc1,[$ap_real,#$i] // in1 + cmp $in1infty,#0 // ~$in1intfy, remember? + ldp $acc2,$acc3,[$ap_real,#$i+16] + csel $t0,$a0,$t0,ne + csel $t1,$a1,$t1,ne + ldp $a0,$a1,[sp,#$res_x+$i+32] // res + csel $t2,$a2,$t2,ne + csel $t3,$a3,$t3,ne + cmp $in2infty,#0 // ~$in2intfy, remember? + ldp $a2,$a3,[sp,#$res_x+$i+48] + csel $acc0,$t0,$acc0,ne + csel $acc1,$t1,$acc1,ne + ldp $t0,$t1,[$bp_real,#$i+32] // in2 + csel $acc2,$t2,$acc2,ne + csel $acc3,$t3,$acc3,ne + ldp $t2,$t3,[$bp_real,#$i+48] + stp $acc0,$acc1,[$rp_real,#$i] + stp $acc2,$acc3,[$rp_real,#$i+16] +___ +} +$code.=<<___; + ldp $acc0,$acc1,[$ap_real,#$i] // in1 + cmp $in1infty,#0 // ~$in1intfy, remember? + ldp $acc2,$acc3,[$ap_real,#$i+16] + csel $t0,$a0,$t0,ne + csel $t1,$a1,$t1,ne + csel $t2,$a2,$t2,ne + csel $t3,$a3,$t3,ne + cmp $in2infty,#0 // ~$in2intfy, remember? + csel $acc0,$t0,$acc0,ne + csel $acc1,$t1,$acc1,ne + csel $acc2,$t2,$acc2,ne + csel $acc3,$t3,$acc3,ne + stp $acc0,$acc1,[$rp_real,#$i] + stp $acc2,$acc3,[$rp_real,#$i+16] + +.Ladd_done: + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_point_add,.-ecp_nistz256_point_add +___ +} + +######################################################################## +# void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1, +# const P256_POINT_AFFINE *in2); +{ +my ($res_x,$res_y,$res_z, + $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..9)); +my $Z1sqr = $S2; +# above map() describes stack layout with 10 temporary +# 256-bit vectors on top. +my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("x$_",(21..26)); + +$code.=<<___; +.globl ecp_nistz256_point_add_affine +.type ecp_nistz256_point_add_affine,%function +.align 5 +ecp_nistz256_point_add_affine: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + sub sp,sp,#32*10 + + mov $rp_real,$rp + mov $ap_real,$ap + mov $bp_real,$bp + adrp $poly3,:pg_hi21:.Lpoly + add $poly3,$poly3,:lo12:.Lpoly + ldr $poly1,[$poly3,#8] + ldr $poly3,[$poly3,#24] + + ldp $a0,$a1,[$ap,#64] // in1_z + ldp $a2,$a3,[$ap,#64+16] + orr $t0,$a0,$a1 + orr $t2,$a2,$a3 + orr $in1infty,$t0,$t2 + cmp $in1infty,#0 + csetm $in1infty,ne // ~in1infty + + ldp $acc0,$acc1,[$bp] // in2_x + ldp $acc2,$acc3,[$bp,#16] + ldp $t0,$t1,[$bp,#32] // in2_y + ldp $t2,$t3,[$bp,#48] + orr $acc0,$acc0,$acc1 + orr $acc2,$acc2,$acc3 + orr $t0,$t0,$t1 + orr $t2,$t2,$t3 + orr $acc0,$acc0,$acc2 + orr $t0,$t0,$t2 + orr $in2infty,$acc0,$t0 + cmp $in2infty,#0 + csetm $in2infty,ne // ~in2infty + + add $rp,sp,#$Z1sqr + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); + + mov $a0,$acc0 + mov $a1,$acc1 + mov $a2,$acc2 + mov $a3,$acc3 + ldr $bi,[$bp_real] + add $bp,$bp_real,#0 + add $rp,sp,#$U2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x); + + add $bp,$ap_real,#0 + ldr $bi,[$ap_real,#64] // forward load for p256_mul_mont + ldp $a0,$a1,[sp,#$Z1sqr] + ldp $a2,$a3,[sp,#$Z1sqr+16] + add $rp,sp,#$H + bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x); + + add $bp,$ap_real,#64 + add $rp,sp,#$S2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); + + ldr $bi,[$ap_real,#64] + ldp $a0,$a1,[sp,#$H] + ldp $a2,$a3,[sp,#$H+16] + add $bp,$ap_real,#64 + add $rp,sp,#$res_z + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); + + ldr $bi,[$bp_real,#32] + ldp $a0,$a1,[sp,#$S2] + ldp $a2,$a3,[sp,#$S2+16] + add $bp,$bp_real,#32 + add $rp,sp,#$S2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); + + add $bp,$ap_real,#32 + ldp $a0,$a1,[sp,#$H] // forward load for p256_sqr_mont + ldp $a2,$a3,[sp,#$H+16] + add $rp,sp,#$R + bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y); + + add $rp,sp,#$Hsqr + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); + + ldp $a0,$a1,[sp,#$R] + ldp $a2,$a3,[sp,#$R+16] + add $rp,sp,#$Rsqr + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); + + ldr $bi,[sp,#$H] + ldp $a0,$a1,[sp,#$Hsqr] + ldp $a2,$a3,[sp,#$Hsqr+16] + add $bp,sp,#$H + add $rp,sp,#$Hcub + bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); + + ldr $bi,[$ap_real] + ldp $a0,$a1,[sp,#$Hsqr] + ldp $a2,$a3,[sp,#$Hsqr+16] + add $bp,$ap_real,#0 + add $rp,sp,#$U2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr); + + mov $t0,$acc0 + mov $t1,$acc1 + mov $t2,$acc2 + mov $t3,$acc3 + add $rp,sp,#$Hsqr + bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); + + add $bp,sp,#$Rsqr + add $rp,sp,#$res_x + bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); + + add $bp,sp,#$Hcub + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); + + add $bp,sp,#$U2 + ldr $bi,[$ap_real,#32] // forward load for p256_mul_mont + ldp $a0,$a1,[sp,#$Hcub] + ldp $a2,$a3,[sp,#$Hcub+16] + add $rp,sp,#$res_y + bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); + + add $bp,$ap_real,#32 + add $rp,sp,#$S2 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub); + + ldr $bi,[sp,#$R] + ldp $a0,$a1,[sp,#$res_y] + ldp $a2,$a3,[sp,#$res_y+16] + add $bp,sp,#$R + add $rp,sp,#$res_y + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); + + add $bp,sp,#$S2 + bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); + + ldp $a0,$a1,[sp,#$res_x] // res + ldp $a2,$a3,[sp,#$res_x+16] + ldp $t0,$t1,[$bp_real] // in2 + ldp $t2,$t3,[$bp_real,#16] +___ +for($i=0;$i<64;$i+=32) { # conditional moves +$code.=<<___; + ldp $acc0,$acc1,[$ap_real,#$i] // in1 + cmp $in1infty,#0 // ~$in1intfy, remember? + ldp $acc2,$acc3,[$ap_real,#$i+16] + csel $t0,$a0,$t0,ne + csel $t1,$a1,$t1,ne + ldp $a0,$a1,[sp,#$res_x+$i+32] // res + csel $t2,$a2,$t2,ne + csel $t3,$a3,$t3,ne + cmp $in2infty,#0 // ~$in2intfy, remember? + ldp $a2,$a3,[sp,#$res_x+$i+48] + csel $acc0,$t0,$acc0,ne + csel $acc1,$t1,$acc1,ne + ldp $t0,$t1,[$bp_real,#$i+32] // in2 + csel $acc2,$t2,$acc2,ne + csel $acc3,$t3,$acc3,ne + ldp $t2,$t3,[$bp_real,#$i+48] + stp $acc0,$acc1,[$rp_real,#$i] + stp $acc2,$acc3,[$rp_real,#$i+16] +___ +$code.=<<___ if ($i == 0); + adrp $bp_real,:pg_hi21:.Lone_mont-64 + add $bp_real,$bp_real,:lo12:.Lone_mont-64 +___ +} +$code.=<<___; + ldp $acc0,$acc1,[$ap_real,#$i] // in1 + cmp $in1infty,#0 // ~$in1intfy, remember? + ldp $acc2,$acc3,[$ap_real,#$i+16] + csel $t0,$a0,$t0,ne + csel $t1,$a1,$t1,ne + csel $t2,$a2,$t2,ne + csel $t3,$a3,$t3,ne + cmp $in2infty,#0 // ~$in2intfy, remember? + csel $acc0,$t0,$acc0,ne + csel $acc1,$t1,$acc1,ne + csel $acc2,$t2,$acc2,ne + csel $acc3,$t3,$acc3,ne + stp $acc0,$acc1,[$rp_real,#$i] + stp $acc2,$acc3,[$rp_real,#$i+16] + + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x29,x30,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine +___ +} if (1) { -my ($rp,$ap,$bp,$bi,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3,$poly1,$poly3, - $acc0,$acc1,$acc2,$acc3,$acc4,$acc5) = - map("x$_",(0..17,19,20)); - -my $poly1w = $poly1 =~ s/x/w/r; my ($ord0,$ord1) = ($poly1,$poly3); my ($ord2,$ord3,$ordk,$t4) = map("x$_",(21..24)); -my $acc6 = $ap; my $acc7 = $bi; $code.=<<___; -.section .rodata - -.align 5 -p256_constants: -.Lord: -.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 -.LordK: -.quad 0xccd1c8aaee00bc4f - .asciz "ECP_NISTZ256 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>" - .text - +//////////////////////////////////////////////////////////////////////// // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4], // uint64_t b[4]); .globl ecp_nistz256_ord_mul_mont @@ -333,7 +1269,7 @@ ldp $ord0,$ord1,[$ordk,#0] ldp $ord2,$ord3,[$ordk,#16] - ldr $ordk,[$ordk,#32] // LordK + ldr $ordk,[$ordk,#32] mul $acc0,$a0,$bi // a[0]*b[0] umulh $t0,$a0,$bi @@ -607,6 +1543,135 @@ ___ } } +######################################################################## +# select subroutines +# These select functions are similar to those in p256-x86_64-asm.pl +# They load all points in the lookup table +# keeping in the output only the one corresponding to the input index. +{ +my ($val,$in_t)=map("x$_",(0..1)); +my ($index)=("w2"); +my ($Idx_ctr,$Val_in, $Mask_64)=("w9", "x10", "x11"); +my ($Mask)=("v3"); +my ($Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("v$_",(16..21)); +my ($T0a,$T0b,$T0c,$T0d,$T0e,$T0f)=map("v$_",(22..27)); +$code.=<<___; +//////////////////////////////////////////////////////////////////////// +// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w5 +.type ecp_nistz256_select_w5,%function +.align 4 +ecp_nistz256_select_w5: + AARCH64_VALID_CALL_TARGET + + // $Val_in := $val + // $Idx_ctr := 0; loop counter and incremented internal index + mov $Val_in, $val + mov $Idx_ctr, #0 + + // [$Ra-$Rf] := 0 + movi $Ra.16b, #0 + movi $Rb.16b, #0 + movi $Rc.16b, #0 + movi $Rd.16b, #0 + movi $Re.16b, #0 + movi $Rf.16b, #0 + +.Lselect_w5_loop: + // Loop 16 times. + + // Increment index (loop counter); tested at the end of the loop + add $Idx_ctr, $Idx_ctr, #1 + + // [$T0a-$T0f] := Load a (3*256-bit = 6*128-bit) table entry starting at $in_t + // and advance $in_t to point to the next entry + ld1 {$T0a.2d, $T0b.2d, $T0c.2d, $T0d.2d}, [$in_t],#64 + + // $Mask_64 := ($Idx_ctr == $index)? All 1s : All 0s + cmp $Idx_ctr, $index + csetm $Mask_64, eq + + // continue loading ... + ld1 {$T0e.2d, $T0f.2d}, [$in_t],#32 + + // duplicate mask_64 into Mask (all 0s or all 1s) + dup $Mask.2d, $Mask_64 + + // [$Ra-$Rd] := (Mask == all 1s)? [$T0a-$T0d] : [$Ra-$Rd] + // i.e., values in output registers will remain the same if $Idx_ctr != $index + bit $Ra.16b, $T0a.16b, $Mask.16b + bit $Rb.16b, $T0b.16b, $Mask.16b + + bit $Rc.16b, $T0c.16b, $Mask.16b + bit $Rd.16b, $T0d.16b, $Mask.16b + + bit $Re.16b, $T0e.16b, $Mask.16b + bit $Rf.16b, $T0f.16b, $Mask.16b + + // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back + tbz $Idx_ctr, #4, .Lselect_w5_loop + + // Write [$Ra-$Rf] to memory at the output pointer + st1 {$Ra.2d, $Rb.2d, $Rc.2d, $Rd.2d}, [$Val_in],#64 + st1 {$Re.2d, $Rf.2d}, [$Val_in] + + ret +.size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5 + + +//////////////////////////////////////////////////////////////////////// +// void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w7 +.type ecp_nistz256_select_w7,%function +.align 4 +ecp_nistz256_select_w7: + AARCH64_VALID_CALL_TARGET + + // $Idx_ctr := 0; loop counter and incremented internal index + mov $Idx_ctr, #0 + + // [$Ra-$Rf] := 0 + movi $Ra.16b, #0 + movi $Rb.16b, #0 + movi $Rc.16b, #0 + movi $Rd.16b, #0 + +.Lselect_w7_loop: + // Loop 64 times. + + // Increment index (loop counter); tested at the end of the loop + add $Idx_ctr, $Idx_ctr, #1 + + // [$T0a-$T0d] := Load a (2*256-bit = 4*128-bit) table entry starting at $in_t + // and advance $in_t to point to the next entry + ld1 {$T0a.2d, $T0b.2d, $T0c.2d, $T0d.2d}, [$in_t],#64 + + // $Mask_64 := ($Idx_ctr == $index)? All 1s : All 0s + cmp $Idx_ctr, $index + csetm $Mask_64, eq + + // duplicate mask_64 into Mask (all 0s or all 1s) + dup $Mask.2d, $Mask_64 + + // [$Ra-$Rd] := (Mask == all 1s)? [$T0a-$T0d] : [$Ra-$Rd] + // i.e., values in output registers will remain the same if $Idx_ctr != $index + bit $Ra.16b, $T0a.16b, $Mask.16b + bit $Rb.16b, $T0b.16b, $Mask.16b + + bit $Rc.16b, $T0c.16b, $Mask.16b + bit $Rd.16b, $T0d.16b, $Mask.16b + + // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back + tbz $Idx_ctr, #6, .Lselect_w7_loop + + // Write [$Ra-$Rd] to memory at the output pointer + st1 {$Ra.2d, $Rb.2d, $Rc.2d, $Rd.2d}, [$val] + + ret +.size ecp_nistz256_select_w7,.-ecp_nistz256_select_w7 +___ +} + foreach (split("\n",$code)) { s/\`([^\`]*)\`/eval $1/ge;
diff --git a/crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl b/crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl index 700070e..cc7a36b 100755 --- a/crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl +++ b/crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl
@@ -25,6 +25,27 @@ # 256 Bit Primes" # Further optimization by <appro@openssl.org>: +# +# this/original with/without -DECP_NISTZ256_ASM(*) +# Opteron +15-49% +150-195% +# Bulldozer +18-45% +175-240% +# P4 +24-46% +100-150% +# Westmere +18-34% +87-160% +# Sandy Bridge +14-35% +120-185% +# Ivy Bridge +11-35% +125-180% +# Haswell +10-37% +160-200% +# Broadwell +24-58% +210-270% +# Atom +20-50% +180-240% +# VIA Nano +50-160% +480-480% +# +# (*) "without -DECP_NISTZ256_ASM" refers to build with +# "enable-ec_nistp_64_gcc_128"; +# +# Ranges denote minimum and maximum improvement coefficients depending +# on benchmark. In "this/original" column lower coefficient is for +# ECDSA sign, while in "with/without" - for ECDH key agreement, and +# higher - for ECDSA sign, relatively fastest server-side operation. +# Keep in mind that +100% means 2x improvement. $flavour = shift; $output = shift; @@ -46,8 +67,23 @@ $code.=<<___; .text -# Constants for computations modulo ord(p256) +# The polynomial +.section .rodata +.align 64 p256_constants: +.Lpoly: +.quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 + +.LOne: +.long 1,1,1,1,1,1,1,1 +.LTwo: +.long 2,2,2,2,2,2,2,2 +.LThree: +.long 3,3,3,3,3,3,3,3 +.LONE_mont: +.quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe + +# Constants for computations modulo ord(p256) .Lord: .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 .LordK: @@ -55,7 +91,71 @@ .text ___ +{ +my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11)); +my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13"); +my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx"); +$code.=<<___; + +################################################################################ +# void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]); +.globl ecp_nistz256_neg +.type ecp_nistz256_neg,\@function,2 +.align 32 +ecp_nistz256_neg: +.cfi_startproc + _CET_ENDBR + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 +.Lneg_body: + + xor $a0, $a0 + xor $a1, $a1 + xor $a2, $a2 + xor $a3, $a3 + xor $t4, $t4 + + sub 8*0($a_ptr), $a0 + sbb 8*1($a_ptr), $a1 + sbb 8*2($a_ptr), $a2 + mov $a0, $t0 + sbb 8*3($a_ptr), $a3 + lea .Lpoly(%rip), $a_ptr + mov $a1, $t1 + sbb \$0, $t4 + + add 8*0($a_ptr), $a0 + mov $a2, $t2 + adc 8*1($a_ptr), $a1 + adc 8*2($a_ptr), $a2 + mov $a3, $t3 + adc 8*3($a_ptr), $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + mov 0(%rsp),%r13 +.cfi_restore %r13 + mov 8(%rsp),%r12 +.cfi_restore %r12 + lea 16(%rsp),%rsp +.cfi_adjust_cfa_offset -16 +.Lneg_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_neg,.-ecp_nistz256_neg +___ +} { my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx"); my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15)); @@ -75,8 +175,6 @@ ecp_nistz256_ord_mul_mont_nohw: .cfi_startproc _CET_ENDBR -___ -$code.=<<___; push %rbp .cfi_push %rbp push %rbx @@ -400,8 +498,6 @@ ecp_nistz256_ord_sqr_mont_nohw: .cfi_startproc _CET_ENDBR -___ -$code.=<<___; push %rbp .cfi_push %rbp push %rbx @@ -1135,7 +1231,2574 @@ .size ecp_nistz256_ord_sqr_mont_adx,.-ecp_nistz256_ord_sqr_mont_adx ___ +$code.=<<___; +################################################################################ +# void ecp_nistz256_mul_mont( +# uint64_t res[4], +# uint64_t a[4], +# uint64_t b[4]); + +.globl ecp_nistz256_mul_mont_nohw +.type ecp_nistz256_mul_mont_nohw,\@function,3 +.align 32 +ecp_nistz256_mul_mont_nohw: +.cfi_startproc + _CET_ENDBR + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lmul_body: + mov $b_org, $b_ptr + mov 8*0($b_org), %rax + mov 8*0($a_ptr), $acc1 + mov 8*1($a_ptr), $acc2 + mov 8*2($a_ptr), $acc3 + mov 8*3($a_ptr), $acc4 + + call __ecp_nistz256_mul_montq + + mov 0(%rsp),%r15 +.cfi_restore %r15 + mov 8(%rsp),%r14 +.cfi_restore %r14 + mov 16(%rsp),%r13 +.cfi_restore %r13 + mov 24(%rsp),%r12 +.cfi_restore %r12 + mov 32(%rsp),%rbx +.cfi_restore %rbx + mov 40(%rsp),%rbp +.cfi_restore %rbp + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lmul_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_mul_mont_nohw,.-ecp_nistz256_mul_mont_nohw + +.type __ecp_nistz256_mul_montq,\@abi-omnipotent +.align 32 +__ecp_nistz256_mul_montq: +.cfi_startproc + ######################################################################## + # Multiply a by b[0] + mov %rax, $t1 + mulq $acc1 + mov .Lpoly+8*1(%rip),$poly1 + mov %rax, $acc0 + mov $t1, %rax + mov %rdx, $acc1 + + mulq $acc2 + mov .Lpoly+8*3(%rip),$poly3 + add %rax, $acc1 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $acc2 + + mulq $acc3 + add %rax, $acc2 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $acc3 + + mulq $acc4 + add %rax, $acc3 + mov $acc0, %rax + adc \$0, %rdx + xor $acc5, $acc5 + mov %rdx, $acc4 + + ######################################################################## + # First reduction step + # Basically now we want to multiply acc[0] by p256, + # and add the result to the acc. + # Due to the special form of p256 we do some optimizations + # + # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0] + # then we add acc[0] and get acc[0] x 2^96 + + mov $acc0, $t1 + shl \$32, $acc0 + mulq $poly3 + shr \$32, $t1 + add $acc0, $acc1 # +=acc[0]<<96 + adc $t1, $acc2 + adc %rax, $acc3 + mov 8*1($b_ptr), %rax + adc %rdx, $acc4 + adc \$0, $acc5 + xor $acc0, $acc0 + + ######################################################################## + # Multiply by b[1] + mov %rax, $t1 + mulq 8*0($a_ptr) + add %rax, $acc1 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*1($a_ptr) + add $t0, $acc2 + adc \$0, %rdx + add %rax, $acc2 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*2($a_ptr) + add $t0, $acc3 + adc \$0, %rdx + add %rax, $acc3 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*3($a_ptr) + add $t0, $acc4 + adc \$0, %rdx + add %rax, $acc4 + mov $acc1, %rax + adc %rdx, $acc5 + adc \$0, $acc0 + + ######################################################################## + # Second reduction step + mov $acc1, $t1 + shl \$32, $acc1 + mulq $poly3 + shr \$32, $t1 + add $acc1, $acc2 + adc $t1, $acc3 + adc %rax, $acc4 + mov 8*2($b_ptr), %rax + adc %rdx, $acc5 + adc \$0, $acc0 + xor $acc1, $acc1 + + ######################################################################## + # Multiply by b[2] + mov %rax, $t1 + mulq 8*0($a_ptr) + add %rax, $acc2 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*1($a_ptr) + add $t0, $acc3 + adc \$0, %rdx + add %rax, $acc3 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*2($a_ptr) + add $t0, $acc4 + adc \$0, %rdx + add %rax, $acc4 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*3($a_ptr) + add $t0, $acc5 + adc \$0, %rdx + add %rax, $acc5 + mov $acc2, %rax + adc %rdx, $acc0 + adc \$0, $acc1 + + ######################################################################## + # Third reduction step + mov $acc2, $t1 + shl \$32, $acc2 + mulq $poly3 + shr \$32, $t1 + add $acc2, $acc3 + adc $t1, $acc4 + adc %rax, $acc5 + mov 8*3($b_ptr), %rax + adc %rdx, $acc0 + adc \$0, $acc1 + xor $acc2, $acc2 + + ######################################################################## + # Multiply by b[3] + mov %rax, $t1 + mulq 8*0($a_ptr) + add %rax, $acc3 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*1($a_ptr) + add $t0, $acc4 + adc \$0, %rdx + add %rax, $acc4 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*2($a_ptr) + add $t0, $acc5 + adc \$0, %rdx + add %rax, $acc5 + mov $t1, %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq 8*3($a_ptr) + add $t0, $acc0 + adc \$0, %rdx + add %rax, $acc0 + mov $acc3, %rax + adc %rdx, $acc1 + adc \$0, $acc2 + + ######################################################################## + # Final reduction step + mov $acc3, $t1 + shl \$32, $acc3 + mulq $poly3 + shr \$32, $t1 + add $acc3, $acc4 + adc $t1, $acc5 + mov $acc4, $t0 + adc %rax, $acc0 + adc %rdx, $acc1 + mov $acc5, $t1 + adc \$0, $acc2 + + ######################################################################## + # Branch-less conditional subtraction of P + sub \$-1, $acc4 # .Lpoly[0] + mov $acc0, $t2 + sbb $poly1, $acc5 # .Lpoly[1] + sbb \$0, $acc0 # .Lpoly[2] + mov $acc1, $t3 + sbb $poly3, $acc1 # .Lpoly[3] + sbb \$0, $acc2 + + cmovc $t0, $acc4 + cmovc $t1, $acc5 + mov $acc4, 8*0($r_ptr) + cmovc $t2, $acc0 + mov $acc5, 8*1($r_ptr) + cmovc $t3, $acc1 + mov $acc0, 8*2($r_ptr) + mov $acc1, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq + +################################################################################ +# void ecp_nistz256_sqr_mont( +# uint64_t res[4], +# uint64_t a[4]); + +# we optimize the square according to S.Gueron and V.Krasnov, +# "Speeding up Big-Number Squaring" +.globl ecp_nistz256_sqr_mont_nohw +.type ecp_nistz256_sqr_mont_nohw,\@function,2 +.align 32 +ecp_nistz256_sqr_mont_nohw: +.cfi_startproc + _CET_ENDBR + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lsqr_body: + mov 8*0($a_ptr), %rax + mov 8*1($a_ptr), $acc6 + mov 8*2($a_ptr), $acc7 + mov 8*3($a_ptr), $acc0 + + call __ecp_nistz256_sqr_montq + + mov 0(%rsp),%r15 +.cfi_restore %r15 + mov 8(%rsp),%r14 +.cfi_restore %r14 + mov 16(%rsp),%r13 +.cfi_restore %r13 + mov 24(%rsp),%r12 +.cfi_restore %r12 + mov 32(%rsp),%rbx +.cfi_restore %rbx + mov 40(%rsp),%rbp +.cfi_restore %rbp + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lsqr_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_sqr_mont_nohw,.-ecp_nistz256_sqr_mont_nohw + +.type __ecp_nistz256_sqr_montq,\@abi-omnipotent +.align 32 +__ecp_nistz256_sqr_montq: +.cfi_startproc + mov %rax, $acc5 + mulq $acc6 # a[1]*a[0] + mov %rax, $acc1 + mov $acc7, %rax + mov %rdx, $acc2 + + mulq $acc5 # a[0]*a[2] + add %rax, $acc2 + mov $acc0, %rax + adc \$0, %rdx + mov %rdx, $acc3 + + mulq $acc5 # a[0]*a[3] + add %rax, $acc3 + mov $acc7, %rax + adc \$0, %rdx + mov %rdx, $acc4 + + ################################# + mulq $acc6 # a[1]*a[2] + add %rax, $acc3 + mov $acc0, %rax + adc \$0, %rdx + mov %rdx, $t1 + + mulq $acc6 # a[1]*a[3] + add %rax, $acc4 + mov $acc0, %rax + adc \$0, %rdx + add $t1, $acc4 + mov %rdx, $acc5 + adc \$0, $acc5 + + ################################# + mulq $acc7 # a[2]*a[3] + xor $acc7, $acc7 + add %rax, $acc5 + mov 8*0($a_ptr), %rax + mov %rdx, $acc6 + adc \$0, $acc6 + + add $acc1, $acc1 # acc1:6<<1 + adc $acc2, $acc2 + adc $acc3, $acc3 + adc $acc4, $acc4 + adc $acc5, $acc5 + adc $acc6, $acc6 + adc \$0, $acc7 + + mulq %rax + mov %rax, $acc0 + mov 8*1($a_ptr), %rax + mov %rdx, $t0 + + mulq %rax + add $t0, $acc1 + adc %rax, $acc2 + mov 8*2($a_ptr), %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq %rax + add $t0, $acc3 + adc %rax, $acc4 + mov 8*3($a_ptr), %rax + adc \$0, %rdx + mov %rdx, $t0 + + mulq %rax + add $t0, $acc5 + adc %rax, $acc6 + mov $acc0, %rax + adc %rdx, $acc7 + + mov .Lpoly+8*1(%rip), $a_ptr + mov .Lpoly+8*3(%rip), $t1 + + ########################################## + # Now the reduction + # First iteration + mov $acc0, $t0 + shl \$32, $acc0 + mulq $t1 + shr \$32, $t0 + add $acc0, $acc1 # +=acc[0]<<96 + adc $t0, $acc2 + adc %rax, $acc3 + mov $acc1, %rax + adc \$0, %rdx + + ########################################## + # Second iteration + mov $acc1, $t0 + shl \$32, $acc1 + mov %rdx, $acc0 + mulq $t1 + shr \$32, $t0 + add $acc1, $acc2 + adc $t0, $acc3 + adc %rax, $acc0 + mov $acc2, %rax + adc \$0, %rdx + + ########################################## + # Third iteration + mov $acc2, $t0 + shl \$32, $acc2 + mov %rdx, $acc1 + mulq $t1 + shr \$32, $t0 + add $acc2, $acc3 + adc $t0, $acc0 + adc %rax, $acc1 + mov $acc3, %rax + adc \$0, %rdx + + ########################################### + # Last iteration + mov $acc3, $t0 + shl \$32, $acc3 + mov %rdx, $acc2 + mulq $t1 + shr \$32, $t0 + add $acc3, $acc0 + adc $t0, $acc1 + adc %rax, $acc2 + adc \$0, %rdx + xor $acc3, $acc3 + + ############################################ + # Add the rest of the acc + add $acc0, $acc4 + adc $acc1, $acc5 + mov $acc4, $acc0 + adc $acc2, $acc6 + adc %rdx, $acc7 + mov $acc5, $acc1 + adc \$0, $acc3 + + sub \$-1, $acc4 # .Lpoly[0] + mov $acc6, $acc2 + sbb $a_ptr, $acc5 # .Lpoly[1] + sbb \$0, $acc6 # .Lpoly[2] + mov $acc7, $t0 + sbb $t1, $acc7 # .Lpoly[3] + sbb \$0, $acc3 + + cmovc $acc0, $acc4 + cmovc $acc1, $acc5 + mov $acc4, 8*0($r_ptr) + cmovc $acc2, $acc6 + mov $acc5, 8*1($r_ptr) + cmovc $t0, $acc7 + mov $acc6, 8*2($r_ptr) + mov $acc7, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq +___ + +if ($addx) { +$code.=<<___; +.globl ecp_nistz256_mul_mont_adx +.type ecp_nistz256_mul_mont_adx,\@function,3 +.align 32 +ecp_nistz256_mul_mont_adx: +.cfi_startproc + _CET_ENDBR + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lmulx_body: + mov $b_org, $b_ptr + mov 8*0($b_org), %rdx + mov 8*0($a_ptr), $acc1 + mov 8*1($a_ptr), $acc2 + mov 8*2($a_ptr), $acc3 + mov 8*3($a_ptr), $acc4 + lea -128($a_ptr), $a_ptr # control u-op density + + call __ecp_nistz256_mul_montx + + mov 0(%rsp),%r15 +.cfi_restore %r15 + mov 8(%rsp),%r14 +.cfi_restore %r14 + mov 16(%rsp),%r13 +.cfi_restore %r13 + mov 24(%rsp),%r12 +.cfi_restore %r12 + mov 32(%rsp),%rbx +.cfi_restore %rbx + mov 40(%rsp),%rbp +.cfi_restore %rbp + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lmulx_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_mul_mont_adx,.-ecp_nistz256_mul_mont_adx + +.type __ecp_nistz256_mul_montx,\@abi-omnipotent +.align 32 +__ecp_nistz256_mul_montx: +.cfi_startproc + ######################################################################## + # Multiply by b[0] + mulx $acc1, $acc0, $acc1 + mulx $acc2, $t0, $acc2 + mov \$32, $poly1 + xor $acc5, $acc5 # cf=0 + mulx $acc3, $t1, $acc3 + mov .Lpoly+8*3(%rip), $poly3 + adc $t0, $acc1 + mulx $acc4, $t0, $acc4 + mov $acc0, %rdx + adc $t1, $acc2 + shlx $poly1,$acc0,$t1 + adc $t0, $acc3 + shrx $poly1,$acc0,$t0 + adc \$0, $acc4 + + ######################################################################## + # First reduction step + add $t1, $acc1 + adc $t0, $acc2 + + mulx $poly3, $t0, $t1 + mov 8*1($b_ptr), %rdx + adc $t0, $acc3 + adc $t1, $acc4 + adc \$0, $acc5 + xor $acc0, $acc0 # $acc0=0,cf=0,of=0 + + ######################################################################## + # Multiply by b[1] + mulx 8*0+128($a_ptr), $t0, $t1 + adcx $t0, $acc1 + adox $t1, $acc2 + + mulx 8*1+128($a_ptr), $t0, $t1 + adcx $t0, $acc2 + adox $t1, $acc3 + + mulx 8*2+128($a_ptr), $t0, $t1 + adcx $t0, $acc3 + adox $t1, $acc4 + + mulx 8*3+128($a_ptr), $t0, $t1 + mov $acc1, %rdx + adcx $t0, $acc4 + shlx $poly1, $acc1, $t0 + adox $t1, $acc5 + shrx $poly1, $acc1, $t1 + + adcx $acc0, $acc5 + adox $acc0, $acc0 + adc \$0, $acc0 + + ######################################################################## + # Second reduction step + add $t0, $acc2 + adc $t1, $acc3 + + mulx $poly3, $t0, $t1 + mov 8*2($b_ptr), %rdx + adc $t0, $acc4 + adc $t1, $acc5 + adc \$0, $acc0 + xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0 + + ######################################################################## + # Multiply by b[2] + mulx 8*0+128($a_ptr), $t0, $t1 + adcx $t0, $acc2 + adox $t1, $acc3 + + mulx 8*1+128($a_ptr), $t0, $t1 + adcx $t0, $acc3 + adox $t1, $acc4 + + mulx 8*2+128($a_ptr), $t0, $t1 + adcx $t0, $acc4 + adox $t1, $acc5 + + mulx 8*3+128($a_ptr), $t0, $t1 + mov $acc2, %rdx + adcx $t0, $acc5 + shlx $poly1, $acc2, $t0 + adox $t1, $acc0 + shrx $poly1, $acc2, $t1 + + adcx $acc1, $acc0 + adox $acc1, $acc1 + adc \$0, $acc1 + + ######################################################################## + # Third reduction step + add $t0, $acc3 + adc $t1, $acc4 + + mulx $poly3, $t0, $t1 + mov 8*3($b_ptr), %rdx + adc $t0, $acc5 + adc $t1, $acc0 + adc \$0, $acc1 + xor $acc2, $acc2 # $acc2=0,cf=0,of=0 + + ######################################################################## + # Multiply by b[3] + mulx 8*0+128($a_ptr), $t0, $t1 + adcx $t0, $acc3 + adox $t1, $acc4 + + mulx 8*1+128($a_ptr), $t0, $t1 + adcx $t0, $acc4 + adox $t1, $acc5 + + mulx 8*2+128($a_ptr), $t0, $t1 + adcx $t0, $acc5 + adox $t1, $acc0 + + mulx 8*3+128($a_ptr), $t0, $t1 + mov $acc3, %rdx + adcx $t0, $acc0 + shlx $poly1, $acc3, $t0 + adox $t1, $acc1 + shrx $poly1, $acc3, $t1 + + adcx $acc2, $acc1 + adox $acc2, $acc2 + adc \$0, $acc2 + + ######################################################################## + # Fourth reduction step + add $t0, $acc4 + adc $t1, $acc5 + + mulx $poly3, $t0, $t1 + mov $acc4, $t2 + mov .Lpoly+8*1(%rip), $poly1 + adc $t0, $acc0 + mov $acc5, $t3 + adc $t1, $acc1 + adc \$0, $acc2 + + ######################################################################## + # Branch-less conditional subtraction of P + xor %eax, %eax + mov $acc0, $t0 + sbb \$-1, $acc4 # .Lpoly[0] + sbb $poly1, $acc5 # .Lpoly[1] + sbb \$0, $acc0 # .Lpoly[2] + mov $acc1, $t1 + sbb $poly3, $acc1 # .Lpoly[3] + sbb \$0, $acc2 + + cmovc $t2, $acc4 + cmovc $t3, $acc5 + mov $acc4, 8*0($r_ptr) + cmovc $t0, $acc0 + mov $acc5, 8*1($r_ptr) + cmovc $t1, $acc1 + mov $acc0, 8*2($r_ptr) + mov $acc1, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx + +.globl ecp_nistz256_sqr_mont_adx +.type ecp_nistz256_sqr_mont_adx,\@function,2 +.align 32 +ecp_nistz256_sqr_mont_adx: +.cfi_startproc + _CET_ENDBR + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 +.Lsqrx_body: + mov 8*0($a_ptr), %rdx + mov 8*1($a_ptr), $acc6 + mov 8*2($a_ptr), $acc7 + mov 8*3($a_ptr), $acc0 + lea -128($a_ptr), $a_ptr # control u-op density + + call __ecp_nistz256_sqr_montx + + mov 0(%rsp),%r15 +.cfi_restore %r15 + mov 8(%rsp),%r14 +.cfi_restore %r14 + mov 16(%rsp),%r13 +.cfi_restore %r13 + mov 24(%rsp),%r12 +.cfi_restore %r12 + mov 32(%rsp),%rbx +.cfi_restore %rbx + mov 40(%rsp),%rbp +.cfi_restore %rbp + lea 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lsqrx_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_sqr_mont_adx,.-ecp_nistz256_sqr_mont_adx + +.type __ecp_nistz256_sqr_montx,\@abi-omnipotent +.align 32 +__ecp_nistz256_sqr_montx: +.cfi_startproc + mulx $acc6, $acc1, $acc2 # a[0]*a[1] + mulx $acc7, $t0, $acc3 # a[0]*a[2] + xor %eax, %eax + adc $t0, $acc2 + mulx $acc0, $t1, $acc4 # a[0]*a[3] + mov $acc6, %rdx + adc $t1, $acc3 + adc \$0, $acc4 + xor $acc5, $acc5 # $acc5=0,cf=0,of=0 + + ################################# + mulx $acc7, $t0, $t1 # a[1]*a[2] + adcx $t0, $acc3 + adox $t1, $acc4 + + mulx $acc0, $t0, $t1 # a[1]*a[3] + mov $acc7, %rdx + adcx $t0, $acc4 + adox $t1, $acc5 + adc \$0, $acc5 + + ################################# + mulx $acc0, $t0, $acc6 # a[2]*a[3] + mov 8*0+128($a_ptr), %rdx + xor $acc7, $acc7 # $acc7=0,cf=0,of=0 + adcx $acc1, $acc1 # acc1:6<<1 + adox $t0, $acc5 + adcx $acc2, $acc2 + adox $acc7, $acc6 # of=0 + + mulx %rdx, $acc0, $t1 + mov 8*1+128($a_ptr), %rdx + adcx $acc3, $acc3 + adox $t1, $acc1 + adcx $acc4, $acc4 + mulx %rdx, $t0, $t4 + mov 8*2+128($a_ptr), %rdx + adcx $acc5, $acc5 + adox $t0, $acc2 + adcx $acc6, $acc6 + .byte 0x67 + mulx %rdx, $t0, $t1 + mov 8*3+128($a_ptr), %rdx + adox $t4, $acc3 + adcx $acc7, $acc7 + adox $t0, $acc4 + mov \$32, $a_ptr + adox $t1, $acc5 + .byte 0x67,0x67 + mulx %rdx, $t0, $t4 + mov .Lpoly+8*3(%rip), %rdx + adox $t0, $acc6 + shlx $a_ptr, $acc0, $t0 + adox $t4, $acc7 + shrx $a_ptr, $acc0, $t4 + mov %rdx,$t1 + + # reduction step 1 + add $t0, $acc1 + adc $t4, $acc2 + + mulx $acc0, $t0, $acc0 + adc $t0, $acc3 + shlx $a_ptr, $acc1, $t0 + adc \$0, $acc0 + shrx $a_ptr, $acc1, $t4 + + # reduction step 2 + add $t0, $acc2 + adc $t4, $acc3 + + mulx $acc1, $t0, $acc1 + adc $t0, $acc0 + shlx $a_ptr, $acc2, $t0 + adc \$0, $acc1 + shrx $a_ptr, $acc2, $t4 + + # reduction step 3 + add $t0, $acc3 + adc $t4, $acc0 + + mulx $acc2, $t0, $acc2 + adc $t0, $acc1 + shlx $a_ptr, $acc3, $t0 + adc \$0, $acc2 + shrx $a_ptr, $acc3, $t4 + + # reduction step 4 + add $t0, $acc0 + adc $t4, $acc1 + + mulx $acc3, $t0, $acc3 + adc $t0, $acc2 + adc \$0, $acc3 + + xor $t3, $t3 + add $acc0, $acc4 # accumulate upper half + mov .Lpoly+8*1(%rip), $a_ptr + adc $acc1, $acc5 + mov $acc4, $acc0 + adc $acc2, $acc6 + adc $acc3, $acc7 + mov $acc5, $acc1 + adc \$0, $t3 + + sub \$-1, $acc4 # .Lpoly[0] + mov $acc6, $acc2 + sbb $a_ptr, $acc5 # .Lpoly[1] + sbb \$0, $acc6 # .Lpoly[2] + mov $acc7, $acc3 + sbb $t1, $acc7 # .Lpoly[3] + sbb \$0, $t3 + + cmovc $acc0, $acc4 + cmovc $acc1, $acc5 + mov $acc4, 8*0($r_ptr) + cmovc $acc2, $acc6 + mov $acc5, 8*1($r_ptr) + cmovc $acc3, $acc7 + mov $acc6, 8*2($r_ptr) + mov $acc7, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx +___ } +} +{ +my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); +my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7)); +my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15)); +my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15)); + +$code.=<<___; +################################################################################ +# void ecp_nistz256_select_w5_nohw(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w5_nohw +.type ecp_nistz256_select_w5_nohw,\@abi-omnipotent +.align 32 +ecp_nistz256_select_w5_nohw: +.cfi_startproc + _CET_ENDBR +___ +$code.=<<___ if ($win64); + lea -0x88(%rsp), %rax +.LSEH_begin_ecp_nistz256_select_w5_nohw: + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp + .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax) + .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax) + .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax) + .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax) + .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax) + .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax) + .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax) + .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax) + .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax) + .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax) +___ +$code.=<<___; + movdqa .LOne(%rip), $ONE + movd $index, $INDEX + + pxor $Ra, $Ra + pxor $Rb, $Rb + pxor $Rc, $Rc + pxor $Rd, $Rd + pxor $Re, $Re + pxor $Rf, $Rf + + movdqa $ONE, $M0 + pshufd \$0, $INDEX, $INDEX + + mov \$16, %rax +.Lselect_loop_sse_w5: + + movdqa $M0, $TMP0 + paddd $ONE, $M0 + pcmpeqd $INDEX, $TMP0 + + movdqa 16*0($in_t), $T0a + movdqa 16*1($in_t), $T0b + movdqa 16*2($in_t), $T0c + movdqa 16*3($in_t), $T0d + movdqa 16*4($in_t), $T0e + movdqa 16*5($in_t), $T0f + lea 16*6($in_t), $in_t + + pand $TMP0, $T0a + pand $TMP0, $T0b + por $T0a, $Ra + pand $TMP0, $T0c + por $T0b, $Rb + pand $TMP0, $T0d + por $T0c, $Rc + pand $TMP0, $T0e + por $T0d, $Rd + pand $TMP0, $T0f + por $T0e, $Re + por $T0f, $Rf + + dec %rax + jnz .Lselect_loop_sse_w5 + + movdqu $Ra, 16*0($val) + movdqu $Rb, 16*1($val) + movdqu $Rc, 16*2($val) + movdqu $Rd, 16*3($val) + movdqu $Re, 16*4($val) + movdqu $Rf, 16*5($val) +___ +$code.=<<___ if ($win64); + movaps (%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + lea 0xa8(%rsp), %rsp +___ +$code.=<<___; + ret +.cfi_endproc +.LSEH_end_ecp_nistz256_select_w5_nohw: +.size ecp_nistz256_select_w5_nohw,.-ecp_nistz256_select_w5_nohw + +################################################################################ +# void ecp_nistz256_select_w7_nohw(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w7_nohw +.type ecp_nistz256_select_w7_nohw,\@abi-omnipotent +.align 32 +ecp_nistz256_select_w7_nohw: +.cfi_startproc + _CET_ENDBR +___ +$code.=<<___ if ($win64); + lea -0x88(%rsp), %rax +.LSEH_begin_ecp_nistz256_select_w7_nohw: + .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp + .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax) + .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax) + .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax) + .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax) + .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax) + .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax) + .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax) + .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax) + .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax) + .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax) +___ +$code.=<<___; + movdqa .LOne(%rip), $M0 + movd $index, $INDEX + + pxor $Ra, $Ra + pxor $Rb, $Rb + pxor $Rc, $Rc + pxor $Rd, $Rd + + movdqa $M0, $ONE + pshufd \$0, $INDEX, $INDEX + mov \$64, %rax + +.Lselect_loop_sse_w7: + movdqa $M0, $TMP0 + paddd $ONE, $M0 + movdqa 16*0($in_t), $T0a + movdqa 16*1($in_t), $T0b + pcmpeqd $INDEX, $TMP0 + movdqa 16*2($in_t), $T0c + movdqa 16*3($in_t), $T0d + lea 16*4($in_t), $in_t + + pand $TMP0, $T0a + pand $TMP0, $T0b + por $T0a, $Ra + pand $TMP0, $T0c + por $T0b, $Rb + pand $TMP0, $T0d + por $T0c, $Rc + prefetcht0 255($in_t) + por $T0d, $Rd + + dec %rax + jnz .Lselect_loop_sse_w7 + + movdqu $Ra, 16*0($val) + movdqu $Rb, 16*1($val) + movdqu $Rc, 16*2($val) + movdqu $Rd, 16*3($val) +___ +$code.=<<___ if ($win64); + movaps (%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + lea 0xa8(%rsp), %rsp +___ +$code.=<<___; + ret +.cfi_endproc +.LSEH_end_ecp_nistz256_select_w7_nohw: +.size ecp_nistz256_select_w7_nohw,.-ecp_nistz256_select_w7_nohw +___ +} +if ($avx>1) { +my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); +my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4)); +my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9)); +my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14)); + +$code.=<<___; +################################################################################ +# void ecp_nistz256_select_w5_avx2(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w5_avx2 +.type ecp_nistz256_select_w5_avx2,\@abi-omnipotent +.align 32 +ecp_nistz256_select_w5_avx2: +.cfi_startproc + _CET_ENDBR + vzeroupper +___ +$code.=<<___ if ($win64); + lea -0x88(%rsp), %rax + mov %rsp,%r11 +.LSEH_begin_ecp_nistz256_select_w5_avx2: + .byte 0x48,0x8d,0x60,0xe0 # lea -0x20(%rax), %rsp + .byte 0xc5,0xf8,0x29,0x70,0xe0 # vmovaps %xmm6, -0x20(%rax) + .byte 0xc5,0xf8,0x29,0x78,0xf0 # vmovaps %xmm7, -0x10(%rax) + .byte 0xc5,0x78,0x29,0x40,0x00 # vmovaps %xmm8, 8(%rax) + .byte 0xc5,0x78,0x29,0x48,0x10 # vmovaps %xmm9, 0x10(%rax) + .byte 0xc5,0x78,0x29,0x50,0x20 # vmovaps %xmm10, 0x20(%rax) + .byte 0xc5,0x78,0x29,0x58,0x30 # vmovaps %xmm11, 0x30(%rax) + .byte 0xc5,0x78,0x29,0x60,0x40 # vmovaps %xmm12, 0x40(%rax) + .byte 0xc5,0x78,0x29,0x68,0x50 # vmovaps %xmm13, 0x50(%rax) + .byte 0xc5,0x78,0x29,0x70,0x60 # vmovaps %xmm14, 0x60(%rax) + .byte 0xc5,0x78,0x29,0x78,0x70 # vmovaps %xmm15, 0x70(%rax) +___ +$code.=<<___; + vmovdqa .LTwo(%rip), $TWO + + vpxor $Ra, $Ra, $Ra + vpxor $Rb, $Rb, $Rb + vpxor $Rc, $Rc, $Rc + + vmovdqa .LOne(%rip), $M0 + vmovdqa .LTwo(%rip), $M1 + + vmovd $index, %xmm1 + vpermd $INDEX, $Ra, $INDEX + + mov \$8, %rax +.Lselect_loop_avx2_w5: + + vmovdqa 32*0($in_t), $T0a + vmovdqa 32*1($in_t), $T0b + vmovdqa 32*2($in_t), $T0c + + vmovdqa 32*3($in_t), $T1a + vmovdqa 32*4($in_t), $T1b + vmovdqa 32*5($in_t), $T1c + + vpcmpeqd $INDEX, $M0, $TMP0 + vpcmpeqd $INDEX, $M1, $TMP1 + + vpaddd $TWO, $M0, $M0 + vpaddd $TWO, $M1, $M1 + lea 32*6($in_t), $in_t + + vpand $TMP0, $T0a, $T0a + vpand $TMP0, $T0b, $T0b + vpand $TMP0, $T0c, $T0c + vpand $TMP1, $T1a, $T1a + vpand $TMP1, $T1b, $T1b + vpand $TMP1, $T1c, $T1c + + vpxor $T0a, $Ra, $Ra + vpxor $T0b, $Rb, $Rb + vpxor $T0c, $Rc, $Rc + vpxor $T1a, $Ra, $Ra + vpxor $T1b, $Rb, $Rb + vpxor $T1c, $Rc, $Rc + + dec %rax + jnz .Lselect_loop_avx2_w5 + + vmovdqu $Ra, 32*0($val) + vmovdqu $Rb, 32*1($val) + vmovdqu $Rc, 32*2($val) + vzeroupper +___ +$code.=<<___ if ($win64); + movaps (%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + lea (%r11), %rsp +___ +$code.=<<___; + ret +.cfi_endproc +.LSEH_end_ecp_nistz256_select_w5_avx2: +.size ecp_nistz256_select_w5_avx2,.-ecp_nistz256_select_w5_avx2 +___ +} +if ($avx>1) { +my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); +my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3)); +my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7)); +my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11)); +my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15)); + +$code.=<<___; + +################################################################################ +# void ecp_nistz256_select_w7_avx2(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w7_avx2 +.type ecp_nistz256_select_w7_avx2,\@abi-omnipotent +.align 32 +ecp_nistz256_select_w7_avx2: +.cfi_startproc + _CET_ENDBR + vzeroupper +___ +$code.=<<___ if ($win64); + mov %rsp,%r11 + lea -0x88(%rsp), %rax +.LSEH_begin_ecp_nistz256_select_w7_avx2: + .byte 0x48,0x8d,0x60,0xe0 # lea -0x20(%rax), %rsp + .byte 0xc5,0xf8,0x29,0x70,0xe0 # vmovaps %xmm6, -0x20(%rax) + .byte 0xc5,0xf8,0x29,0x78,0xf0 # vmovaps %xmm7, -0x10(%rax) + .byte 0xc5,0x78,0x29,0x40,0x00 # vmovaps %xmm8, 8(%rax) + .byte 0xc5,0x78,0x29,0x48,0x10 # vmovaps %xmm9, 0x10(%rax) + .byte 0xc5,0x78,0x29,0x50,0x20 # vmovaps %xmm10, 0x20(%rax) + .byte 0xc5,0x78,0x29,0x58,0x30 # vmovaps %xmm11, 0x30(%rax) + .byte 0xc5,0x78,0x29,0x60,0x40 # vmovaps %xmm12, 0x40(%rax) + .byte 0xc5,0x78,0x29,0x68,0x50 # vmovaps %xmm13, 0x50(%rax) + .byte 0xc5,0x78,0x29,0x70,0x60 # vmovaps %xmm14, 0x60(%rax) + .byte 0xc5,0x78,0x29,0x78,0x70 # vmovaps %xmm15, 0x70(%rax) +___ +$code.=<<___; + vmovdqa .LThree(%rip), $THREE + + vpxor $Ra, $Ra, $Ra + vpxor $Rb, $Rb, $Rb + + vmovdqa .LOne(%rip), $M0 + vmovdqa .LTwo(%rip), $M1 + vmovdqa .LThree(%rip), $M2 + + vmovd $index, %xmm1 + vpermd $INDEX, $Ra, $INDEX + # Skip index = 0, because it is implicitly the point at infinity + + mov \$21, %rax +.Lselect_loop_avx2_w7: + + vmovdqa 32*0($in_t), $T0a + vmovdqa 32*1($in_t), $T0b + + vmovdqa 32*2($in_t), $T1a + vmovdqa 32*3($in_t), $T1b + + vmovdqa 32*4($in_t), $T2a + vmovdqa 32*5($in_t), $T2b + + vpcmpeqd $INDEX, $M0, $TMP0 + vpcmpeqd $INDEX, $M1, $TMP1 + vpcmpeqd $INDEX, $M2, $TMP2 + + vpaddd $THREE, $M0, $M0 + vpaddd $THREE, $M1, $M1 + vpaddd $THREE, $M2, $M2 + lea 32*6($in_t), $in_t + + vpand $TMP0, $T0a, $T0a + vpand $TMP0, $T0b, $T0b + vpand $TMP1, $T1a, $T1a + vpand $TMP1, $T1b, $T1b + vpand $TMP2, $T2a, $T2a + vpand $TMP2, $T2b, $T2b + + vpxor $T0a, $Ra, $Ra + vpxor $T0b, $Rb, $Rb + vpxor $T1a, $Ra, $Ra + vpxor $T1b, $Rb, $Rb + vpxor $T2a, $Ra, $Ra + vpxor $T2b, $Rb, $Rb + + dec %rax + jnz .Lselect_loop_avx2_w7 + + + vmovdqa 32*0($in_t), $T0a + vmovdqa 32*1($in_t), $T0b + + vpcmpeqd $INDEX, $M0, $TMP0 + + vpand $TMP0, $T0a, $T0a + vpand $TMP0, $T0b, $T0b + + vpxor $T0a, $Ra, $Ra + vpxor $T0b, $Rb, $Rb + + vmovdqu $Ra, 32*0($val) + vmovdqu $Rb, 32*1($val) + vzeroupper +___ +$code.=<<___ if ($win64); + movaps (%rsp), %xmm6 + movaps 0x10(%rsp), %xmm7 + movaps 0x20(%rsp), %xmm8 + movaps 0x30(%rsp), %xmm9 + movaps 0x40(%rsp), %xmm10 + movaps 0x50(%rsp), %xmm11 + movaps 0x60(%rsp), %xmm12 + movaps 0x70(%rsp), %xmm13 + movaps 0x80(%rsp), %xmm14 + movaps 0x90(%rsp), %xmm15 + lea (%r11), %rsp +___ +$code.=<<___; + ret +.cfi_endproc +.LSEH_end_ecp_nistz256_select_w7_avx2: +.size ecp_nistz256_select_w7_avx2,.-ecp_nistz256_select_w7_avx2 +___ +} else { +$code.=<<___; +.globl ecp_nistz256_select_w7_avx2 +.type ecp_nistz256_select_w7_avx2,\@function,3 +.align 32 +ecp_nistz256_select_w7_avx2: + _CET_ENDBR + .byte 0x0f,0x0b # ud2 + ret +.size ecp_nistz256_select_w7_avx2,.-ecp_nistz256_select_w7_avx2 +___ +} +{{{ +######################################################################## +# This block implements higher level point_double, point_add and +# point_add_affine. The key to performance in this case is to allow +# out-of-order execution logic to overlap computations from next step +# with tail processing from current step. By using tailored calling +# sequence we minimize inter-step overhead to give processor better +# shot at overlapping operations... +# +# You will notice that input data is copied to stack. Trouble is that +# there are no registers to spare for holding original pointers and +# reloading them, pointers, would create undesired dependencies on +# effective addresses calculation paths. In other words it's too done +# to favour out-of-order execution logic. +# <appro@openssl.org> + +my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx"); +my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15)); +my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4); +my ($poly1,$poly3)=($acc6,$acc7); + +sub load_for_mul () { +my ($a,$b,$src0) = @_; +my $bias = $src0 eq "%rax" ? 0 : -128; + +" mov $b, $src0 + lea $b, $b_ptr + mov 8*0+$a, $acc1 + mov 8*1+$a, $acc2 + lea $bias+$a, $a_ptr + mov 8*2+$a, $acc3 + mov 8*3+$a, $acc4" +} + +sub load_for_sqr () { +my ($a,$src0) = @_; +my $bias = $src0 eq "%rax" ? 0 : -128; + +" mov 8*0+$a, $src0 + mov 8*1+$a, $acc6 + lea $bias+$a, $a_ptr + mov 8*2+$a, $acc7 + mov 8*3+$a, $acc0" +} + + { +######################################################################## +# operate in 4-5-0-1 "name space" that matches multiplication output +# +my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); + +$code.=<<___; +.type __ecp_nistz256_add_toq,\@abi-omnipotent +.align 32 +__ecp_nistz256_add_toq: +.cfi_startproc + xor $t4,$t4 + add 8*0($b_ptr), $a0 + adc 8*1($b_ptr), $a1 + mov $a0, $t0 + adc 8*2($b_ptr), $a2 + adc 8*3($b_ptr), $a3 + mov $a1, $t1 + adc \$0, $t4 + + sub \$-1, $a0 + mov $a2, $t2 + sbb $poly1, $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb $poly3, $a3 + sbb \$0, $t4 + + cmovc $t0, $a0 + cmovc $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovc $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovc $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq + +.type __ecp_nistz256_sub_fromq,\@abi-omnipotent +.align 32 +__ecp_nistz256_sub_fromq: +.cfi_startproc + sub 8*0($b_ptr), $a0 + sbb 8*1($b_ptr), $a1 + mov $a0, $t0 + sbb 8*2($b_ptr), $a2 + sbb 8*3($b_ptr), $a3 + mov $a1, $t1 + sbb $t4, $t4 + + add \$-1, $a0 + mov $a2, $t2 + adc $poly1, $a1 + adc \$0, $a2 + mov $a3, $t3 + adc $poly3, $a3 + test $t4, $t4 + + cmovz $t0, $a0 + cmovz $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovz $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovz $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq + +.type __ecp_nistz256_subq,\@abi-omnipotent +.align 32 +__ecp_nistz256_subq: +.cfi_startproc + sub $a0, $t0 + sbb $a1, $t1 + mov $t0, $a0 + sbb $a2, $t2 + sbb $a3, $t3 + mov $t1, $a1 + sbb $t4, $t4 + + add \$-1, $t0 + mov $t2, $a2 + adc $poly1, $t1 + adc \$0, $t2 + mov $t3, $a3 + adc $poly3, $t3 + test $t4, $t4 + + cmovnz $t0, $a0 + cmovnz $t1, $a1 + cmovnz $t2, $a2 + cmovnz $t3, $a3 + + ret +.cfi_endproc +.size __ecp_nistz256_subq,.-__ecp_nistz256_subq + +.type __ecp_nistz256_mul_by_2q,\@abi-omnipotent +.align 32 +__ecp_nistz256_mul_by_2q: +.cfi_startproc + xor $t4, $t4 + add $a0, $a0 # a0:a3+a0:a3 + adc $a1, $a1 + mov $a0, $t0 + adc $a2, $a2 + adc $a3, $a3 + mov $a1, $t1 + adc \$0, $t4 + + sub \$-1, $a0 + mov $a2, $t2 + sbb $poly1, $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb $poly3, $a3 + sbb \$0, $t4 + + cmovc $t0, $a0 + cmovc $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovc $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovc $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q +___ + } +sub gen_double () { + my $x = shift; + my ($src0,$sfx,$bias); + my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4)); + + if ($x ne "x") { + $src0 = "%rax"; + $sfx = "_nohw"; + $bias = 0; + } else { + $src0 = "%rdx"; + $sfx = "_adx"; + $bias = 128; + } +$code.=<<___; +.globl ecp_nistz256_point_double$sfx +.type ecp_nistz256_point_double$sfx,\@function,2 +.align 32 +ecp_nistz256_point_double$sfx: +.cfi_startproc + _CET_ENDBR + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + sub \$32*5+8, %rsp +.cfi_adjust_cfa_offset 32*5+8 +.Lpoint_double${x}_body: + +.Lpoint_double_shortcut$x: + movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x + mov $a_ptr, $b_ptr # backup copy + movdqu 0x10($a_ptr), %xmm1 + mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order + mov 0x20+8*1($a_ptr), $acc5 + mov 0x20+8*2($a_ptr), $acc0 + mov 0x20+8*3($a_ptr), $acc1 + mov .Lpoly+8*1(%rip), $poly1 + mov .Lpoly+8*3(%rip), $poly3 + movdqa %xmm0, $in_x(%rsp) + movdqa %xmm1, $in_x+0x10(%rsp) + lea 0x20($r_ptr), $acc2 + lea 0x40($r_ptr), $acc3 + movq $r_ptr, %xmm0 + movq $acc2, %xmm1 + movq $acc3, %xmm2 + + lea $S(%rsp), $r_ptr + call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y); + + mov 0x40+8*0($a_ptr), $src0 + mov 0x40+8*1($a_ptr), $acc6 + mov 0x40+8*2($a_ptr), $acc7 + mov 0x40+8*3($a_ptr), $acc0 + lea 0x40-$bias($a_ptr), $a_ptr + lea $Zsqr(%rsp), $r_ptr + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z); + + `&load_for_sqr("$S(%rsp)", "$src0")` + lea $S(%rsp), $r_ptr + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S); + + mov 0x20($b_ptr), $src0 # $b_ptr is still valid + mov 0x40+8*0($b_ptr), $acc1 + mov 0x40+8*1($b_ptr), $acc2 + mov 0x40+8*2($b_ptr), $acc3 + mov 0x40+8*3($b_ptr), $acc4 + lea 0x40-$bias($b_ptr), $a_ptr + lea 0x20($b_ptr), $b_ptr + movq %xmm2, $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y); + call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z); + + mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order + mov $in_x+8*1(%rsp), $acc5 + lea $Zsqr(%rsp), $b_ptr + mov $in_x+8*2(%rsp), $acc0 + mov $in_x+8*3(%rsp), $acc1 + lea $M(%rsp), $r_ptr + call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr); + + mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order + mov $in_x+8*1(%rsp), $acc5 + lea $Zsqr(%rsp), $b_ptr + mov $in_x+8*2(%rsp), $acc0 + mov $in_x+8*3(%rsp), $acc1 + lea $Zsqr(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr); + + `&load_for_sqr("$S(%rsp)", "$src0")` + movq %xmm1, $r_ptr + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S); +___ +{ +######## ecp_nistz256_div_by_2(res_y, res_y); ########################## +# operate in 4-5-6-7 "name space" that matches squaring output +# +my ($poly1,$poly3)=($a_ptr,$t1); +my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2); + +$code.=<<___; + xor $t4, $t4 + mov $a0, $t0 + add \$-1, $a0 + mov $a1, $t1 + adc $poly1, $a1 + mov $a2, $t2 + adc \$0, $a2 + mov $a3, $t3 + adc $poly3, $a3 + adc \$0, $t4 + xor $a_ptr, $a_ptr # borrow $a_ptr + test \$1, $t0 + + cmovz $t0, $a0 + cmovz $t1, $a1 + cmovz $t2, $a2 + cmovz $t3, $a3 + cmovz $a_ptr, $t4 + + mov $a1, $t0 # a0:a3>>1 + shr \$1, $a0 + shl \$63, $t0 + mov $a2, $t1 + shr \$1, $a1 + or $t0, $a0 + shl \$63, $t1 + mov $a3, $t2 + shr \$1, $a2 + or $t1, $a1 + shl \$63, $t2 + mov $a0, 8*0($r_ptr) + shr \$1, $a3 + mov $a1, 8*1($r_ptr) + shl \$63, $t4 + or $t2, $a2 + or $t4, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) +___ +} +$code.=<<___; + `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")` + lea $M(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr); + + lea $tmp0(%rsp), $r_ptr + call __ecp_nistz256_mul_by_2$x + + lea $M(%rsp), $b_ptr + lea $M(%rsp), $r_ptr + call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M); + + `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")` + lea $S(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x); + + lea $tmp0(%rsp), $r_ptr + call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S); + + `&load_for_sqr("$M(%rsp)", "$src0")` + movq %xmm0, $r_ptr + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M); + + lea $tmp0(%rsp), $b_ptr + mov $acc6, $acc0 # harmonize sqr output and sub input + mov $acc7, $acc1 + mov $a_ptr, $poly1 + mov $t1, $poly3 + call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0); + + mov $S+8*0(%rsp), $t0 + mov $S+8*1(%rsp), $t1 + mov $S+8*2(%rsp), $t2 + mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order + lea $S(%rsp), $r_ptr + call __ecp_nistz256_sub$x # p256_sub(S, S, res_x); + + mov $M(%rsp), $src0 + lea $M(%rsp), $b_ptr + mov $acc4, $acc6 # harmonize sub output and mul input + xor %ecx, %ecx + mov $acc4, $S+8*0(%rsp) # have to save:-( + mov $acc5, $acc2 + mov $acc5, $S+8*1(%rsp) + cmovz $acc0, $acc3 + mov $acc0, $S+8*2(%rsp) + lea $S-$bias(%rsp), $a_ptr + cmovz $acc1, $acc4 + mov $acc1, $S+8*3(%rsp) + mov $acc6, $acc1 + lea $S(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M); + + movq %xmm1, $b_ptr + movq %xmm1, $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y); + + lea 32*5+56(%rsp), %rsi +.cfi_def_cfa %rsi,8 + mov -48(%rsi),%r15 +.cfi_restore %r15 + mov -40(%rsi),%r14 +.cfi_restore %r14 + mov -32(%rsi),%r13 +.cfi_restore %r13 + mov -24(%rsi),%r12 +.cfi_restore %r12 + mov -16(%rsi),%rbx +.cfi_restore %rbx + mov -8(%rsi),%rbp +.cfi_restore %rbp + lea (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Lpoint_double${x}_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx +___ +} +&gen_double("q"); + +sub gen_add () { + my $x = shift; + my ($src0,$sfx,$bias); + my ($H,$Hsqr,$R,$Rsqr,$Hcub, + $U1,$U2,$S1,$S2, + $res_x,$res_y,$res_z, + $in1_x,$in1_y,$in1_z, + $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17)); + my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr); + + if ($x ne "x") { + $src0 = "%rax"; + $sfx = "_nohw"; + $bias = 0; + } else { + $src0 = "%rdx"; + $sfx = "_adx"; + $bias = 128; + } +$code.=<<___; +.globl ecp_nistz256_point_add$sfx +.type ecp_nistz256_point_add$sfx,\@function,3 +.align 32 +ecp_nistz256_point_add$sfx: +.cfi_startproc + _CET_ENDBR + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + sub \$32*18+8, %rsp +.cfi_adjust_cfa_offset 32*18+8 +.Lpoint_add${x}_body: + + movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr + movdqu 0x10($a_ptr), %xmm1 + movdqu 0x20($a_ptr), %xmm2 + movdqu 0x30($a_ptr), %xmm3 + movdqu 0x40($a_ptr), %xmm4 + movdqu 0x50($a_ptr), %xmm5 + mov $a_ptr, $b_ptr # reassign + mov $b_org, $a_ptr # reassign + movdqa %xmm0, $in1_x(%rsp) + movdqa %xmm1, $in1_x+0x10(%rsp) + movdqa %xmm2, $in1_y(%rsp) + movdqa %xmm3, $in1_y+0x10(%rsp) + movdqa %xmm4, $in1_z(%rsp) + movdqa %xmm5, $in1_z+0x10(%rsp) + por %xmm4, %xmm5 + + movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr + pshufd \$0xb1, %xmm5, %xmm3 + movdqu 0x10($a_ptr), %xmm1 + movdqu 0x20($a_ptr), %xmm2 + por %xmm3, %xmm5 + movdqu 0x30($a_ptr), %xmm3 + mov 0x40+8*0($a_ptr), $src0 # load original in2_z + mov 0x40+8*1($a_ptr), $acc6 + mov 0x40+8*2($a_ptr), $acc7 + mov 0x40+8*3($a_ptr), $acc0 + movdqa %xmm0, $in2_x(%rsp) + pshufd \$0x1e, %xmm5, %xmm4 + movdqa %xmm1, $in2_x+0x10(%rsp) + movdqu 0x40($a_ptr),%xmm0 # in2_z again + movdqu 0x50($a_ptr),%xmm1 + movdqa %xmm2, $in2_y(%rsp) + movdqa %xmm3, $in2_y+0x10(%rsp) + por %xmm4, %xmm5 + pxor %xmm4, %xmm4 + por %xmm0, %xmm1 + movq $r_ptr, %xmm0 # save $r_ptr + + lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid + mov $src0, $in2_z+8*0(%rsp) # make in2_z copy + mov $acc6, $in2_z+8*1(%rsp) + mov $acc7, $in2_z+8*2(%rsp) + mov $acc0, $in2_z+8*3(%rsp) + lea $Z2sqr(%rsp), $r_ptr # Z2^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z); + + pcmpeqd %xmm4, %xmm5 + pshufd \$0xb1, %xmm1, %xmm4 + por %xmm1, %xmm4 + pshufd \$0, %xmm5, %xmm5 # in1infty + pshufd \$0x1e, %xmm4, %xmm3 + por %xmm3, %xmm4 + pxor %xmm3, %xmm3 + pcmpeqd %xmm3, %xmm4 + pshufd \$0, %xmm4, %xmm4 # in2infty + mov 0x40+8*0($b_ptr), $src0 # load original in1_z + mov 0x40+8*1($b_ptr), $acc6 + mov 0x40+8*2($b_ptr), $acc7 + mov 0x40+8*3($b_ptr), $acc0 + movq $b_ptr, %xmm1 + + lea 0x40-$bias($b_ptr), $a_ptr + lea $Z1sqr(%rsp), $r_ptr # Z1^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z); + + `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")` + lea $S1(%rsp), $r_ptr # S1 = Z2^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z); + + `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr # S2 = Z1^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z); + + `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")` + lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y); + + `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y); + + lea $S1(%rsp), $b_ptr + lea $R(%rsp), $r_ptr # R = S2 - S1 + call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1); + + or $acc5, $acc4 # see if result is zero + movdqa %xmm4, %xmm2 + or $acc0, $acc4 + or $acc1, $acc4 + por %xmm5, %xmm2 # in1infty || in2infty + movq $acc4, %xmm3 + + `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")` + lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr); + + `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")` + lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr); + + lea $U1(%rsp), $b_ptr + lea $H(%rsp), $r_ptr # H = U2 - U1 + call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1); + + or $acc5, $acc4 # see if result is zero + or $acc0, $acc4 + or $acc1, $acc4 # !is_equal(U1, U2) + + movq %xmm2, $acc0 + movq %xmm3, $acc1 + or $acc0, $acc4 + .byte 0x3e # predict taken + jnz .Ladd_proceed$x # !is_equal(U1, U2) || in1infty || in2infty + + # We now know A = B or A = -B and neither is infinity. Compare the + # y-coordinates via S1 and S2. + test $acc1, $acc1 + jz .Ladd_double$x # is_equal(S1, S2) + + # A = -B, so the result is infinity. + # + # TODO(davidben): Does .Ladd_proceed handle this case? It seems to, in + # which case we should eliminate this special-case and simplify the + # timing analysis. + movq %xmm0, $r_ptr # restore $r_ptr + pxor %xmm0, %xmm0 + movdqu %xmm0, 0x00($r_ptr) + movdqu %xmm0, 0x10($r_ptr) + movdqu %xmm0, 0x20($r_ptr) + movdqu %xmm0, 0x30($r_ptr) + movdqu %xmm0, 0x40($r_ptr) + movdqu %xmm0, 0x50($r_ptr) + jmp .Ladd_done$x + +.align 32 +.Ladd_double$x: + movq %xmm1, $a_ptr # restore $a_ptr + movq %xmm0, $r_ptr # restore $r_ptr + add \$`32*(18-5)`, %rsp # difference in frame sizes +.cfi_adjust_cfa_offset `-32*(18-5)` + jmp .Lpoint_double_shortcut$x +.cfi_adjust_cfa_offset `32*(18-5)` + +.align 32 +.Ladd_proceed$x: + `&load_for_sqr("$R(%rsp)", "$src0")` + lea $Rsqr(%rsp), $r_ptr # R^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R); + + `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")` + lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z); + + `&load_for_sqr("$H(%rsp)", "$src0")` + lea $Hsqr(%rsp), $r_ptr # H^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H); + + `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")` + lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z); + + `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")` + lea $Hcub(%rsp), $r_ptr # H^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H); + + `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")` + lea $U2(%rsp), $r_ptr # U1*H^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr); +___ +{ +####################################################################### +# operate in 4-5-0-1 "name space" that matches multiplication output +# +my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); +my ($poly1, $poly3)=($acc6,$acc7); + +$code.=<<___; + #lea $U2(%rsp), $a_ptr + #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2 + #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); + + xor $t4, $t4 + add $acc0, $acc0 # a0:a3+a0:a3 + lea $Rsqr(%rsp), $a_ptr + adc $acc1, $acc1 + mov $acc0, $t0 + adc $acc2, $acc2 + adc $acc3, $acc3 + mov $acc1, $t1 + adc \$0, $t4 + + sub \$-1, $acc0 + mov $acc2, $t2 + sbb $poly1, $acc1 + sbb \$0, $acc2 + mov $acc3, $t3 + sbb $poly3, $acc3 + sbb \$0, $t4 + + cmovc $t0, $acc0 + mov 8*0($a_ptr), $t0 + cmovc $t1, $acc1 + mov 8*1($a_ptr), $t1 + cmovc $t2, $acc2 + mov 8*2($a_ptr), $t2 + cmovc $t3, $acc3 + mov 8*3($a_ptr), $t3 + + call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr); + + lea $Hcub(%rsp), $b_ptr + lea $res_x(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub); + + mov $U2+8*0(%rsp), $t0 + mov $U2+8*1(%rsp), $t1 + mov $U2+8*2(%rsp), $t2 + mov $U2+8*3(%rsp), $t3 + lea $res_y(%rsp), $r_ptr + + call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x); + + mov $acc0, 8*0($r_ptr) # save the result, as + mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't + mov $acc2, 8*2($r_ptr) + mov $acc3, 8*3($r_ptr) +___ +} +$code.=<<___; + `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub); + + `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")` + lea $res_y(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y); + + lea $S2(%rsp), $b_ptr + lea $res_y(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2); + + movq %xmm0, $r_ptr # restore $r_ptr + + movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_z(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_z+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_z(%rsp), %xmm2 + pand $in2_z+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_z(%rsp), %xmm2 + pand $in1_z+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x40($r_ptr) + movdqu %xmm3, 0x50($r_ptr) + + movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_x(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_x+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_x(%rsp), %xmm2 + pand $in2_x+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_x(%rsp), %xmm2 + pand $in1_x+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x00($r_ptr) + movdqu %xmm3, 0x10($r_ptr) + + movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_y(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_y+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_y(%rsp), %xmm2 + pand $in2_y+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_y(%rsp), %xmm2 + pand $in1_y+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x20($r_ptr) + movdqu %xmm3, 0x30($r_ptr) + +.Ladd_done$x: + lea 32*18+56(%rsp), %rsi +.cfi_def_cfa %rsi,8 + mov -48(%rsi),%r15 +.cfi_restore %r15 + mov -40(%rsi),%r14 +.cfi_restore %r14 + mov -32(%rsi),%r13 +.cfi_restore %r13 + mov -24(%rsi),%r12 +.cfi_restore %r12 + mov -16(%rsi),%rbx +.cfi_restore %rbx + mov -8(%rsi),%rbp +.cfi_restore %rbp + lea (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Lpoint_add${x}_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx +___ +} +&gen_add("q"); + +sub gen_add_affine () { + my $x = shift; + my ($src0,$sfx,$bias); + my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr, + $res_x,$res_y,$res_z, + $in1_x,$in1_y,$in1_z, + $in2_x,$in2_y)=map(32*$_,(0..14)); + my $Z1sqr = $S2; + + if ($x ne "x") { + $src0 = "%rax"; + $sfx = "_nohw"; + $bias = 0; + } else { + $src0 = "%rdx"; + $sfx = "_adx"; + $bias = 128; + } +$code.=<<___; +.globl ecp_nistz256_point_add_affine$sfx +.type ecp_nistz256_point_add_affine$sfx,\@function,3 +.align 32 +ecp_nistz256_point_add_affine$sfx: +.cfi_startproc + _CET_ENDBR + push %rbp +.cfi_push %rbp + push %rbx +.cfi_push %rbx + push %r12 +.cfi_push %r12 + push %r13 +.cfi_push %r13 + push %r14 +.cfi_push %r14 + push %r15 +.cfi_push %r15 + sub \$32*15+8, %rsp +.cfi_adjust_cfa_offset 32*15+8 +.Ladd_affine${x}_body: + + movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr + mov $b_org, $b_ptr # reassign + movdqu 0x10($a_ptr), %xmm1 + movdqu 0x20($a_ptr), %xmm2 + movdqu 0x30($a_ptr), %xmm3 + movdqu 0x40($a_ptr), %xmm4 + movdqu 0x50($a_ptr), %xmm5 + mov 0x40+8*0($a_ptr), $src0 # load original in1_z + mov 0x40+8*1($a_ptr), $acc6 + mov 0x40+8*2($a_ptr), $acc7 + mov 0x40+8*3($a_ptr), $acc0 + movdqa %xmm0, $in1_x(%rsp) + movdqa %xmm1, $in1_x+0x10(%rsp) + movdqa %xmm2, $in1_y(%rsp) + movdqa %xmm3, $in1_y+0x10(%rsp) + movdqa %xmm4, $in1_z(%rsp) + movdqa %xmm5, $in1_z+0x10(%rsp) + por %xmm4, %xmm5 + + movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr + pshufd \$0xb1, %xmm5, %xmm3 + movdqu 0x10($b_ptr), %xmm1 + movdqu 0x20($b_ptr), %xmm2 + por %xmm3, %xmm5 + movdqu 0x30($b_ptr), %xmm3 + movdqa %xmm0, $in2_x(%rsp) + pshufd \$0x1e, %xmm5, %xmm4 + movdqa %xmm1, $in2_x+0x10(%rsp) + por %xmm0, %xmm1 + movq $r_ptr, %xmm0 # save $r_ptr + movdqa %xmm2, $in2_y(%rsp) + movdqa %xmm3, $in2_y+0x10(%rsp) + por %xmm2, %xmm3 + por %xmm4, %xmm5 + pxor %xmm4, %xmm4 + por %xmm1, %xmm3 + + lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid + lea $Z1sqr(%rsp), $r_ptr # Z1^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z); + + pcmpeqd %xmm4, %xmm5 + pshufd \$0xb1, %xmm3, %xmm4 + mov 0x00($b_ptr), $src0 # $b_ptr is still valid + #lea 0x00($b_ptr), $b_ptr + mov $acc4, $acc1 # harmonize sqr output and mul input + por %xmm3, %xmm4 + pshufd \$0, %xmm5, %xmm5 # in1infty + pshufd \$0x1e, %xmm4, %xmm3 + mov $acc5, $acc2 + por %xmm3, %xmm4 + pxor %xmm3, %xmm3 + mov $acc6, $acc3 + pcmpeqd %xmm3, %xmm4 + pshufd \$0, %xmm4, %xmm4 # in2infty + + lea $Z1sqr-$bias(%rsp), $a_ptr + mov $acc7, $acc4 + lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x); + + lea $in1_x(%rsp), $b_ptr + lea $H(%rsp), $r_ptr # H = U2 - U1 + call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x); + + `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr # S2 = Z1^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z); + + `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")` + lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z); + + `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y); + + lea $in1_y(%rsp), $b_ptr + lea $R(%rsp), $r_ptr # R = S2 - S1 + call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y); + + `&load_for_sqr("$H(%rsp)", "$src0")` + lea $Hsqr(%rsp), $r_ptr # H^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H); + + `&load_for_sqr("$R(%rsp)", "$src0")` + lea $Rsqr(%rsp), $r_ptr # R^2 + call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R); + + `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")` + lea $Hcub(%rsp), $r_ptr # H^3 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H); + + `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")` + lea $U2(%rsp), $r_ptr # U1*H^2 + call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr); +___ +{ +####################################################################### +# operate in 4-5-0-1 "name space" that matches multiplication output +# +my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); +my ($poly1, $poly3)=($acc6,$acc7); + +$code.=<<___; + #lea $U2(%rsp), $a_ptr + #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2 + #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); + + xor $t4, $t4 + add $acc0, $acc0 # a0:a3+a0:a3 + lea $Rsqr(%rsp), $a_ptr + adc $acc1, $acc1 + mov $acc0, $t0 + adc $acc2, $acc2 + adc $acc3, $acc3 + mov $acc1, $t1 + adc \$0, $t4 + + sub \$-1, $acc0 + mov $acc2, $t2 + sbb $poly1, $acc1 + sbb \$0, $acc2 + mov $acc3, $t3 + sbb $poly3, $acc3 + sbb \$0, $t4 + + cmovc $t0, $acc0 + mov 8*0($a_ptr), $t0 + cmovc $t1, $acc1 + mov 8*1($a_ptr), $t1 + cmovc $t2, $acc2 + mov 8*2($a_ptr), $t2 + cmovc $t3, $acc3 + mov 8*3($a_ptr), $t3 + + call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr); + + lea $Hcub(%rsp), $b_ptr + lea $res_x(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub); + + mov $U2+8*0(%rsp), $t0 + mov $U2+8*1(%rsp), $t1 + mov $U2+8*2(%rsp), $t2 + mov $U2+8*3(%rsp), $t3 + lea $H(%rsp), $r_ptr + + call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x); + + mov $acc0, 8*0($r_ptr) # save the result, as + mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't + mov $acc2, 8*2($r_ptr) + mov $acc3, 8*3($r_ptr) +___ +} +$code.=<<___; + `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")` + lea $S2(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y); + + `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")` + lea $H(%rsp), $r_ptr + call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R); + + lea $S2(%rsp), $b_ptr + lea $res_y(%rsp), $r_ptr + call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2); + + movq %xmm0, $r_ptr # restore $r_ptr + + movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_z(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_z+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand .LONE_mont(%rip), %xmm2 + pand .LONE_mont+0x10(%rip), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_z(%rsp), %xmm2 + pand $in1_z+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x40($r_ptr) + movdqu %xmm3, 0x50($r_ptr) + + movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_x(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_x+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_x(%rsp), %xmm2 + pand $in2_x+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_x(%rsp), %xmm2 + pand $in1_x+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x00($r_ptr) + movdqu %xmm3, 0x10($r_ptr) + + movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty); + movdqa %xmm5, %xmm1 + pandn $res_y(%rsp), %xmm0 + movdqa %xmm5, %xmm2 + pandn $res_y+0x10(%rsp), %xmm1 + movdqa %xmm5, %xmm3 + pand $in2_y(%rsp), %xmm2 + pand $in2_y+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + + movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty); + movdqa %xmm4, %xmm1 + pandn %xmm2, %xmm0 + movdqa %xmm4, %xmm2 + pandn %xmm3, %xmm1 + movdqa %xmm4, %xmm3 + pand $in1_y(%rsp), %xmm2 + pand $in1_y+0x10(%rsp), %xmm3 + por %xmm0, %xmm2 + por %xmm1, %xmm3 + movdqu %xmm2, 0x20($r_ptr) + movdqu %xmm3, 0x30($r_ptr) + + lea 32*15+56(%rsp), %rsi +.cfi_def_cfa %rsi,8 + mov -48(%rsi),%r15 +.cfi_restore %r15 + mov -40(%rsi),%r14 +.cfi_restore %r14 + mov -32(%rsi),%r13 +.cfi_restore %r13 + mov -24(%rsi),%r12 +.cfi_restore %r12 + mov -16(%rsi),%rbx +.cfi_restore %rbx + mov -8(%rsi),%rbp +.cfi_restore %rbp + lea (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Ladd_affine${x}_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx +___ +} +&gen_add_affine("q"); + +######################################################################## +# AD*X magic +# +if ($addx) { { +######################################################################## +# operate in 4-5-0-1 "name space" that matches multiplication output +# +my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); + +$code.=<<___; +.type __ecp_nistz256_add_tox,\@abi-omnipotent +.align 32 +__ecp_nistz256_add_tox: +.cfi_startproc + xor $t4, $t4 + adc 8*0($b_ptr), $a0 + adc 8*1($b_ptr), $a1 + mov $a0, $t0 + adc 8*2($b_ptr), $a2 + adc 8*3($b_ptr), $a3 + mov $a1, $t1 + adc \$0, $t4 + + xor $t3, $t3 + sbb \$-1, $a0 + mov $a2, $t2 + sbb $poly1, $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb $poly3, $a3 + sbb \$0, $t4 + + cmovc $t0, $a0 + cmovc $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovc $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovc $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox + +.type __ecp_nistz256_sub_fromx,\@abi-omnipotent +.align 32 +__ecp_nistz256_sub_fromx: +.cfi_startproc + xor $t4, $t4 + sbb 8*0($b_ptr), $a0 + sbb 8*1($b_ptr), $a1 + mov $a0, $t0 + sbb 8*2($b_ptr), $a2 + sbb 8*3($b_ptr), $a3 + mov $a1, $t1 + sbb \$0, $t4 + + xor $t3, $t3 + adc \$-1, $a0 + mov $a2, $t2 + adc $poly1, $a1 + adc \$0, $a2 + mov $a3, $t3 + adc $poly3, $a3 + + bt \$0, $t4 + cmovnc $t0, $a0 + cmovnc $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovnc $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovnc $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx + +.type __ecp_nistz256_subx,\@abi-omnipotent +.align 32 +__ecp_nistz256_subx: +.cfi_startproc + xor $t4, $t4 + sbb $a0, $t0 + sbb $a1, $t1 + mov $t0, $a0 + sbb $a2, $t2 + sbb $a3, $t3 + mov $t1, $a1 + sbb \$0, $t4 + + xor $a3 ,$a3 + adc \$-1, $t0 + mov $t2, $a2 + adc $poly1, $t1 + adc \$0, $t2 + mov $t3, $a3 + adc $poly3, $t3 + + bt \$0, $t4 + cmovc $t0, $a0 + cmovc $t1, $a1 + cmovc $t2, $a2 + cmovc $t3, $a3 + + ret +.cfi_endproc +.size __ecp_nistz256_subx,.-__ecp_nistz256_subx + +.type __ecp_nistz256_mul_by_2x,\@abi-omnipotent +.align 32 +__ecp_nistz256_mul_by_2x: +.cfi_startproc + xor $t4, $t4 + adc $a0, $a0 # a0:a3+a0:a3 + adc $a1, $a1 + mov $a0, $t0 + adc $a2, $a2 + adc $a3, $a3 + mov $a1, $t1 + adc \$0, $t4 + + xor $t3, $t3 + sbb \$-1, $a0 + mov $a2, $t2 + sbb $poly1, $a1 + sbb \$0, $a2 + mov $a3, $t3 + sbb $poly3, $a3 + sbb \$0, $t4 + + cmovc $t0, $a0 + cmovc $t1, $a1 + mov $a0, 8*0($r_ptr) + cmovc $t2, $a2 + mov $a1, 8*1($r_ptr) + cmovc $t3, $a3 + mov $a2, 8*2($r_ptr) + mov $a3, 8*3($r_ptr) + + ret +.cfi_endproc +.size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x +___ + } +&gen_double("x"); +&gen_add("x"); +&gen_add_affine("x"); +} +}}} # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, # CONTEXT *context,DISPATCHER_CONTEXT *disp) @@ -1280,6 +3943,10 @@ .section .pdata .align 4 + .rva .LSEH_begin_ecp_nistz256_neg + .rva .LSEH_end_ecp_nistz256_neg + .rva .LSEH_info_ecp_nistz256_neg + .rva .LSEH_begin_ecp_nistz256_ord_mul_mont_nohw .rva .LSEH_end_ecp_nistz256_ord_mul_mont_nohw .rva .LSEH_info_ecp_nistz256_ord_mul_mont_nohw @@ -1298,9 +3965,75 @@ .rva .LSEH_info_ecp_nistz256_ord_sqr_mont_adx ___ $code.=<<___; + .rva .LSEH_begin_ecp_nistz256_mul_mont_nohw + .rva .LSEH_end_ecp_nistz256_mul_mont_nohw + .rva .LSEH_info_ecp_nistz256_mul_mont_nohw + + .rva .LSEH_begin_ecp_nistz256_sqr_mont_nohw + .rva .LSEH_end_ecp_nistz256_sqr_mont_nohw + .rva .LSEH_info_ecp_nistz256_sqr_mont_nohw +___ +$code.=<<___ if ($addx); + .rva .LSEH_begin_ecp_nistz256_mul_mont_adx + .rva .LSEH_end_ecp_nistz256_mul_mont_adx + .rva .LSEH_info_ecp_nistz256_mul_mont_adx + + .rva .LSEH_begin_ecp_nistz256_sqr_mont_adx + .rva .LSEH_end_ecp_nistz256_sqr_mont_adx + .rva .LSEH_info_ecp_nistz256_sqr_mont_adx +___ +$code.=<<___; + .rva .LSEH_begin_ecp_nistz256_select_w5_nohw + .rva .LSEH_end_ecp_nistz256_select_w5_nohw + .rva .LSEH_info_ecp_nistz256_select_wX_nohw + + .rva .LSEH_begin_ecp_nistz256_select_w7_nohw + .rva .LSEH_end_ecp_nistz256_select_w7_nohw + .rva .LSEH_info_ecp_nistz256_select_wX_nohw +___ +$code.=<<___ if ($avx>1); + .rva .LSEH_begin_ecp_nistz256_select_w5_avx2 + .rva .LSEH_end_ecp_nistz256_select_w5_avx2 + .rva .LSEH_info_ecp_nistz256_select_wX_avx2 + + .rva .LSEH_begin_ecp_nistz256_select_w7_avx2 + .rva .LSEH_end_ecp_nistz256_select_w7_avx2 + .rva .LSEH_info_ecp_nistz256_select_wX_avx2 +___ +$code.=<<___; + .rva .LSEH_begin_ecp_nistz256_point_double_nohw + .rva .LSEH_end_ecp_nistz256_point_double_nohw + .rva .LSEH_info_ecp_nistz256_point_double_nohw + + .rva .LSEH_begin_ecp_nistz256_point_add_nohw + .rva .LSEH_end_ecp_nistz256_point_add_nohw + .rva .LSEH_info_ecp_nistz256_point_add_nohw + + .rva .LSEH_begin_ecp_nistz256_point_add_affine_nohw + .rva .LSEH_end_ecp_nistz256_point_add_affine_nohw + .rva .LSEH_info_ecp_nistz256_point_add_affine_nohw +___ +$code.=<<___ if ($addx); + .rva .LSEH_begin_ecp_nistz256_point_double_adx + .rva .LSEH_end_ecp_nistz256_point_double_adx + .rva .LSEH_info_ecp_nistz256_point_double_adx + + .rva .LSEH_begin_ecp_nistz256_point_add_adx + .rva .LSEH_end_ecp_nistz256_point_add_adx + .rva .LSEH_info_ecp_nistz256_point_add_adx + + .rva .LSEH_begin_ecp_nistz256_point_add_affine_adx + .rva .LSEH_end_ecp_nistz256_point_add_affine_adx + .rva .LSEH_info_ecp_nistz256_point_add_affine_adx +___ +$code.=<<___; .section .xdata .align 8 +.LSEH_info_ecp_nistz256_neg: + .byte 9,0,0,0 + .rva short_handler + .rva .Lneg_body,.Lneg_epilogue # HandlerData[] .LSEH_info_ecp_nistz256_ord_mul_mont_nohw: .byte 9,0,0,0 .rva full_handler @@ -1324,7 +4057,98 @@ .rva .Lord_sqrx_body,.Lord_sqrx_epilogue # HandlerData[] .long 48,0 ___ - +$code.=<<___; +.LSEH_info_ecp_nistz256_mul_mont_nohw: + .byte 9,0,0,0 + .rva full_handler + .rva .Lmul_body,.Lmul_epilogue # HandlerData[] + .long 48,0 +.LSEH_info_ecp_nistz256_sqr_mont_nohw: + .byte 9,0,0,0 + .rva full_handler + .rva .Lsqr_body,.Lsqr_epilogue # HandlerData[] + .long 48,0 +___ +$code.=<<___ if ($addx); +.LSEH_info_ecp_nistz256_mul_mont_adx: + .byte 9,0,0,0 + .rva full_handler + .rva .Lmulx_body,.Lmulx_epilogue # HandlerData[] + .long 48,0 +.LSEH_info_ecp_nistz256_sqr_mont_adx: + .byte 9,0,0,0 + .rva full_handler + .rva .Lsqrx_body,.Lsqrx_epilogue # HandlerData[] + .long 48,0 +___ +$code.=<<___; +.LSEH_info_ecp_nistz256_select_wX_nohw: + .byte 0x01,0x33,0x16,0x00 + .byte 0x33,0xf8,0x09,0x00 #movaps 0x90(rsp),xmm15 + .byte 0x2e,0xe8,0x08,0x00 #movaps 0x80(rsp),xmm14 + .byte 0x29,0xd8,0x07,0x00 #movaps 0x70(rsp),xmm13 + .byte 0x24,0xc8,0x06,0x00 #movaps 0x60(rsp),xmm12 + .byte 0x1f,0xb8,0x05,0x00 #movaps 0x50(rsp),xmm11 + .byte 0x1a,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10 + .byte 0x15,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9 + .byte 0x10,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8 + .byte 0x0c,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7 + .byte 0x08,0x68,0x00,0x00 #movaps 0x00(rsp),xmm6 + .byte 0x04,0x01,0x15,0x00 #sub rsp,0xa8 + .align 8 +___ +$code.=<<___ if ($avx>1); +.LSEH_info_ecp_nistz256_select_wX_avx2: + .byte 0x01,0x36,0x17,0x0b + .byte 0x36,0xf8,0x09,0x00 # vmovaps 0x90(rsp),xmm15 + .byte 0x31,0xe8,0x08,0x00 # vmovaps 0x80(rsp),xmm14 + .byte 0x2c,0xd8,0x07,0x00 # vmovaps 0x70(rsp),xmm13 + .byte 0x27,0xc8,0x06,0x00 # vmovaps 0x60(rsp),xmm12 + .byte 0x22,0xb8,0x05,0x00 # vmovaps 0x50(rsp),xmm11 + .byte 0x1d,0xa8,0x04,0x00 # vmovaps 0x40(rsp),xmm10 + .byte 0x18,0x98,0x03,0x00 # vmovaps 0x30(rsp),xmm9 + .byte 0x13,0x88,0x02,0x00 # vmovaps 0x20(rsp),xmm8 + .byte 0x0e,0x78,0x01,0x00 # vmovaps 0x10(rsp),xmm7 + .byte 0x09,0x68,0x00,0x00 # vmovaps 0x00(rsp),xmm6 + .byte 0x04,0x01,0x15,0x00 # sub rsp,0xa8 + .byte 0x00,0xb3,0x00,0x00 # set_frame r11 + .align 8 +___ +$code.=<<___; +.LSEH_info_ecp_nistz256_point_double_nohw: + .byte 9,0,0,0 + .rva full_handler + .rva .Lpoint_doubleq_body,.Lpoint_doubleq_epilogue # HandlerData[] + .long 32*5+56,0 +.LSEH_info_ecp_nistz256_point_add_nohw: + .byte 9,0,0,0 + .rva full_handler + .rva .Lpoint_addq_body,.Lpoint_addq_epilogue # HandlerData[] + .long 32*18+56,0 +.LSEH_info_ecp_nistz256_point_add_affine_nohw: + .byte 9,0,0,0 + .rva full_handler + .rva .Ladd_affineq_body,.Ladd_affineq_epilogue # HandlerData[] + .long 32*15+56,0 +___ +$code.=<<___ if ($addx); +.align 8 +.LSEH_info_ecp_nistz256_point_double_adx: + .byte 9,0,0,0 + .rva full_handler + .rva .Lpoint_doublex_body,.Lpoint_doublex_epilogue # HandlerData[] + .long 32*5+56,0 +.LSEH_info_ecp_nistz256_point_add_adx: + .byte 9,0,0,0 + .rva full_handler + .rva .Lpoint_addx_body,.Lpoint_addx_epilogue # HandlerData[] + .long 32*18+56,0 +.LSEH_info_ecp_nistz256_point_add_affine_adx: + .byte 9,0,0,0 + .rva full_handler + .rva .Ladd_affinex_body,.Ladd_affinex_epilogue # HandlerData[] + .long 32*15+56,0 +___ } $code =~ s/\`([^\`]*)\`/eval $1/gem;
diff --git a/crypto/fipsmodule/ec/ec.cc.inc b/crypto/fipsmodule/ec/ec.cc.inc index e23d436..5c795d9 100644 --- a/crypto/fipsmodule/ec/ec.cc.inc +++ b/crypto/fipsmodule/ec/ec.cc.inc
@@ -99,7 +99,13 @@ ec_group_init_static_mont(&out->order, std::size(kP256Order), kP256Order, kP256OrderRR, kP256OrderN0); +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ + !defined(OPENSSL_SMALL) + out->meth = EC_GFp_nistz256_method(); +#else out->meth = EC_GFp_nistp256_method(); +#endif out->generator.group = out; OPENSSL_memcpy(out->generator.raw.X.words, kP256MontGX, sizeof(kP256MontGX)); OPENSSL_memcpy(out->generator.raw.Y.words, kP256MontGY, sizeof(kP256MontGY)); @@ -762,7 +768,7 @@ // Check the result is on the curve to defend against fault attacks or bugs. // This has negligible cost compared to the multiplication. - if (!constant_time_declassify_int(ec_GFp_simple_is_on_curve(group, r))) { + if (!ec_GFp_simple_is_on_curve(group, r)) { OPENSSL_PUT_ERROR(EC, ERR_R_INTERNAL_ERROR); return 0; }
diff --git a/crypto/fipsmodule/ec/internal.h b/crypto/fipsmodule/ec/internal.h index 14fe006..2261f20 100644 --- a/crypto/fipsmodule/ec/internal.h +++ b/crypto/fipsmodule/ec/internal.h
@@ -666,6 +666,10 @@ const EC_METHOD *EC_GFp_nistp256_method(); +// EC_GFp_nistz256_method is a GFp method using montgomery multiplication, with +// x86-64 optimized P256. See http://eprint.iacr.org/2013/816. +const EC_METHOD *EC_GFp_nistz256_method(); + // An EC_WRAPPED_SCALAR is an `EC_SCALAR` with a parallel `BIGNUM` // representation. It exists to support the `EC_KEY_get0_private_key` API. typedef struct {
diff --git a/crypto/fipsmodule/ec/p256-nistz.cc.inc b/crypto/fipsmodule/ec/p256-nistz.cc.inc new file mode 100644 index 0000000..0e2d282 --- /dev/null +++ b/crypto/fipsmodule/ec/p256-nistz.cc.inc
@@ -0,0 +1,740 @@ +// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. +// Copyright (c) 2014, Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1) +// (1) Intel Corporation, Israel Development Center, Haifa, Israel +// (2) University of Haifa, Israel +// +// Reference: +// S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with +// 256 Bit Primes" + +#include <openssl/ec.h> + +#include <assert.h> +#include <stdint.h> +#include <string.h> + +#include <openssl/bn.h> +#include <openssl/crypto.h> +#include <openssl/err.h> + +#include "../../internal.h" +#include "../bn/internal.h" +#include "../delocate.h" +#include "internal.h" +#include "p256-nistz.h" + + +using namespace bssl; + +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ + !defined(OPENSSL_SMALL) + +typedef P256_POINT_AFFINE PRECOMP256_ROW[64]; + +// One converted into the Montgomery domain +static const BN_ULONG ONE_MONT[P256_LIMBS] = { + TOBN(0x00000000, 0x00000001), + TOBN(0xffffffff, 0x00000000), + TOBN(0xffffffff, 0xffffffff), + TOBN(0x00000000, 0xfffffffe), +}; + +// Precomputed tables for the default generator +#include "p256-nistz-table.h" + +// Recode window to a signed digit, see `ec_GFp_nistp_recode_scalar_bits` in +// util.c for details +static crypto_word_t booth_recode_w5(crypto_word_t in) { + crypto_word_t s, d; + + s = ~((in >> 5) - 1); + d = (1 << 6) - in - 1; + d = (d & s) | (in & ~s); + d = (d >> 1) + (d & 1); + + return (d << 1) + (s & 1); +} + +static crypto_word_t booth_recode_w7(crypto_word_t in) { + crypto_word_t s, d; + + s = ~((in >> 7) - 1); + d = (1 << 8) - in - 1; + d = (d & s) | (in & ~s); + d = (d >> 1) + (d & 1); + + return (d << 1) + (s & 1); +} + +// copy_conditional copies `src` to `dst` if `move` is one and leaves it as-is +// if `move` is zero. +// +// WARNING: this breaks the usual convention of constant-time functions +// returning masks. +static void copy_conditional(BN_ULONG dst[P256_LIMBS], + const BN_ULONG src[P256_LIMBS], BN_ULONG move) { + BN_ULONG mask1 = ((BN_ULONG)0) - move; + BN_ULONG mask2 = ~mask1; + + dst[0] = (src[0] & mask1) ^ (dst[0] & mask2); + dst[1] = (src[1] & mask1) ^ (dst[1] & mask2); + dst[2] = (src[2] & mask1) ^ (dst[2] & mask2); + dst[3] = (src[3] & mask1) ^ (dst[3] & mask2); + if (P256_LIMBS == 8) { + dst[4] = (src[4] & mask1) ^ (dst[4] & mask2); + dst[5] = (src[5] & mask1) ^ (dst[5] & mask2); + dst[6] = (src[6] & mask1) ^ (dst[6] & mask2); + dst[7] = (src[7] & mask1) ^ (dst[7] & mask2); + } +} + +// is_not_zero returns one iff in != 0 and zero otherwise. +// +// WARNING: this breaks the usual convention of constant-time functions +// returning masks. +// +// (define-fun is_not_zero ((in (_ BitVec 64))) (_ BitVec 64) +// (bvlshr (bvor in (bvsub #x0000000000000000 in)) #x000000000000003f) +// ) +// +// (declare-fun x () (_ BitVec 64)) +// +// (assert (and (= x #x0000000000000000) (= (is_not_zero x) +// #x0000000000000001))) (check-sat) +// +// (assert (and (not (= x #x0000000000000000)) (= (is_not_zero x) +// #x0000000000000000))) (check-sat) +// +static BN_ULONG is_not_zero(BN_ULONG in) { + in |= (0 - in); + in >>= BN_BITS2 - 1; + return in; +} + +#if defined(OPENSSL_X86_64) +// Dispatch between CPU variations. The "_adx" suffixed functions use MULX in +// addition to ADCX/ADOX. MULX is part of BMI2, not ADX, so we must check both +// capabilities. +static void ecp_nistz256_mul_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]) { + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + ecp_nistz256_mul_mont_adx(res, a, b); + } else { + ecp_nistz256_mul_mont_nohw(res, a, b); + } +} + +static void ecp_nistz256_sqr_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]) { + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + ecp_nistz256_sqr_mont_adx(res, a); + } else { + ecp_nistz256_sqr_mont_nohw(res, a); + } +} + +static void ecp_nistz256_ord_mul_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]) { + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + ecp_nistz256_ord_mul_mont_adx(res, a, b); + } else { + ecp_nistz256_ord_mul_mont_nohw(res, a, b); + } +} + +static void ecp_nistz256_ord_sqr_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + BN_ULONG rep) { + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + ecp_nistz256_ord_sqr_mont_adx(res, a, rep); + } else { + ecp_nistz256_ord_sqr_mont_nohw(res, a, rep); + } +} + +static void ecp_nistz256_select_w5(P256_POINT *val, const P256_POINT in_t[16], + int index) { + if (CRYPTO_is_AVX2_capable()) { + ecp_nistz256_select_w5_avx2(val, in_t, index); + } else { + ecp_nistz256_select_w5_nohw(val, in_t, index); + } +} + +static void ecp_nistz256_select_w7(P256_POINT_AFFINE *val, + const P256_POINT_AFFINE in_t[64], + int index) { + if (CRYPTO_is_AVX2_capable()) { + ecp_nistz256_select_w7_avx2(val, in_t, index); + } else { + ecp_nistz256_select_w7_nohw(val, in_t, index); + } +} + +static void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a) { + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + ecp_nistz256_point_double_adx(r, a); + } else { + ecp_nistz256_point_double_nohw(r, a); + } +} + +static void ecp_nistz256_point_add(P256_POINT *r, const P256_POINT *a, + const P256_POINT *b) { + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + ecp_nistz256_point_add_adx(r, a, b); + } else { + ecp_nistz256_point_add_nohw(r, a, b); + } +} + +static void ecp_nistz256_point_add_affine(P256_POINT *r, const P256_POINT *a, + const P256_POINT_AFFINE *b) { + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + ecp_nistz256_point_add_affine_adx(r, a, b); + } else { + ecp_nistz256_point_add_affine_nohw(r, a, b); + } +} +#endif // OPENSSL_X86_64 + +// ecp_nistz256_from_mont sets `res` to `in`, converted from Montgomery domain +// by multiplying with 1. +static void ecp_nistz256_from_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG in[P256_LIMBS]) { + static const BN_ULONG ONE[P256_LIMBS] = {1}; + ecp_nistz256_mul_mont(res, in, ONE); +} + +// ecp_nistz256_mod_inverse_sqr_mont sets `r` to (`in` * 2^-256)^-2 * 2^256 mod +// p. That is, `r` is the modular inverse square of `in` for input and output in +// the Montgomery domain. +static void ecp_nistz256_mod_inverse_sqr_mont(BN_ULONG r[P256_LIMBS], + const BN_ULONG in[P256_LIMBS]) { + // This implements the addition chain described in + // https://briansmith.org/ecc-inversion-addition-chains-01#p256_field_inversion + BN_ULONG x2[P256_LIMBS], x3[P256_LIMBS], x6[P256_LIMBS], x12[P256_LIMBS], + x15[P256_LIMBS], x30[P256_LIMBS], x32[P256_LIMBS]; + ecp_nistz256_sqr_mont(x2, in); // 2^2 - 2^1 + ecp_nistz256_mul_mont(x2, x2, in); // 2^2 - 2^0 + + ecp_nistz256_sqr_mont(x3, x2); // 2^3 - 2^1 + ecp_nistz256_mul_mont(x3, x3, in); // 2^3 - 2^0 + + ecp_nistz256_sqr_mont(x6, x3); + for (int i = 1; i < 3; i++) { + ecp_nistz256_sqr_mont(x6, x6); + } // 2^6 - 2^3 + ecp_nistz256_mul_mont(x6, x6, x3); // 2^6 - 2^0 + + ecp_nistz256_sqr_mont(x12, x6); + for (int i = 1; i < 6; i++) { + ecp_nistz256_sqr_mont(x12, x12); + } // 2^12 - 2^6 + ecp_nistz256_mul_mont(x12, x12, x6); // 2^12 - 2^0 + + ecp_nistz256_sqr_mont(x15, x12); + for (int i = 1; i < 3; i++) { + ecp_nistz256_sqr_mont(x15, x15); + } // 2^15 - 2^3 + ecp_nistz256_mul_mont(x15, x15, x3); // 2^15 - 2^0 + + ecp_nistz256_sqr_mont(x30, x15); + for (int i = 1; i < 15; i++) { + ecp_nistz256_sqr_mont(x30, x30); + } // 2^30 - 2^15 + ecp_nistz256_mul_mont(x30, x30, x15); // 2^30 - 2^0 + + ecp_nistz256_sqr_mont(x32, x30); + ecp_nistz256_sqr_mont(x32, x32); // 2^32 - 2^2 + ecp_nistz256_mul_mont(x32, x32, x2); // 2^32 - 2^0 + + BN_ULONG ret[P256_LIMBS]; + ecp_nistz256_sqr_mont(ret, x32); + for (int i = 1; i < 31 + 1; i++) { + ecp_nistz256_sqr_mont(ret, ret); + } // 2^64 - 2^32 + ecp_nistz256_mul_mont(ret, ret, in); // 2^64 - 2^32 + 2^0 + + for (int i = 0; i < 96 + 32; i++) { + ecp_nistz256_sqr_mont(ret, ret); + } // 2^192 - 2^160 + 2^128 + ecp_nistz256_mul_mont(ret, ret, x32); // 2^192 - 2^160 + 2^128 + 2^32 - 2^0 + + for (int i = 0; i < 32; i++) { + ecp_nistz256_sqr_mont(ret, ret); + } // 2^224 - 2^192 + 2^160 + 2^64 - 2^32 + ecp_nistz256_mul_mont(ret, ret, x32); // 2^224 - 2^192 + 2^160 + 2^64 - 2^0 + + for (int i = 0; i < 30; i++) { + ecp_nistz256_sqr_mont(ret, ret); + } // 2^254 - 2^222 + 2^190 + 2^94 - 2^30 + ecp_nistz256_mul_mont(ret, ret, x30); // 2^254 - 2^222 + 2^190 + 2^94 - 2^0 + + ecp_nistz256_sqr_mont(ret, ret); + ecp_nistz256_sqr_mont(r, ret); // 2^256 - 2^224 + 2^192 + 2^96 - 2^2 +} + +// r = p * p_scalar +static void ecp_nistz256_windowed_mul(const EC_GROUP *group, P256_POINT *r, + const EC_JACOBIAN *p, + const EC_SCALAR *p_scalar) { + assert(p != nullptr); + assert(p_scalar != nullptr); + assert(group->field.N.width == P256_LIMBS); + + static const size_t kWindowSize = 5; + static const crypto_word_t kMask = (1 << (5 /* kWindowSize */ + 1)) - 1; + + // A `P256_POINT` is (3 * 32) = 96 bytes, and the 64-byte alignment should + // add no more than 63 bytes of overhead. Thus, `table` should require + // ~1599 ((96 * 16) + 63) bytes of stack space. + alignas(64) P256_POINT table[16]; + uint8_t p_str[33]; + OPENSSL_memcpy(p_str, p_scalar->words, 32); + p_str[32] = 0; + + // table[0] is implicitly (0,0,0) (the point at infinity), therefore it is + // not stored. All other values are actually stored with an offset of -1 in + // table. + P256_POINT *row = table; + assert(group->field.N.width == P256_LIMBS); + OPENSSL_memcpy(row[1 - 1].X, p->X.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(row[1 - 1].Y, p->Y.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(row[1 - 1].Z, p->Z.words, P256_LIMBS * sizeof(BN_ULONG)); + + ecp_nistz256_point_double(&row[2 - 1], &row[1 - 1]); + ecp_nistz256_point_add(&row[3 - 1], &row[2 - 1], &row[1 - 1]); + ecp_nistz256_point_double(&row[4 - 1], &row[2 - 1]); + ecp_nistz256_point_double(&row[6 - 1], &row[3 - 1]); + ecp_nistz256_point_double(&row[8 - 1], &row[4 - 1]); + ecp_nistz256_point_double(&row[12 - 1], &row[6 - 1]); + ecp_nistz256_point_add(&row[5 - 1], &row[4 - 1], &row[1 - 1]); + ecp_nistz256_point_add(&row[7 - 1], &row[6 - 1], &row[1 - 1]); + ecp_nistz256_point_add(&row[9 - 1], &row[8 - 1], &row[1 - 1]); + ecp_nistz256_point_add(&row[13 - 1], &row[12 - 1], &row[1 - 1]); + ecp_nistz256_point_double(&row[14 - 1], &row[7 - 1]); + ecp_nistz256_point_double(&row[10 - 1], &row[5 - 1]); + ecp_nistz256_point_add(&row[15 - 1], &row[14 - 1], &row[1 - 1]); + ecp_nistz256_point_add(&row[11 - 1], &row[10 - 1], &row[1 - 1]); + ecp_nistz256_point_double(&row[16 - 1], &row[8 - 1]); + + BN_ULONG tmp[P256_LIMBS]; + alignas(32) P256_POINT h; + size_t index = 255; + crypto_word_t wvalue = p_str[(index - 1) / 8]; + wvalue = (wvalue >> ((index - 1) % 8)) & kMask; + + ecp_nistz256_select_w5(r, table, booth_recode_w5(wvalue) >> 1); + + while (index >= 5) { + if (index != 255) { + size_t off = (index - 1) / 8; + + wvalue = (crypto_word_t)p_str[off] | (crypto_word_t)p_str[off + 1] << 8; + wvalue = (wvalue >> ((index - 1) % 8)) & kMask; + + wvalue = booth_recode_w5(wvalue); + + ecp_nistz256_select_w5(&h, table, wvalue >> 1); + + ecp_nistz256_neg(tmp, h.Y); + copy_conditional(h.Y, tmp, (wvalue & 1)); + + ecp_nistz256_point_add(r, r, &h); + } + + index -= kWindowSize; + + ecp_nistz256_point_double(r, r); + ecp_nistz256_point_double(r, r); + ecp_nistz256_point_double(r, r); + ecp_nistz256_point_double(r, r); + ecp_nistz256_point_double(r, r); + } + + // Final window + wvalue = p_str[0]; + wvalue = (wvalue << 1) & kMask; + + wvalue = booth_recode_w5(wvalue); + + ecp_nistz256_select_w5(&h, table, wvalue >> 1); + + ecp_nistz256_neg(tmp, h.Y); + copy_conditional(h.Y, tmp, wvalue & 1); + + ecp_nistz256_point_add(r, r, &h); +} + +static crypto_word_t calc_first_wvalue(size_t *index, const uint8_t p_str[33]) { + static const size_t kWindowSize = 7; + static const crypto_word_t kMask = (1 << (7 /* kWindowSize */ + 1)) - 1; + *index = kWindowSize; + + crypto_word_t wvalue = (p_str[0] << 1) & kMask; + return booth_recode_w7(wvalue); +} + +static crypto_word_t calc_wvalue(size_t *index, const uint8_t p_str[33]) { + static const size_t kWindowSize = 7; + static const crypto_word_t kMask = (1 << (7 /* kWindowSize */ + 1)) - 1; + + const size_t off = (*index - 1) / 8; + crypto_word_t wvalue = + (crypto_word_t)p_str[off] | (crypto_word_t)p_str[off + 1] << 8; + wvalue = (wvalue >> ((*index - 1) % 8)) & kMask; + *index += kWindowSize; + + return booth_recode_w7(wvalue); +} + +static void ecp_nistz256_point_mul(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *p, + const EC_SCALAR *scalar) { + alignas(32) P256_POINT out; + ecp_nistz256_windowed_mul(group, &out, p, scalar); + + assert(group->field.N.width == P256_LIMBS); + OPENSSL_memcpy(r->X.words, out.X, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Y.words, out.Y, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Z.words, out.Z, P256_LIMBS * sizeof(BN_ULONG)); +} + +static void ecp_nistz256_point_mul_base(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_SCALAR *scalar) { + uint8_t p_str[33]; + OPENSSL_memcpy(p_str, scalar->words, 32); + p_str[32] = 0; + + // First window + size_t index = 0; + crypto_word_t wvalue = calc_first_wvalue(&index, p_str); + + alignas(32) P256_POINT_AFFINE t; + alignas(32) P256_POINT p; + ecp_nistz256_select_w7(&t, ecp_nistz256_precomputed[0], wvalue >> 1); + ecp_nistz256_neg(p.Z, t.Y); + copy_conditional(t.Y, p.Z, wvalue & 1); + + // Convert `t` from affine to Jacobian coordinates. We set Z to zero if `t` + // is infinity and `ONE_MONT` otherwise. `t` was computed from the table, so + // it is infinity iff `wvalue >> 1` is zero. + OPENSSL_memcpy(p.X, t.X, sizeof(p.X)); + OPENSSL_memcpy(p.Y, t.Y, sizeof(p.Y)); + OPENSSL_memset(p.Z, 0, sizeof(p.Z)); + copy_conditional(p.Z, ONE_MONT, is_not_zero(wvalue >> 1)); + + for (int i = 1; i < 37; i++) { + wvalue = calc_wvalue(&index, p_str); + + ecp_nistz256_select_w7(&t, ecp_nistz256_precomputed[i], wvalue >> 1); + + alignas(32) BN_ULONG neg_Y[P256_LIMBS]; + ecp_nistz256_neg(neg_Y, t.Y); + copy_conditional(t.Y, neg_Y, wvalue & 1); + + // Note `ecp_nistz256_point_add_affine` does not work if `p` and `t` are the + // same non-infinity point. + ecp_nistz256_point_add_affine(&p, &p, &t); + } + + assert(group->field.N.width == P256_LIMBS); + OPENSSL_memcpy(r->X.words, p.X, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Y.words, p.Y, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Z.words, p.Z, P256_LIMBS * sizeof(BN_ULONG)); +} + +static void ecp_nistz256_points_mul_public(const EC_GROUP *group, + EC_JACOBIAN *r, + const EC_SCALAR *g_scalar, + const EC_JACOBIAN *p_, + const EC_SCALAR *p_scalar) { + assert(p_ != nullptr && p_scalar != nullptr && g_scalar != nullptr); + + alignas(32) P256_POINT p; + uint8_t p_str[33]; + OPENSSL_memcpy(p_str, g_scalar->words, 32); + p_str[32] = 0; + + // First window + size_t index = 0; + size_t wvalue = calc_first_wvalue(&index, p_str); + + // Convert `p` from affine to Jacobian coordinates. We set Z to zero if `p` + // is infinity and `ONE_MONT` otherwise. `p` was computed from the table, so + // it is infinity iff `wvalue >> 1` is zero. + if ((wvalue >> 1) != 0) { + OPENSSL_memcpy(p.X, &ecp_nistz256_precomputed[0][(wvalue >> 1) - 1].X, + sizeof(p.X)); + OPENSSL_memcpy(p.Y, &ecp_nistz256_precomputed[0][(wvalue >> 1) - 1].Y, + sizeof(p.Y)); + OPENSSL_memcpy(p.Z, ONE_MONT, sizeof(p.Z)); + } else { + OPENSSL_memset(p.X, 0, sizeof(p.X)); + OPENSSL_memset(p.Y, 0, sizeof(p.Y)); + OPENSSL_memset(p.Z, 0, sizeof(p.Z)); + } + + if ((wvalue & 1) == 1) { + ecp_nistz256_neg(p.Y, p.Y); + } + + for (int i = 1; i < 37; i++) { + wvalue = calc_wvalue(&index, p_str); + if ((wvalue >> 1) == 0) { + continue; + } + + alignas(32) P256_POINT_AFFINE t; + OPENSSL_memcpy(&t, &ecp_nistz256_precomputed[i][(wvalue >> 1) - 1], + sizeof(t)); + if ((wvalue & 1) == 1) { + ecp_nistz256_neg(t.Y, t.Y); + } + + // Note `ecp_nistz256_point_add_affine` does not work if `p` and `t` are + // the same non-infinity point, so it is important that we compute the + // `g_scalar` term before the `p_scalar` term. + ecp_nistz256_point_add_affine(&p, &p, &t); + } + + alignas(32) P256_POINT tmp; + ecp_nistz256_windowed_mul(group, &tmp, p_, p_scalar); + ecp_nistz256_point_add(&p, &p, &tmp); + + assert(group->field.N.width == P256_LIMBS); + OPENSSL_memcpy(r->X.words, p.X, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Y.words, p.Y, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Z.words, p.Z, P256_LIMBS * sizeof(BN_ULONG)); +} + +static int ecp_nistz256_get_affine(const EC_GROUP *group, + const EC_JACOBIAN *point, EC_FELEM *x, + EC_FELEM *y) { + if (constant_time_declassify_int( + ec_GFp_simple_is_at_infinity(group, point))) { + OPENSSL_PUT_ERROR(EC, EC_R_POINT_AT_INFINITY); + return 0; + } + + BN_ULONG z_inv2[P256_LIMBS]; + assert(group->field.N.width == P256_LIMBS); + ecp_nistz256_mod_inverse_sqr_mont(z_inv2, point->Z.words); + + if (x != nullptr) { + ecp_nistz256_mul_mont(x->words, z_inv2, point->X.words); + } + + if (y != nullptr) { + ecp_nistz256_sqr_mont(z_inv2, z_inv2); // z^-4 + ecp_nistz256_mul_mont(y->words, point->Y.words, point->Z.words); // y * z + ecp_nistz256_mul_mont(y->words, y->words, z_inv2); // y * z^-3 + } + + return 1; +} + +static void ecp_nistz256_add(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *a_, const EC_JACOBIAN *b_) { + P256_POINT a, b; + OPENSSL_memcpy(a.X, a_->X.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(a.Y, a_->Y.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(a.Z, a_->Z.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(b.X, b_->X.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(b.Y, b_->Y.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(b.Z, b_->Z.words, P256_LIMBS * sizeof(BN_ULONG)); + ecp_nistz256_point_add(&a, &a, &b); + OPENSSL_memcpy(r->X.words, a.X, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Y.words, a.Y, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Z.words, a.Z, P256_LIMBS * sizeof(BN_ULONG)); +} + +static void ecp_nistz256_dbl(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *a_) { + P256_POINT a; + OPENSSL_memcpy(a.X, a_->X.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(a.Y, a_->Y.words, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(a.Z, a_->Z.words, P256_LIMBS * sizeof(BN_ULONG)); + ecp_nistz256_point_double(&a, &a); + OPENSSL_memcpy(r->X.words, a.X, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Y.words, a.Y, P256_LIMBS * sizeof(BN_ULONG)); + OPENSSL_memcpy(r->Z.words, a.Z, P256_LIMBS * sizeof(BN_ULONG)); +} + +static void ecp_nistz256_inv0_mod_ord(const EC_GROUP *group, EC_SCALAR *out, + const EC_SCALAR *in) { + // table[i] stores a power of `in` corresponding to the matching enum value. + enum { + // The following indices specify the power in binary. + i_1 = 0, + i_10, + i_11, + i_101, + i_111, + i_1010, + i_1111, + i_10101, + i_101010, + i_101111, + // The following indices specify 2^N-1, or N ones in a row. + i_x6, + i_x8, + i_x16, + i_x32 + }; + BN_ULONG table[15][P256_LIMBS]; + + // https://briansmith.org/ecc-inversion-addition-chains-01#p256_scalar_inversion + // + // Even though this code path spares 12 squarings, 4.5%, and 13 + // multiplications, 25%, the overall sign operation is not that much faster, + // not more that 2%. Most of the performance of this function comes from the + // scalar operations. + + // Pre-calculate powers. + OPENSSL_memcpy(table[i_1], in->words, P256_LIMBS * sizeof(BN_ULONG)); + + ecp_nistz256_ord_sqr_mont(table[i_10], table[i_1], 1); + + ecp_nistz256_ord_mul_mont(table[i_11], table[i_1], table[i_10]); + + ecp_nistz256_ord_mul_mont(table[i_101], table[i_11], table[i_10]); + + ecp_nistz256_ord_mul_mont(table[i_111], table[i_101], table[i_10]); + + ecp_nistz256_ord_sqr_mont(table[i_1010], table[i_101], 1); + + ecp_nistz256_ord_mul_mont(table[i_1111], table[i_1010], table[i_101]); + + ecp_nistz256_ord_sqr_mont(table[i_10101], table[i_1010], 1); + ecp_nistz256_ord_mul_mont(table[i_10101], table[i_10101], table[i_1]); + + ecp_nistz256_ord_sqr_mont(table[i_101010], table[i_10101], 1); + + ecp_nistz256_ord_mul_mont(table[i_101111], table[i_101010], table[i_101]); + + ecp_nistz256_ord_mul_mont(table[i_x6], table[i_101010], table[i_10101]); + + ecp_nistz256_ord_sqr_mont(table[i_x8], table[i_x6], 2); + ecp_nistz256_ord_mul_mont(table[i_x8], table[i_x8], table[i_11]); + + ecp_nistz256_ord_sqr_mont(table[i_x16], table[i_x8], 8); + ecp_nistz256_ord_mul_mont(table[i_x16], table[i_x16], table[i_x8]); + + ecp_nistz256_ord_sqr_mont(table[i_x32], table[i_x16], 16); + ecp_nistz256_ord_mul_mont(table[i_x32], table[i_x32], table[i_x16]); + + // Compute `in` raised to the order-2. + ecp_nistz256_ord_sqr_mont(out->words, table[i_x32], 64); + ecp_nistz256_ord_mul_mont(out->words, out->words, table[i_x32]); + static const struct { + uint8_t p, i; + } kChain[27] = {{32, i_x32}, {6, i_101111}, {5, i_111}, {4, i_11}, + {5, i_1111}, {5, i_10101}, {4, i_101}, {3, i_101}, + {3, i_101}, {5, i_111}, {9, i_101111}, {6, i_1111}, + {2, i_1}, {5, i_1}, {6, i_1111}, {5, i_111}, + {4, i_111}, {5, i_111}, {5, i_101}, {3, i_11}, + {10, i_101111}, {2, i_11}, {5, i_11}, {5, i_11}, + {3, i_1}, {7, i_10101}, {6, i_1111}}; + for (const auto &step : kChain) { + ecp_nistz256_ord_sqr_mont(out->words, out->words, step.p); + ecp_nistz256_ord_mul_mont(out->words, out->words, table[step.i]); + } +} + +static int ecp_nistz256_scalar_to_montgomery_inv_vartime(const EC_GROUP *group, + EC_SCALAR *out, + const EC_SCALAR *in) { +#if defined(OPENSSL_X86_64) + if (!CRYPTO_is_AVX_capable()) { + // No AVX support; fallback to generic code. + return ec_simple_scalar_to_montgomery_inv_vartime(group, out, in); + } +#endif + + assert(group->order.N.width == P256_LIMBS); + if (!beeu_mod_inverse_vartime(out->words, in->words, group->order.N.d)) { + return 0; + } + + // The result should be returned in the Montgomery domain. + ec_scalar_to_montgomery(group, out, out); + return 1; +} + +static int ecp_nistz256_cmp_x_coordinate(const EC_GROUP *group, + const EC_JACOBIAN *p, + const EC_SCALAR *r) { + if (ec_GFp_simple_is_at_infinity(group, p)) { + return 0; + } + + assert(group->order.N.width == P256_LIMBS); + assert(group->field.N.width == P256_LIMBS); + + // We wish to compare X/Z^2 with r. This is equivalent to comparing X with + // r*Z^2. Note that X and Z are represented in Montgomery form, while r is + // not. + BN_ULONG r_Z2[P256_LIMBS], Z2_mont[P256_LIMBS], X[P256_LIMBS]; + ecp_nistz256_mul_mont(Z2_mont, p->Z.words, p->Z.words); + ecp_nistz256_mul_mont(r_Z2, r->words, Z2_mont); + ecp_nistz256_from_mont(X, p->X.words); + + if (OPENSSL_memcmp(r_Z2, X, sizeof(r_Z2)) == 0) { + return 1; + } + + // During signing the x coefficient is reduced modulo the group order. + // Therefore there is a small possibility, less than 1/2^128, that group_order + // < p.x < P. in that case we need not only to compare against `r` but also to + // compare against r+group_order. + BN_ULONG carry = bn_add_words(r_Z2, r->words, group->order.N.d, P256_LIMBS); + if (carry == 0 && bn_less_than_words(r_Z2, group->field.N.d, P256_LIMBS)) { + // r + group_order < p, so compare (r + group_order) * Z^2 against X. + ecp_nistz256_mul_mont(r_Z2, r_Z2, Z2_mont); + if (OPENSSL_memcmp(r_Z2, X, sizeof(r_Z2)) == 0) { + return 1; + } + } + + return 0; +} + +BSSL_NAMESPACE_BEGIN + +DEFINE_METHOD_FUNCTION(EC_METHOD, EC_GFp_nistz256_method) { + out->point_get_affine_coordinates = ecp_nistz256_get_affine; + out->add = ecp_nistz256_add; + out->dbl = ecp_nistz256_dbl; + out->mul = ecp_nistz256_point_mul; + out->mul_base = ecp_nistz256_point_mul_base; + out->mul_public = ecp_nistz256_points_mul_public; + out->scalar_inv0_montgomery = ecp_nistz256_inv0_mod_ord; + out->scalar_to_montgomery_inv_vartime = + ecp_nistz256_scalar_to_montgomery_inv_vartime; + out->cmp_x_coordinate = ecp_nistz256_cmp_x_coordinate; +} + +BSSL_NAMESPACE_END + +#endif /* !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ + !defined(OPENSSL_SMALL) */
diff --git a/crypto/fipsmodule/ec/p256-nistz.h b/crypto/fipsmodule/ec/p256-nistz.h new file mode 100644 index 0000000..43da01c --- /dev/null +++ b/crypto/fipsmodule/ec/p256-nistz.h
@@ -0,0 +1,207 @@ +// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. +// Copyright (c) 2014, Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1) +// (1) Intel Corporation, Israel Development Center, Haifa, Israel +// (2) University of Haifa, Israel +// +// Reference: +// S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with +// 256 Bit Primes" + +#ifndef OPENSSL_HEADER_CRYPTO_FIPSMODULE_EC_P256_NISTZ_H +#define OPENSSL_HEADER_CRYPTO_FIPSMODULE_EC_P256_NISTZ_H + +#include <openssl/base.h> + +#include <openssl/bn.h> + +#include "../bn/internal.h" + +#if defined(__cplusplus) +extern "C" { +#endif + + +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ + !defined(OPENSSL_SMALL) + +// P-256 field operations. +// +// An element mod P in P-256 is represented as a little-endian array of +// `P256_LIMBS` `BN_ULONG`s, spanning the full range of values. +// +// The following functions take fully-reduced inputs mod P and give +// fully-reduced outputs. They may be used in-place. + +#define P256_LIMBS (256 / BN_BITS2) + +// ecp_nistz256_neg sets `res` to -`a` mod P. +void ecp_nistz256_neg(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]); + +// ecp_nistz256_mul_mont sets `res` to `a` * `b` * 2^-256 mod P. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_mul_mont_nohw(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +void ecp_nistz256_mul_mont_adx(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +#else +void ecp_nistz256_mul_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +#endif + +// ecp_nistz256_sqr_mont sets `res` to `a` * `a` * 2^-256 mod P. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_sqr_mont_nohw(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +void ecp_nistz256_sqr_mont_adx(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +#else +void ecp_nistz256_sqr_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS]); +#endif + + +// P-256 scalar operations. +// +// The following functions compute modulo N, where N is the order of P-256. They +// take fully-reduced inputs and give fully-reduced outputs. + +// ecp_nistz256_ord_mul_mont sets `res` to `a` * `b` where inputs and outputs +// are in Montgomery form. That is, `res` is `a` * `b` * 2^-256 mod N. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_ord_mul_mont_nohw(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +void ecp_nistz256_ord_mul_mont_adx(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +#else +void ecp_nistz256_ord_mul_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); +#endif + +// ecp_nistz256_ord_sqr_mont sets `res` to `a`^(2*`rep`) where inputs and +// outputs are in Montgomery form. That is, `res` is +// (`a` * 2^-256)^(2*`rep`) * 2^256 mod N. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_ord_sqr_mont_nohw(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], BN_ULONG rep); +void ecp_nistz256_ord_sqr_mont_adx(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], BN_ULONG rep); +#else +void ecp_nistz256_ord_sqr_mont(BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], BN_ULONG rep); +#endif + +// beeu_mod_inverse_vartime sets out = a^-1 mod p using a Euclidean algorithm. +// Assumption: 0 < a < p < 2^(256) and p is odd. +int beeu_mod_inverse_vartime(BN_ULONG out[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG p[P256_LIMBS]); + + +// P-256 point operations. +// +// The following functions may be used in-place. All coordinates are in the +// Montgomery domain. + +// A P256_POINT represents a P-256 point in Jacobian coordinates. +typedef struct { + BN_ULONG X[P256_LIMBS]; + BN_ULONG Y[P256_LIMBS]; + BN_ULONG Z[P256_LIMBS]; +} P256_POINT; + +// A P256_POINT_AFFINE represents a P-256 point in affine coordinates. Infinity +// is encoded as (0, 0). +typedef struct { + BN_ULONG X[P256_LIMBS]; + BN_ULONG Y[P256_LIMBS]; +} P256_POINT_AFFINE; + +// ecp_nistz256_select_w5 sets `*val` to `in_t[index-1]` if 1 <= `index` <= 16 +// and all zeros (the point at infinity) if `index` is 0. This is done in +// constant time. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_select_w5_nohw(P256_POINT *val, const P256_POINT in_t[16], + int index); +void ecp_nistz256_select_w5_avx2(P256_POINT *val, const P256_POINT in_t[16], + int index); +#else +void ecp_nistz256_select_w5(P256_POINT *val, const P256_POINT in_t[16], + int index); +#endif + +// ecp_nistz256_select_w7 sets `*val` to `in_t[index-1]` if 1 <= `index` <= 64 +// and all zeros (the point at infinity) if `index` is 0. This is done in +// constant time. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_select_w7_nohw(P256_POINT_AFFINE *val, + const P256_POINT_AFFINE in_t[64], int index); +void ecp_nistz256_select_w7_avx2(P256_POINT_AFFINE *val, + const P256_POINT_AFFINE in_t[64], int index); +#else +void ecp_nistz256_select_w7(P256_POINT_AFFINE *val, + const P256_POINT_AFFINE in_t[64], int index); +#endif + +// ecp_nistz256_point_double sets `r` to `a` doubled. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_point_double_nohw(P256_POINT *r, const P256_POINT *a); +void ecp_nistz256_point_double_adx(P256_POINT *r, const P256_POINT *a); +#else +void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a); +#endif + +// ecp_nistz256_point_add adds `a` to `b` and places the result in `r`. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_point_add_nohw(P256_POINT *r, const P256_POINT *a, + const P256_POINT *b); +void ecp_nistz256_point_add_adx(P256_POINT *r, const P256_POINT *a, + const P256_POINT *b); +#else +void ecp_nistz256_point_add(P256_POINT *r, const P256_POINT *a, + const P256_POINT *b); +#endif + +// ecp_nistz256_point_add_affine adds `a` to `b` and places the result in +// `r`. `a` and `b` must not represent the same point unless they are both +// infinity. +#if defined(OPENSSL_X86_64) +void ecp_nistz256_point_add_affine_adx(P256_POINT *r, const P256_POINT *a, + const P256_POINT_AFFINE *b); +void ecp_nistz256_point_add_affine_nohw(P256_POINT *r, const P256_POINT *a, + const P256_POINT_AFFINE *b); +#else +void ecp_nistz256_point_add_affine(P256_POINT *r, const P256_POINT *a, + const P256_POINT_AFFINE *b); +#endif + +#endif /* !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ + !defined(OPENSSL_SMALL) */ + + +#if defined(__cplusplus) +} // extern C +#endif + +#endif // OPENSSL_HEADER_CRYPTO_FIPSMODULE_EC_P256_NISTZ_H
diff --git a/crypto/fipsmodule/ec/p256-nistz_test.cc b/crypto/fipsmodule/ec/p256-nistz_test.cc new file mode 100644 index 0000000..e32894e --- /dev/null +++ b/crypto/fipsmodule/ec/p256-nistz_test.cc
@@ -0,0 +1,690 @@ +// Copyright 2016 The BoringSSL Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <openssl/base.h> + +#include <stdio.h> +#include <string.h> + +#include <gtest/gtest.h> + +#include <openssl/bn.h> +#include <openssl/ec.h> +#include <openssl/mem.h> +#include <openssl/nid.h> + +#include "../../internal.h" +#include "../../test/abi_test.h" +#include "../../test/file_test.h" +#include "../../test/test_util.h" +#include "../bn/internal.h" +#include "internal.h" +#include "p256-nistz.h" + + +BSSL_NAMESPACE_BEGIN +namespace { + +// Disable tests if BORINGSSL_SHARED_LIBRARY is defined. These tests need access +// to internal functions. +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ + !defined(OPENSSL_SMALL) && !defined(BORINGSSL_SHARED_LIBRARY) + +struct P256NistzSelectImpl { + const char *name; + void (*select_w5)(P256_POINT *val, const P256_POINT in_t[16], int index); + void (*select_w7)(P256_POINT_AFFINE *val, const P256_POINT_AFFINE in_t[64], + int index); +}; + +static std::vector<P256NistzSelectImpl> AllP256NistzSelectImpls() { + std::vector<P256NistzSelectImpl> impls; +#if defined(OPENSSL_X86_64) + impls.push_back({ + "NoHW", + ecp_nistz256_select_w5_nohw, + ecp_nistz256_select_w7_nohw, + }); + if (CRYPTO_is_AVX2_capable()) { + impls.push_back({ + "AVX2", + ecp_nistz256_select_w5_nohw, + ecp_nistz256_select_w7_nohw, + }); + } +#else + impls.push_back({ + "Impl", + ecp_nistz256_select_w5, + ecp_nistz256_select_w7, + }); +#endif + return impls; +} + +class P256NistzSelectImplTest : public testing::TestWithParam<P256NistzSelectImpl> { + protected: + const P256NistzSelectImpl &impl() const { return GetParam(); } +}; + +INSTANTIATE_TEST_SUITE_P(All, P256NistzSelectImplTest, + testing::ValuesIn(AllP256NistzSelectImpls()), + [](const testing::TestParamInfo<P256NistzSelectImpl> ¶ms) + -> std::string { return params.param.name; }); + +TEST_P(P256NistzSelectImplTest, SelectW5) { + // Fill a table with some garbage input. + alignas(64) P256_POINT table[16]; + for (size_t i = 0; i < 16; i++) { + OPENSSL_memset(table[i].X, static_cast<uint8_t>(3 * i), sizeof(table[i].X)); + OPENSSL_memset(table[i].Y, static_cast<uint8_t>(3 * i + 1), + sizeof(table[i].Y)); + OPENSSL_memset(table[i].Z, static_cast<uint8_t>(3 * i + 2), + sizeof(table[i].Z)); + } + + for (int i = 0; i <= 16; i++) { + P256_POINT val; + impl().select_w5(&val, table, i); + + P256_POINT expected; + if (i == 0) { + OPENSSL_memset(&expected, 0, sizeof(expected)); + } else { + expected = table[i-1]; + } + + EXPECT_EQ(Bytes(reinterpret_cast<const char *>(&expected), sizeof(expected)), + Bytes(reinterpret_cast<const char *>(&val), sizeof(val))); + } + + // This is a constant-time function, so it is only necessary to instrument one + // index for ABI checking. + P256_POINT val; + CHECK_ABI(impl().select_w5, &val, table, 7); +} + +TEST_P(P256NistzSelectImplTest, SelectW7) { + // Fill a table with some garbage input. + alignas(64) P256_POINT_AFFINE table[64]; + for (size_t i = 0; i < 64; i++) { + OPENSSL_memset(table[i].X, static_cast<uint8_t>(2 * i), sizeof(table[i].X)); + OPENSSL_memset(table[i].Y, static_cast<uint8_t>(2 * i + 1), + sizeof(table[i].Y)); + } + + for (int i = 0; i <= 64; i++) { + P256_POINT_AFFINE val; + impl().select_w7(&val, table, i); + + P256_POINT_AFFINE expected; + if (i == 0) { + OPENSSL_memset(&expected, 0, sizeof(expected)); + } else { + expected = table[i-1]; + } + + EXPECT_EQ(Bytes(reinterpret_cast<const char *>(&expected), sizeof(expected)), + Bytes(reinterpret_cast<const char *>(&val), sizeof(val))); + } + + // This is a constant-time function, so it is only necessary to instrument one + // index for ABI checking. + P256_POINT_AFFINE val; + CHECK_ABI(impl().select_w7, &val, table, 42); +} + + +static std::string FieldElementToString(const BN_ULONG a[P256_LIMBS]) { + std::string ret; + for (size_t i = P256_LIMBS-1; i < P256_LIMBS; i--) { + char buf[2 * BN_BYTES + 1]; + snprintf(buf, sizeof(buf), BN_HEX_FMT2, a[i]); + ret += buf; + } + return ret; +} + +static testing::AssertionResult ExpectFieldElementsEqual( + const char *expected_expr, const char *actual_expr, + const BN_ULONG expected[P256_LIMBS], const BN_ULONG actual[P256_LIMBS]) { + if (OPENSSL_memcmp(expected, actual, sizeof(BN_ULONG) * P256_LIMBS) == 0) { + return testing::AssertionSuccess(); + } + + return testing::AssertionFailure() + << "Expected: " << FieldElementToString(expected) << " (" + << expected_expr << ")\n" + << "Actual: " << FieldElementToString(actual) << " (" << actual_expr + << ")"; +} + +#define EXPECT_FIELD_ELEMENTS_EQUAL(a, b) \ + EXPECT_PRED_FORMAT2(ExpectFieldElementsEqual, a, b) + +// P-256 scalars and field elements are the same size. +#define EXPECT_SCALARS_EQUAL(a, b) EXPECT_FIELD_ELEMENTS_EQUAL(a, b) + +TEST(P256_NistzTest, BEEU) { +#if defined(OPENSSL_X86_64) + if (!CRYPTO_is_AVX_capable()) { + // No AVX support; cannot run the BEEU code. + return; + } +#endif + + const EC_GROUP *group = EC_group_p256(); + BN_ULONG order_words[P256_LIMBS]; + ASSERT_TRUE( + bn_copy_words(order_words, P256_LIMBS, EC_GROUP_get0_order(group))); + + BN_ULONG in[P256_LIMBS], out[P256_LIMBS]; + EC_SCALAR in_scalar, out_scalar, result; + OPENSSL_memset(in, 0, sizeof(in)); + + // Trying to find the inverse of zero should fail. + ASSERT_FALSE(beeu_mod_inverse_vartime(out, in, order_words)); + // This is not a constant-time function, so instrument both zero and a few + // inputs below. + ASSERT_FALSE(CHECK_ABI(beeu_mod_inverse_vartime, out, in, order_words)); + + BN_ULONG kOne[P256_LIMBS] = {}; + kOne[0] = 1; + + auto beeu_test = [&] { + EXPECT_TRUE(bn_less_than_words(in, order_words, P256_LIMBS)); + ASSERT_TRUE(beeu_mod_inverse_vartime(out, in, order_words)); + EXPECT_TRUE(bn_less_than_words(out, order_words, P256_LIMBS)); + + // Calculate out*in and confirm that it equals one, modulo the order. We + // only have Montgomery multiplication, but if we convert one input, and not + // the other, this gives the result outside the Montgomery domain. + OPENSSL_memcpy(in_scalar.words, in, sizeof(in)); + OPENSSL_memcpy(out_scalar.words, out, sizeof(out)); + ec_scalar_to_montgomery(group, &in_scalar, &in_scalar); + ec_scalar_mul_montgomery(group, &result, &in_scalar, &out_scalar); + EXPECT_SCALARS_EQUAL(kOne, result.words); + + // Invert the result and expect to get back to the original value. + ASSERT_TRUE(beeu_mod_inverse_vartime(out, out, order_words)); + EXPECT_SCALARS_EQUAL(in, out); + }; + + for (BN_ULONG i = 1; i < 2000; i++) { + SCOPED_TRACE(i); + in[0] = i; + if (i >= 1000) { + in[1] = i << 8; + in[2] = i << 32; + in[3] = i << 48; + } else { + in[1] = in[2] = in[3] = 0; + } + + beeu_test(); + if (i < 5) { + EXPECT_TRUE(CHECK_ABI(beeu_mod_inverse_vartime, out, in, order_words)); + } + } + + for (int i = 0; i < 255; i++) { + SCOPED_TRACE(i); + // Test `in` = 2^i. + OPENSSL_memset(in, 0, sizeof(in)); + in[i / BN_BITS2] = BN_ULONG{1} << (i % BN_BITS2); + beeu_test(); + + // Test `in` = N - 2^i. + bn_sub_words(in, order_words, in, P256_LIMBS); + beeu_test(); + } +} + +static bool GetFieldElement(FileTest *t, BN_ULONG out[P256_LIMBS], + const char *name) { + std::vector<uint8_t> bytes; + if (!t->GetBytes(&bytes, name)) { + return false; + } + + if (bytes.size() != BN_BYTES * P256_LIMBS) { + ADD_FAILURE() << "Invalid length: " << name; + return false; + } + + // `byte` contains bytes in big-endian while `out` should contain `BN_ULONG`s + // in little-endian. + OPENSSL_memset(out, 0, P256_LIMBS * sizeof(BN_ULONG)); + for (size_t i = 0; i < bytes.size(); i++) { + out[P256_LIMBS - 1 - (i / BN_BYTES)] <<= 8; + out[P256_LIMBS - 1 - (i / BN_BYTES)] |= bytes[i]; + } + + return true; +} + +static bool PointToAffine(P256_POINT_AFFINE *out, const P256_POINT *in) { + static const uint8_t kP[] = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; + + UniquePtr<BIGNUM> x(BN_new()), y(BN_new()), z(BN_new()); + UniquePtr<BIGNUM> p(BN_bin2bn(kP, sizeof(kP), nullptr)); + if (!x || !y || !z || !p || !bn_set_words(x.get(), in->X, P256_LIMBS) || + !bn_set_words(y.get(), in->Y, P256_LIMBS) || + !bn_set_words(z.get(), in->Z, P256_LIMBS)) { + return false; + } + + // Coordinates must be fully-reduced. + if (BN_cmp(x.get(), p.get()) >= 0 || + BN_cmp(y.get(), p.get()) >= 0 || + BN_cmp(z.get(), p.get()) >= 0) { + return false; + } + + if (BN_is_zero(z.get())) { + // The point at infinity is represented as (0, 0). + OPENSSL_memset(out, 0, sizeof(P256_POINT_AFFINE)); + return true; + } + + UniquePtr<BN_CTX> ctx(BN_CTX_new()); + UniquePtr<BN_MONT_CTX> mont(BN_MONT_CTX_new_for_modulus(p.get(), ctx.get())); + if (!ctx || !mont || + // Invert Z. + !BN_from_montgomery(z.get(), z.get(), mont.get(), ctx.get()) || + !BN_mod_inverse(z.get(), z.get(), p.get(), ctx.get()) || + !BN_to_montgomery(z.get(), z.get(), mont.get(), ctx.get()) || + // Convert (X, Y, Z) to (X/Z^2, Y/Z^3). + !BN_mod_mul_montgomery(x.get(), x.get(), z.get(), mont.get(), + ctx.get()) || + !BN_mod_mul_montgomery(x.get(), x.get(), z.get(), mont.get(), + ctx.get()) || + !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), + ctx.get()) || + !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), + ctx.get()) || + !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), + ctx.get()) || + !bn_copy_words(out->X, P256_LIMBS, x.get()) || + !bn_copy_words(out->Y, P256_LIMBS, y.get())) { + return false; + } + return true; +} + +static testing::AssertionResult ExpectPointsEqual( + const char *expected_expr, const char *actual_expr, + const P256_POINT_AFFINE *expected, const P256_POINT *actual) { + // There are multiple representations of the same `P256_POINT`, so convert to + // `P256_POINT_AFFINE` and compare. + P256_POINT_AFFINE affine; + if (!PointToAffine(&affine, actual)) { + return testing::AssertionFailure() + << "Could not convert " << actual_expr << " to affine: (" + << FieldElementToString(actual->X) << ", " + << FieldElementToString(actual->Y) << ", " + << FieldElementToString(actual->Z) << ")"; + } + + if (OPENSSL_memcmp(expected, &affine, sizeof(P256_POINT_AFFINE)) != 0) { + return testing::AssertionFailure() + << "Expected: (" << FieldElementToString(expected->X) << ", " + << FieldElementToString(expected->Y) << ") (" << expected_expr + << "; affine)\n" + << "Actual: (" << FieldElementToString(affine.X) << ", " + << FieldElementToString(affine.Y) << ") (" << actual_expr << ")"; + } + + return testing::AssertionSuccess(); +} + +#define EXPECT_POINTS_EQUAL(a, b) EXPECT_PRED_FORMAT2(ExpectPointsEqual, a, b) + +struct P256NistzImpl { + const char *name; + void (*neg)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]); + void (*mul_mont)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); + void (*sqr_mont)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]); + void (*ord_mul_mont)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]); + void (*ord_sqr_mont)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS], + BN_ULONG rep); + void (*point_double)(P256_POINT *r, const P256_POINT *a); + void (*point_add)(P256_POINT *r, const P256_POINT *a, const P256_POINT *b); + void (*point_add_affine)(P256_POINT *r, const P256_POINT *a, + const P256_POINT_AFFINE *b); +}; + +static std::vector<P256NistzImpl> AllP256NistzImpls() { + std::vector<P256NistzImpl> impls; +#if defined(OPENSSL_X86_64) + impls.push_back({ + "NoHW", + ecp_nistz256_neg, + ecp_nistz256_mul_mont_nohw, + ecp_nistz256_sqr_mont_nohw, + ecp_nistz256_ord_mul_mont_nohw, + ecp_nistz256_ord_sqr_mont_nohw, + ecp_nistz256_point_double_nohw, + ecp_nistz256_point_add_nohw, + ecp_nistz256_point_add_affine_nohw, + }); + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + impls.push_back({ + "ADX", + ecp_nistz256_neg, + ecp_nistz256_mul_mont_adx, + ecp_nistz256_sqr_mont_adx, + ecp_nistz256_ord_mul_mont_adx, + ecp_nistz256_ord_sqr_mont_adx, + ecp_nistz256_point_double_adx, + ecp_nistz256_point_add_adx, + ecp_nistz256_point_add_affine_adx, + }); + } +#else + impls.push_back({ + "Impl", + ecp_nistz256_neg, + ecp_nistz256_mul_mont, + ecp_nistz256_sqr_mont, + ecp_nistz256_ord_mul_mont, + ecp_nistz256_ord_sqr_mont, + ecp_nistz256_point_double, + ecp_nistz256_point_add, + ecp_nistz256_point_add_affine, + }); +#endif + return impls; +} + +class P256NistzImplTest : public testing::TestWithParam<P256NistzImpl> { + protected: + const P256NistzImpl &impl() const { return GetParam(); } +}; + +INSTANTIATE_TEST_SUITE_P(All, P256NistzImplTest, + testing::ValuesIn(AllP256NistzImpls()), + [](const testing::TestParamInfo<P256NistzImpl> ¶ms) + -> std::string { return params.param.name; }); + +static void TestNegate(FileTest *t, const P256NistzImpl &impl) { + BN_ULONG a[P256_LIMBS], b[P256_LIMBS]; + ASSERT_TRUE(GetFieldElement(t, a, "A")); + ASSERT_TRUE(GetFieldElement(t, b, "B")); + + // Test that -A = B. + BN_ULONG ret[P256_LIMBS]; + impl.neg(ret, a); + EXPECT_FIELD_ELEMENTS_EQUAL(b, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + impl.neg(ret, ret /* a */); + EXPECT_FIELD_ELEMENTS_EQUAL(b, ret); + + // Test that -B = A. + impl.neg(ret, b); + EXPECT_FIELD_ELEMENTS_EQUAL(a, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + impl.neg(ret, ret /* b */); + EXPECT_FIELD_ELEMENTS_EQUAL(a, ret); +} + +static void TestMulMont(FileTest *t, const P256NistzImpl &impl) { + BN_ULONG a[P256_LIMBS], b[P256_LIMBS], result[P256_LIMBS]; + ASSERT_TRUE(GetFieldElement(t, a, "A")); + ASSERT_TRUE(GetFieldElement(t, b, "B")); + ASSERT_TRUE(GetFieldElement(t, result, "Result")); + + BN_ULONG ret[P256_LIMBS]; + impl.mul_mont(ret, a, b); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + impl.mul_mont(ret, b, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + impl.mul_mont(ret, ret /* a */, b); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + impl.mul_mont(ret, b, ret); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + impl.mul_mont(ret, a, ret /* b */); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + impl.mul_mont(ret, ret /* b */, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { + impl.sqr_mont(ret, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + impl.sqr_mont(ret, ret /* a */); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + } +} + +static void TestPointAdd(FileTest *t, const P256NistzImpl &impl) { + P256_POINT a, b; + P256_POINT_AFFINE result; + ASSERT_TRUE(GetFieldElement(t, a.X, "A.X")); + ASSERT_TRUE(GetFieldElement(t, a.Y, "A.Y")); + ASSERT_TRUE(GetFieldElement(t, a.Z, "A.Z")); + ASSERT_TRUE(GetFieldElement(t, b.X, "B.X")); + ASSERT_TRUE(GetFieldElement(t, b.Y, "B.Y")); + ASSERT_TRUE(GetFieldElement(t, b.Z, "B.Z")); + ASSERT_TRUE(GetFieldElement(t, result.X, "Result.X")); + ASSERT_TRUE(GetFieldElement(t, result.Y, "Result.Y")); + + P256_POINT ret; + impl.point_add(&ret, &a, &b); + EXPECT_POINTS_EQUAL(&result, &ret); + + impl.point_add(&ret, &b, &a); + EXPECT_POINTS_EQUAL(&result, &ret); + + OPENSSL_memcpy(&ret, &a, sizeof(ret)); + impl.point_add(&ret, &ret /* a */, &b); + EXPECT_POINTS_EQUAL(&result, &ret); + + OPENSSL_memcpy(&ret, &a, sizeof(ret)); + impl.point_add(&ret, &b, &ret /* a */); + EXPECT_POINTS_EQUAL(&result, &ret); + + OPENSSL_memcpy(&ret, &b, sizeof(ret)); + impl.point_add(&ret, &a, &ret /* b */); + EXPECT_POINTS_EQUAL(&result, &ret); + + OPENSSL_memcpy(&ret, &b, sizeof(ret)); + impl.point_add(&ret, &ret /* b */, &a); + EXPECT_POINTS_EQUAL(&result, &ret); + + P256_POINT_AFFINE a_affine, b_affine, infinity; + OPENSSL_memset(&infinity, 0, sizeof(infinity)); + ASSERT_TRUE(PointToAffine(&a_affine, &a)); + ASSERT_TRUE(PointToAffine(&b_affine, &b)); + + // ecp_nistz256_point_add_affine does not work when a == b unless doubling the + // point at infinity. + if (OPENSSL_memcmp(&a_affine, &b_affine, sizeof(a_affine)) != 0 || + OPENSSL_memcmp(&a_affine, &infinity, sizeof(a_affine)) == 0) { + impl.point_add_affine(&ret, &a, &b_affine); + EXPECT_POINTS_EQUAL(&result, &ret); + + OPENSSL_memcpy(&ret, &a, sizeof(ret)); + impl.point_add_affine(&ret, &ret /* a */, &b_affine); + EXPECT_POINTS_EQUAL(&result, &ret); + + impl.point_add_affine(&ret, &b, &a_affine); + EXPECT_POINTS_EQUAL(&result, &ret); + + OPENSSL_memcpy(&ret, &b, sizeof(ret)); + impl.point_add_affine(&ret, &ret /* b */, &a_affine); + EXPECT_POINTS_EQUAL(&result, &ret); + } + + if (OPENSSL_memcmp(&a, &b, sizeof(a)) == 0) { + impl.point_double(&ret, &a); + EXPECT_POINTS_EQUAL(&result, &ret); + + ret = a; + impl.point_double(&ret, &ret /* a */); + EXPECT_POINTS_EQUAL(&result, &ret); + } +} + +static void TestOrdMulMont(FileTest *t, const P256NistzImpl &impl) { + // This test works on scalars rather than field elements, but the + // representation is the same. + BN_ULONG a[P256_LIMBS], b[P256_LIMBS], result[P256_LIMBS]; + ASSERT_TRUE(GetFieldElement(t, a, "A")); + ASSERT_TRUE(GetFieldElement(t, b, "B")); + ASSERT_TRUE(GetFieldElement(t, result, "Result")); + + BN_ULONG ret[P256_LIMBS]; + impl.ord_mul_mont(ret, a, b); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + impl.ord_mul_mont(ret, b, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + impl.ord_mul_mont(ret, ret /* a */, b); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + impl.ord_mul_mont(ret, b, ret); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + impl.ord_mul_mont(ret, a, ret /* b */); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + impl.ord_mul_mont(ret, ret /* b */, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { + impl.ord_sqr_mont(ret, a, 1); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + impl.ord_sqr_mont(ret, ret /* a */, 1); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + } +} + +TEST_P(P256NistzImplTest, TestVectors) { + return FileTestGTest("crypto/fipsmodule/ec/p256-nistz_tests.txt", + [&](FileTest *t) { + if (t->GetParameter() == "Negate") { + TestNegate(t, impl()); + } else if (t->GetParameter() == "MulMont") { + TestMulMont(t, impl()); + } else if (t->GetParameter() == "PointAdd") { + TestPointAdd(t, impl()); + } else if (t->GetParameter() == "OrdMulMont") { + TestOrdMulMont(t, impl()); + } else { + FAIL() << "Unknown test type:" << t->GetParameter(); + } + }); +} + +// Instrument the functions covered in TestVectors for ABI checking. +TEST_P(P256NistzImplTest, ABI) { + BN_ULONG a[P256_LIMBS], b[P256_LIMBS], c[P256_LIMBS]; + OPENSSL_memset(a, 0x01, sizeof(a)); + // These functions are all branchless, so it is only necessary to + // instrument one call each for ABI checking. + CHECK_ABI(impl().neg, b, a); + CHECK_ABI(impl().mul_mont, c, a, b); + CHECK_ABI(impl().sqr_mont, c, a); + CHECK_ABI(impl().ord_mul_mont, c, a, b); + + // Check a few different loop counts. + CHECK_ABI(impl().ord_sqr_mont, b, a, 1); + CHECK_ABI(impl().ord_sqr_mont, b, a, 3); + + // Point addition has some special cases around infinity and doubling. Test + // a few different scenarios. + static const P256_POINT kA = { + {TOBN(0x60559ac7, 0xc8d0d89d), TOBN(0x6cda3400, 0x545f7e2c), + TOBN(0x9b5159e0, 0x323e6048), TOBN(0xcb8dea33, 0x27057fe6)}, + {TOBN(0x81a2d3bc, 0xc93a2d53), TOBN(0x81f40762, 0xa4f33ccf), + TOBN(0xc3c3300a, 0xa8ad50ea), TOBN(0x553de89b, 0x31719830)}, + {TOBN(0x3fd9470f, 0xb277d181), TOBN(0xc191b8d5, 0x6376f206), + TOBN(0xb2572c1f, 0x45eda26f), TOBN(0x4589e40d, 0xf2efc546)}, + }; + static const P256_POINT kB = { + {TOBN(0x3cf0b0aa, 0x92054341), TOBN(0xb949bb80, 0xdab57807), + TOBN(0x99de6814, 0xefd21b3e), TOBN(0x32ad5649, 0x7c6c6e83)}, + {TOBN(0x06afaa02, 0x688399e0), TOBN(0x75f2d096, 0x2a3ce65c), + TOBN(0xf6a31eb7, 0xca0244b3), TOBN(0x57b33b7a, 0xcfeee75e)}, + {TOBN(0x7617d2e0, 0xb4f1d35f), TOBN(0xa922cb10, 0x7f592b65), + TOBN(0x12fd6c7a, 0x51a2f474), TOBN(0x337d5e1e, 0xc2fc711b)}, + }; + // This file represents Jacobian infinity as (*, *, 0). + static const P256_POINT kInfinity = { + {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, + {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, + {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, + }; + + P256_POINT p; + CHECK_ABI(impl().point_add, &p, &kA, &kB); + CHECK_ABI(impl().point_add, &p, &kA, &kA); + OPENSSL_memcpy(&p, &kA, sizeof(P256_POINT)); + impl().neg(p.Y, p.Y); + CHECK_ABI(impl().point_add, &p, &kA, &p); // A + -A + CHECK_ABI(impl().point_add, &p, &kA, &kInfinity); + CHECK_ABI(impl().point_add, &p, &kInfinity, &kA); + CHECK_ABI(impl().point_add, &p, &kInfinity, &kInfinity); + CHECK_ABI(impl().point_double, &p, &kA); + CHECK_ABI(impl().point_double, &p, &kInfinity); + + static const P256_POINT_AFFINE kC = { + {TOBN(0x7e3ad339, 0xfb3fa5f0), TOBN(0x559d669d, 0xe3a047b2), + TOBN(0x8883b298, 0x7042e595), TOBN(0xfabada65, 0x7e477f08)}, + {TOBN(0xd9cfceb8, 0xda1c3e85), TOBN(0x80863761, 0x0ce6d6bc), + TOBN(0xa8409d84, 0x66034f02), TOBN(0x05519925, 0x31a68d55)}, + }; + // This file represents affine infinity as (0, 0). + static const P256_POINT_AFFINE kInfinityAffine = { + {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, + {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, + }; + + CHECK_ABI(impl().point_add_affine, &p, &kA, &kC); + CHECK_ABI(impl().point_add_affine, &p, &kA, &kInfinityAffine); + CHECK_ABI(impl().point_add_affine, &p, &kInfinity, &kInfinityAffine); + CHECK_ABI(impl().point_add_affine, &p, &kInfinity, &kC); +} + +#endif + +} // namespace +BSSL_NAMESPACE_END
diff --git a/crypto/fipsmodule/ec/p256-tests.txt b/crypto/fipsmodule/ec/p256-nistz_tests.txt similarity index 100% rename from crypto/fipsmodule/ec/p256-tests.txt rename to crypto/fipsmodule/ec/p256-nistz_tests.txt
diff --git a/crypto/fipsmodule/ec/p256.cc.inc b/crypto/fipsmodule/ec/p256.cc.inc index 6ce9471..a67923d 100644 --- a/crypto/fipsmodule/ec/p256.cc.inc +++ b/crypto/fipsmodule/ec/p256.cc.inc
@@ -1,6 +1,4 @@ // Copyright 2020 The BoringSSL Authors -// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. -// Copyright (c) 2014, Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,346 +12,81 @@ // See the License for the specific language governing permissions and // limitations under the License. +// An implementation of the NIST P-256 elliptic curve point multiplication. +// 256-bit Montgomery form for 64 and 32-bit. Field operations are generated by +// Fiat, which lives in //third_party/fiat. + #include <openssl/base.h> #include <openssl/bn.h> #include <openssl/ec.h> +#include <openssl/err.h> #include <openssl/mem.h> #include <assert.h> +#include <string.h> #include <iterator> #include "../../internal.h" #include "../delocate.h" -#include "internal.h" +#include "./internal.h" -#include "p256_internal.h" +#include "../../../third_party/fiat/p256_field.c.inc" +#include "../../../third_party/fiat/p256_point.br.c.inc" + using namespace bssl; -static void fiat_p256_opp_conditional(fiat_p256_felem x, crypto_word_t c) { - alignas(32) fiat_p256_felem n; - fiat_p256_opp(n, x); - for (size_t i = 0; i < P256_LIMBS; ++i) { - x[i] = br_cmov(c, n[i], x[i]); +// utility functions, handwritten + +#if defined(OPENSSL_64_BIT) +#define FIAT_P256_NLIMBS 4 +typedef uint64_t fiat_p256_limb_t; +typedef uint64_t fiat_p256_felem[FIAT_P256_NLIMBS]; +static const fiat_p256_felem fiat_p256_one = {0x1, 0xffffffff00000000, + 0xffffffffffffffff, 0xfffffffe}; +#else // 64BIT; else 32BIT +#define FIAT_P256_NLIMBS 8 +typedef uint32_t fiat_p256_limb_t; +typedef uint32_t fiat_p256_felem[FIAT_P256_NLIMBS]; +static const fiat_p256_felem fiat_p256_one = { + 0x1, 0x0, 0x0, 0xffffffff, 0xffffffff, 0xffffffff, 0xfffffffe, 0x0}; +#endif // 64BIT + + +static void fiat_p256_copy(fiat_p256_limb_t out[FIAT_P256_NLIMBS], + const fiat_p256_limb_t in1[FIAT_P256_NLIMBS]) { + for (size_t i = 0; i < FIAT_P256_NLIMBS; i++) { + out[i] = in1[i]; } } -// bit returns the `i`th bit in `in`. -static crypto_word_t bit(const uint8_t in[32], int i) { - if (i < 0 || i >= 256) { - return 0; - } - return (in[i >> 3] >> (i & 7)) & 1; +static void fiat_p256_cmovznz(fiat_p256_limb_t out[FIAT_P256_NLIMBS], + fiat_p256_limb_t t, + const fiat_p256_limb_t z[FIAT_P256_NLIMBS], + const fiat_p256_limb_t nz[FIAT_P256_NLIMBS]) { + fiat_p256_selectznz(out, !!t, z, nz); } -// point multiplication -static void fiat_p256_select_point_16(fiat_p256_felem dst[3], - const fiat_p256_felem src[16][3], - size_t i) { - OPENSSL_memset(dst, 0, 3 * sizeof(fiat_p256_felem)); - OPENSSL_CLANG_PRAGMA("clang loop unroll_count(4)") - for (size_t j = 0; j < 16; j++) { - constant_time_conditional_memxor(dst, &src[j], 3 * sizeof(fiat_p256_felem), - constant_time_eq_w(i, j)); - } +static void fiat_p256_from_words(fiat_p256_felem out, + const BN_ULONG in[32 / sizeof(BN_ULONG)]) { + // Typically, `BN_ULONG` and `fiat_p256_limb_t` will be the same type, but on + // 64-bit platforms without `uint128_t`, they are different. However, on + // little-endian systems, `uint64_t[4]` and `uint32_t[8]` have the same + // layout. + OPENSSL_memcpy(out, in, 32); } -// Precompute multiples of `p`. p_pre_comp[i] is (i+1) * `p`. -static void p256_point_mul(fiat_p256_felem out[3], const fiat_p256_felem p[3], - const uint8_t s[32]) { - alignas(32) fiat_p256_felem p_pre_comp[16][3]; - OPENSSL_memcpy(p_pre_comp[0], p, sizeof(p_pre_comp[0])); - for (size_t j = 2; j <= 16; ++j) { - if (j & 1) { - p256_point_add_vartime_if_doubling((uintptr_t)p_pre_comp[j - 1], - (uintptr_t)p_pre_comp[j - 2], - (uintptr_t)p_pre_comp[0]); - } else { - p256_point_double((uintptr_t)p_pre_comp[j - 1], - (uintptr_t)p_pre_comp[(j - 1) / 2]); - } - } - alignas(32) fiat_p256_felem ret[3]; - bool ret_is_zero = true; - - for (size_t i = 51; i < 52; i--) { - if (!ret_is_zero) { - for (size_t k = 4; k < 5; k--) { - p256_point_double((uintptr_t)ret, (uintptr_t)ret); - } - } - - crypto_word_t bits = 0; - OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") - for (size_t k = 5; k < 6; k--) { - bits |= bit(s, i * 5 - 1 + k) << k; - } - crypto_word_t sign, digit; - ec_GFp_nistp_recode_scalar_bits(&sign, &digit, bits); - - alignas(32) fiat_p256_felem t[3]; - fiat_p256_select_point_16(t, p_pre_comp, digit - 1); - fiat_p256_opp_conditional(t[1], sign); - - if (!ret_is_zero) { - p256_point_add_vartime_if_doubling((uintptr_t)ret, (uintptr_t)ret, - (uintptr_t)t); - } else { - OPENSSL_memcpy(ret, t, sizeof(ret)); - ret_is_zero = false; - } - } - // The first loop iteration will initialize `ret`. - assert(!ret_is_zero); - - OPENSSL_memcpy(out, ret, sizeof(ret)); +static void fiat_p256_from_generic(fiat_p256_felem out, const EC_FELEM *in) { + fiat_p256_from_words(out, in->words); } -// point_mul_public -#include "./p256_table.h" - -static void ec_GFp_nistp256_point_mul_public(const EC_GROUP *group, - EC_JACOBIAN *r, - const EC_SCALAR *gs, - const EC_JACOBIAN *p, - const EC_SCALAR *ps) { - const uint8_t *g_scalar = (uint8_t *)gs->words; - int8_t p_wNAF[257] = {0}; - - alignas(32) fiat_p256_felem p_pre_comp[1 << (4 - 1)][3]; - OPENSSL_memcpy(p_pre_comp[0][2], &p->Z, 32); - if (!p256_point_iszero((uintptr_t)p_pre_comp[0])) { - ec_compute_wNAF(group, p_wNAF, ps, /*bits=*/256, /*w=*/4); - // Precompute multiples of `p`. p_pre_comp[i] is (2*i+1) * `p`. - OPENSSL_memcpy(p_pre_comp[0][0], &p->X, 32); - OPENSSL_memcpy(p_pre_comp[0][1], &p->Y, 32); - alignas(32) fiat_p256_felem p2[3]; - p256_point_double((uintptr_t)p2, (uintptr_t)p_pre_comp[0]); - for (size_t i = 1; i < std::size(p_pre_comp); i++) { - p256_point_add_nz_nz_neq((uintptr_t)p_pre_comp[i], - (uintptr_t)p_pre_comp[i - 1], (uintptr_t)p2); - } - } - - alignas(32) fiat_p256_felem ret[3] = {}; - bool ret_is_zero = true; // Save some point operations, avoid 0+Q - for (int i = 256; i >= 0; i--) { - if (!ret_is_zero) { - p256_point_double((uintptr_t)ret, (uintptr_t)ret); - } - - if (i <= 31) { - OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") - for (size_t j = 1; j < 2; j--) { - crypto_word_t bits = 0; - OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") - for (size_t k = 3; k < 4; k--) { - bits |= bit(g_scalar, i + j * 32 + k * 64) << k; - } - if (bits != 0) { - if (!ret_is_zero) { - alignas(32) fiat_p256_felem t[3]; - fiat_p256_set_one(t[2]); - OPENSSL_memcpy(t, fiat_p256_g_pre_comp[j][bits - 1], 64); - p256_point_add_affinenz_conditional_vartime_if_doubling( - (uintptr_t)ret, (uintptr_t)ret, (uintptr_t)t, 1); - ret_is_zero = p256_point_iszero((uintptr_t)ret); - } else { - OPENSSL_memcpy(ret, fiat_p256_g_pre_comp[j][bits - 1], - sizeof(fiat_p256_g_pre_comp[j][bits - 1])); - fiat_p256_set_one(ret[2]); - ret_is_zero = false; - } - } - } - } - - int digit = p_wNAF[i]; - if (digit != 0) { - assert(digit & 1); - size_t idx = (size_t)(digit < 0 ? (-digit) >> 1 : digit >> 1); - fiat_p256_felem t[3]; - OPENSSL_memcpy(t, p_pre_comp[idx], sizeof(t)); - if (digit < 0) { - fiat_p256_opp(t[1], t[1]); - } - if (!ret_is_zero) { - p256_point_add_vartime_if_doubling((uintptr_t)ret, (uintptr_t)ret, - (uintptr_t)t); - ret_is_zero = p256_point_iszero((uintptr_t)ret); - } else { - OPENSSL_memcpy(ret, t, sizeof(ret)); - ret_is_zero = false; - } - } - } - - OPENSSL_memcpy(&r->X, ret[0], 32); - OPENSSL_memcpy(&r->Y, ret[1], 32); - OPENSSL_memcpy(&r->Z, ret[2], 32); +static void fiat_p256_to_generic(EC_FELEM *out, const fiat_p256_felem in) { + // See `fiat_p256_from_words`. + OPENSSL_memcpy(out->words, in, 32); } -// p256_point_mul_base (small and full) - -// fiat_p256_select_point_affine selects the `i`th point from a precomputation -// table and copies it to `dst`. If `n<=i`, the output is (0, 0). -static void fiat_p256_select_point_affine(fiat_p256_felem dst[2], - const fiat_p256_felem src[/*n*/][2], - size_t i, size_t n) { - OPENSSL_memset(dst, 0, 2 * sizeof(fiat_p256_felem)); - OPENSSL_CLANG_PRAGMA("clang loop unroll_count(8)") - for (size_t j = 0; j < n; j++) { - constant_time_conditional_memxor(dst, &src[j], 2 * sizeof(fiat_p256_felem), - constant_time_eq_w(i, j)); - } -} - -static void fiat_p256_conditional_zero_or_one(fiat_p256_felem out, - crypto_word_t c) { - fiat_p256_set_one(out); - crypto_word_t mask = value_barrier_w(~constant_time_is_zero_w(c)); - for (size_t i = 0; i < P256_LIMBS; ++i) { - out[i] &= mask; - } -} - -#if defined(OPENSSL_SMALL) - -#ifdef __clang__ -__attribute__((noinline)) // Do not inline into unrolled loop below. -#endif -static void fiat_p256_select_point_affine_15( - fiat_p256_felem dst[2], const fiat_p256_felem src[/*n*/][2], size_t i) { - fiat_p256_select_point_affine(dst, src, i, 15); // Intended to be inlined. -} - -static void p256_point_mul_base(fiat_p256_felem ret[3], const uint8_t s[32]) { - bool ret_is_zero = true; // Save two point operations in the first round. - alignas(32) fiat_p256_felem t[3]; - fiat_p256_set_one(t[2]); - for (size_t i = 31; i < 32; i--) { - if (!ret_is_zero) { - p256_point_double((uintptr_t)ret, (uintptr_t)ret); - } - OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") - for (size_t j = 1; j < 2; j--) { - crypto_word_t bits = 0; - OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") - for (size_t k = 3; k < 4; k--) { - bits |= bit(s, i + 32 * j + 64 * k) << k; - } - fiat_p256_select_point_affine_15(t, fiat_p256_g_pre_comp[j], bits - 1); - - if (!ret_is_zero) { - p256_point_add_affinenz_conditional_vartime_if_doubling( - (uintptr_t)ret, (uintptr_t)ret, (uintptr_t)t, (uintptr_t)bits); - } else { - OPENSSL_memcpy(ret, t, sizeof(t)); - fiat_p256_conditional_zero_or_one(ret[2], bits); - ret_is_zero = false; - } - } - } - - // The first loop iteration will initialize `ret`. - assert(!ret_is_zero); -} -#else // defined(OPENSSL_SMALL) - -/* - * Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1) - * (1) Intel Corporation, Israel Development Center, Haifa, Israel - * (2) University of Haifa, Israel - * - * Reference: - * S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with - * 256 Bit Primes" - */ - -// Precomputed tables for the default generator -typedef fiat_p256_felem PRECOMP256_ROW[64][2]; -#include "p256-nistz-table.h" - -#ifdef __clang__ -__attribute__((noinline)) // Do not inline into unrolled loop below. -#endif -static void fiat_p256_select_point_affine_16(fiat_p256_felem dst[2], - const fiat_p256_felem src[16][2], - size_t i) { - fiat_p256_select_point_affine(dst, src, i, 16); -} - -#ifdef __clang__ -__attribute__((noinline)) // Do not inline into unrolled loop below. -#endif -static void fiat_p256_select_point_affine_64(fiat_p256_felem dst[2], - const fiat_p256_felem src[64][2], - size_t i) { - fiat_p256_select_point_affine(dst, src, i, 64); -} - -// See `ec_GFp_nistp_recode_scalar_bits` in util.c for details -static crypto_word_t booth_recode_w7(crypto_word_t in) { - crypto_word_t s, d; - s = ~((in >> 7) - 1); - d = (1 << 8) - in - 1; - d = (d & s) | (in & ~s); - d = (d >> 1) + (d & 1); - return (d << 1) + (s & 1); -} - -static void p256_point_mul_base(fiat_p256_felem ret[3], const uint8_t s[32]) { - bool ret_is_zero = true; - alignas(32) fiat_p256_felem t[3]; - fiat_p256_set_one(t[2]); - for (size_t i = 36; i < 37; i--) { - // Load 7-bit windows, plus one bit below the window for `booth_recode_w7`, - // i.e. 8 bits from 7 * i - 1 to 7 * (i + 1). - constexpr size_t kMask = (1 << (7 + 1)) - 1; - crypto_word_t wvalue; - if (i == 0) { - wvalue = (s[0] << 1) & kMask; - } else { - size_t first_bit = 7 * i - 1; - size_t idx = first_bit / 8; - // The window may span two bytes. - wvalue = - s[idx] | (static_cast<crypto_word_t>(idx < 31 ? s[idx + 1] : 0) << 8); - wvalue = (wvalue >> (first_bit % 8)) & kMask; - } - wvalue = booth_recode_w7(wvalue); - if (i == 36) { - // The last window has only 4 bits instead of the full 7. - declassify_assert((wvalue >> 1) <= 16); - fiat_p256_select_point_affine_16(t, ecp_nistz256_precomputed[i], - (wvalue >> 1) - 1); - } else { - fiat_p256_select_point_affine_64(t, ecp_nistz256_precomputed[i], - (wvalue >> 1) - 1); - } - fiat_p256_opp_conditional(t[1], wvalue & 1); - - if (!ret_is_zero) { - p256_point_add_affinenz_conditional_vartime_if_doubling( - (uintptr_t)ret, (uintptr_t)ret, (uintptr_t)t, wvalue >> 1); - } else { - OPENSSL_memcpy(ret, t, sizeof(t)); - fiat_p256_conditional_zero_or_one(ret[2], wvalue >> 1); - ret_is_zero = false; - } - } - - // The first loop iteration will initialize `ret`. - assert(!ret_is_zero); -} -#endif // !OPENSSL_SMALL - - -// FIELD-ELEMENT INVERSION - - // fiat_p256_inv_square calculates `out` = `in`^{-2} // // Based on Fermat's Little Theorem: @@ -365,182 +98,156 @@ // This implements the addition chain described in // https://briansmith.org/ecc-inversion-addition-chains-01#p256_field_inversion fiat_p256_felem x2, x3, x6, x12, x15, x30, x32; - p256_coord_sqr(x2, in); // 2^2 - 2^1 - p256_coord_mul(x2, x2, in); // 2^2 - 2^0 + fiat_p256_square(x2, in); // 2^2 - 2^1 + fiat_p256_mul(x2, x2, in); // 2^2 - 2^0 - p256_coord_sqr(x3, x2); // 2^3 - 2^1 - p256_coord_mul(x3, x3, in); // 2^3 - 2^0 + fiat_p256_square(x3, x2); // 2^3 - 2^1 + fiat_p256_mul(x3, x3, in); // 2^3 - 2^0 - p256_coord_sqr(x6, x3); + fiat_p256_square(x6, x3); for (int i = 1; i < 3; i++) { - p256_coord_sqr(x6, x6); // 2^6 - 2^3 - } - p256_coord_mul(x6, x6, x3); // 2^6 - 2^0 + fiat_p256_square(x6, x6); + } // 2^6 - 2^3 + fiat_p256_mul(x6, x6, x3); // 2^6 - 2^0 - p256_coord_sqr(x12, x6); + fiat_p256_square(x12, x6); for (int i = 1; i < 6; i++) { - p256_coord_sqr(x12, x12); // 2^12 - 2^6 - } - p256_coord_mul(x12, x12, x6); // 2^12 - 2^0 + fiat_p256_square(x12, x12); + } // 2^12 - 2^6 + fiat_p256_mul(x12, x12, x6); // 2^12 - 2^0 - p256_coord_sqr(x15, x12); + fiat_p256_square(x15, x12); for (int i = 1; i < 3; i++) { - p256_coord_sqr(x15, x15); // 2^15 - 2^3 - } - p256_coord_mul(x15, x15, x3); // 2^15 - 2^0 + fiat_p256_square(x15, x15); + } // 2^15 - 2^3 + fiat_p256_mul(x15, x15, x3); // 2^15 - 2^0 - p256_coord_sqr(x30, x15); + fiat_p256_square(x30, x15); for (int i = 1; i < 15; i++) { - p256_coord_sqr(x30, x30); // 2^30 - 2^15 - } - p256_coord_mul(x30, x30, x15); // 2^30 - 2^0 + fiat_p256_square(x30, x30); + } // 2^30 - 2^15 + fiat_p256_mul(x30, x30, x15); // 2^30 - 2^0 - p256_coord_sqr(x32, x30); - p256_coord_sqr(x32, x32); // 2^32 - 2^2 - p256_coord_mul(x32, x32, x2); // 2^32 - 2^0 + fiat_p256_square(x32, x30); + fiat_p256_square(x32, x32); // 2^32 - 2^2 + fiat_p256_mul(x32, x32, x2); // 2^32 - 2^0 fiat_p256_felem ret; - p256_coord_sqr(ret, x32); + fiat_p256_square(ret, x32); for (int i = 1; i < 31 + 1; i++) { - p256_coord_sqr(ret, ret); // 2^64 - 2^32 - } - p256_coord_mul(ret, ret, in); // 2^64 - 2^32 + 2^0 + fiat_p256_square(ret, ret); + } // 2^64 - 2^32 + fiat_p256_mul(ret, ret, in); // 2^64 - 2^32 + 2^0 for (int i = 0; i < 96 + 32; i++) { - p256_coord_sqr(ret, ret); // 2^192 - 2^160 + 2^128 - } - p256_coord_mul(ret, ret, x32); // 2^192 - 2^160 + 2^128 + 2^32 - 2^0 + fiat_p256_square(ret, ret); + } // 2^192 - 2^160 + 2^128 + fiat_p256_mul(ret, ret, x32); // 2^192 - 2^160 + 2^128 + 2^32 - 2^0 for (int i = 0; i < 32; i++) { - p256_coord_sqr(ret, ret); // 2^224 - 2^192 + 2^160 + 2^64 - 2^32 - } - p256_coord_mul(ret, ret, x32); // 2^224 - 2^192 + 2^160 + 2^64 - 2^0 + fiat_p256_square(ret, ret); + } // 2^224 - 2^192 + 2^160 + 2^64 - 2^32 + fiat_p256_mul(ret, ret, x32); // 2^224 - 2^192 + 2^160 + 2^64 - 2^0 for (int i = 0; i < 30; i++) { - p256_coord_sqr(ret, ret); // 2^254 - 2^222 + 2^190 + 2^94 - 2^30 - } - p256_coord_mul(ret, ret, x30); // 2^254 - 2^222 + 2^190 + 2^94 - 2^0 + fiat_p256_square(ret, ret); + } // 2^254 - 2^222 + 2^190 + 2^94 - 2^30 + fiat_p256_mul(ret, ret, x30); // 2^254 - 2^222 + 2^190 + 2^94 - 2^0 - p256_coord_sqr(ret, ret); - p256_coord_sqr(out, ret); // 2^256 - 2^224 + 2^192 + 2^96 - 2^2 + fiat_p256_square(ret, ret); + fiat_p256_square(out, ret); // 2^256 - 2^224 + 2^192 + 2^96 - 2^2 } +// Group operations +// ---------------- +// +// Building on top of the field operations we have the operations on the +// elliptic curve group itself. Points on the curve are represented in Jacobian +// coordinates. -// Arithmetic modulo curve order +static void fiat_p256_point_double(fiat_p256_felem x_out, fiat_p256_felem y_out, + fiat_p256_felem z_out, + const fiat_p256_felem x_in, + const fiat_p256_felem y_in, + const fiat_p256_felem z_in) { + uint8_t out[3*32], in[3*32]; + static_assert(sizeof(fiat_p256_felem) == 32); + OPENSSL_memcpy(&in[0], x_in, 32); + OPENSSL_memcpy(&in[32], y_in, 32); + OPENSSL_memcpy(&in[64], z_in, 32); + p256_point_double((br_word_t)out, (br_word_t)in); + OPENSSL_memcpy(x_out, &out[0], 32); + OPENSSL_memcpy(y_out, &out[32], 32); + OPENSSL_memcpy(z_out, &out[64], 32); +} +static void fiat_p256_point_add(fiat_p256_felem x3, fiat_p256_felem y3, + fiat_p256_felem z3, const fiat_p256_felem x1, + const fiat_p256_felem y1, + const fiat_p256_felem z1, + const fiat_p256_felem x2, + const fiat_p256_felem y2, + const fiat_p256_felem z2) { + uint8_t out[3 * 32], in1[3 * 32], in2[3 * 32]; + static_assert(sizeof(fiat_p256_felem) == 32); + OPENSSL_memcpy(&in1[0], x1, 32); + OPENSSL_memcpy(&in1[32], y1, 32); + OPENSSL_memcpy(&in1[64], z1, 32); + OPENSSL_memcpy(&in2[0], x2, 32); + OPENSSL_memcpy(&in2[32], y2, 32); + OPENSSL_memcpy(&in2[64], z2, 32); + p256_point_add_vartime_if_doubling((br_word_t)out, (br_word_t)in1, + (br_word_t)in2); + OPENSSL_memcpy(x3, &out[0], 32); + OPENSSL_memcpy(y3, &out[32], 32); + OPENSSL_memcpy(z3, &out[64], 32); +} +#include "./p256_table.h" -static void p256_order_inv0(const EC_GROUP *group, EC_SCALAR *out, - const EC_SCALAR *in) { - // table[i] stores a power of `in` corresponding to the matching enum value. - enum { - // The following indices specify the power in binary. - i_1 = 0, - i_10, - i_11, - i_101, - i_111, - i_1010, - i_1111, - i_10101, - i_101010, - i_101111, - // The following indices specify 2^N-1, or N ones in a row. - i_x6, - i_x8, - i_x16, - i_x32 - }; - BN_ULONG table[15][P256_LIMBS]; +// fiat_p256_select_point_affine selects the `idx-1`th point from a +// precomputation table and copies it to out. If `idx` is zero, the output is +// the point at infinity. +static void fiat_p256_select_point_affine( + const fiat_p256_limb_t idx, size_t size, + const fiat_p256_felem pre_comp[/*size*/][2], fiat_p256_felem out[3]) { + OPENSSL_memset(out, 0, sizeof(fiat_p256_felem) * 3); + for (size_t i = 0; i < size; i++) { + fiat_p256_limb_t mismatch = i ^ (idx - 1); + fiat_p256_cmovznz(out[0], mismatch, pre_comp[i][0], out[0]); + fiat_p256_cmovznz(out[1], mismatch, pre_comp[i][1], out[1]); + } + fiat_p256_cmovznz(out[2], idx, out[2], fiat_p256_one); +} - // https://briansmith.org/ecc-inversion-addition-chains-01#p256_scalar_inversion - // This code uses specialized field arithmetic and saves 12 sqr and 13 mul. - - // Pre-calculate powers. - OPENSSL_memcpy(table[i_1], in->words, P256_LIMBS * sizeof(BN_ULONG)); - - p256_order_sqr(group, table[i_10], table[i_1], 1); - - p256_order_mul(group, table[i_11], table[i_1], table[i_10]); - - p256_order_mul(group, table[i_101], table[i_11], table[i_10]); - - p256_order_mul(group, table[i_111], table[i_101], table[i_10]); - - p256_order_sqr(group, table[i_1010], table[i_101], 1); - - p256_order_mul(group, table[i_1111], table[i_1010], table[i_101]); - - p256_order_sqr(group, table[i_10101], table[i_1010], 1); - p256_order_mul(group, table[i_10101], table[i_10101], table[i_1]); - - p256_order_sqr(group, table[i_101010], table[i_10101], 1); - - p256_order_mul(group, table[i_101111], table[i_101010], table[i_101]); - - p256_order_mul(group, table[i_x6], table[i_101010], table[i_10101]); - - p256_order_sqr(group, table[i_x8], table[i_x6], 2); - p256_order_mul(group, table[i_x8], table[i_x8], table[i_11]); - - p256_order_sqr(group, table[i_x16], table[i_x8], 8); - p256_order_mul(group, table[i_x16], table[i_x16], table[i_x8]); - - p256_order_sqr(group, table[i_x32], table[i_x16], 16); - p256_order_mul(group, table[i_x32], table[i_x32], table[i_x16]); - - // Compute `in` raised to the order-2. - p256_order_sqr(group, out->words, table[i_x32], 64); - p256_order_mul(group, out->words, out->words, table[i_x32]); - static const struct { - uint8_t p, i; - } kChain[27] = {{32, i_x32}, {6, i_101111}, {5, i_111}, {4, i_11}, - {5, i_1111}, {5, i_10101}, {4, i_101}, {3, i_101}, - {3, i_101}, {5, i_111}, {9, i_101111}, {6, i_1111}, - {2, i_1}, {5, i_1}, {6, i_1111}, {5, i_111}, - {4, i_111}, {5, i_111}, {5, i_101}, {3, i_11}, - {10, i_101111}, {2, i_11}, {5, i_11}, {5, i_11}, - {3, i_1}, {7, i_10101}, {6, i_1111}}; - for (const auto &step : kChain) { - p256_order_sqr(group, out->words, out->words, step.p); - p256_order_mul(group, out->words, out->words, table[step.i]); +// fiat_p256_select_point selects the `idx`th point from a precomputation table +// and copies it to out. +static void fiat_p256_select_point(const fiat_p256_limb_t idx, size_t size, + const fiat_p256_felem pre_comp[/*size*/][3], + fiat_p256_felem out[3]) { + OPENSSL_memset(out, 0, sizeof(fiat_p256_felem) * 3); + for (size_t i = 0; i < size; i++) { + fiat_p256_limb_t mismatch = i ^ idx; + fiat_p256_cmovznz(out[0], mismatch, pre_comp[i][0], out[0]); + fiat_p256_cmovznz(out[1], mismatch, pre_comp[i][1], out[1]); + fiat_p256_cmovznz(out[2], mismatch, pre_comp[i][2], out[2]); } } -static int p256_order_mont_inv_vartime(const EC_GROUP *group, EC_SCALAR *out, - const EC_SCALAR *in) { -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) -#if defined(OPENSSL_X86_64) - if (!CRYPTO_is_AVX_capable()) { - // No AVX support; fallback to generic code. - return ec_simple_scalar_to_montgomery_inv_vartime(group, out, in); - } -#endif - - assert(group->order.N.width == P256_LIMBS); - if (!beeu_mod_inverse_vartime(out->words, in->words, group->order.N.d)) { +// fiat_p256_get_bit returns the `i`th bit in `in`. +static crypto_word_t fiat_p256_get_bit(const EC_SCALAR *in, int i) { + if (i < 0 || i >= 256) { return 0; } - - // The result should be returned in the Montgomery domain. - ec_scalar_to_montgomery(group, out, out); - return 1; +#if defined(OPENSSL_64_BIT) + static_assert(sizeof(BN_ULONG) == 8, "BN_ULONG was not 64-bit"); + return (in->words[i >> 6] >> (i & 63)) & 1; #else - return ec_simple_scalar_to_montgomery_inv_vartime(group, out, in); + static_assert(sizeof(BN_ULONG) == 4, "BN_ULONG was not 32-bit"); + return (in->words[i >> 5] >> (i & 31)) & 1; #endif } - -// OPENSSL EC_METHOD WRAPPERS - - -static void fiat_p256_from_generic(fiat_p256_felem out, const EC_FELEM *in) { - OPENSSL_memcpy(out, in->words, 32); -} - -static void fiat_p256_to_generic(EC_FELEM *out, const fiat_p256_felem in) { - OPENSSL_memcpy(out->words, in, 32); -} +// OPENSSL EC_METHOD FUNCTIONS // Takes the Jacobian coordinates (X, Y, Z) of a point and returns (X', Y') = // (X/Z^2, Y/Z^3). @@ -560,22 +267,252 @@ if (x_out != nullptr) { fiat_p256_felem x; fiat_p256_from_generic(x, &point->X); - p256_coord_mul(x, x, z2); + fiat_p256_mul(x, x, z2); fiat_p256_to_generic(x_out, x); } if (y_out != nullptr) { fiat_p256_felem y; fiat_p256_from_generic(y, &point->Y); - p256_coord_sqr(z2, z2); // z^-4 - p256_coord_mul(y, y, z1); // y * z - p256_coord_mul(y, y, z2); // y * z^-3 + fiat_p256_square(z2, z2); // z^-4 + fiat_p256_mul(y, y, z1); // y * z + fiat_p256_mul(y, y, z2); // y * z^-3 fiat_p256_to_generic(y_out, y); } return 1; } +static void ec_GFp_nistp256_add(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *a, const EC_JACOBIAN *b) { + fiat_p256_felem x1, y1, z1, x2, y2, z2; + fiat_p256_from_generic(x1, &a->X); + fiat_p256_from_generic(y1, &a->Y); + fiat_p256_from_generic(z1, &a->Z); + fiat_p256_from_generic(x2, &b->X); + fiat_p256_from_generic(y2, &b->Y); + fiat_p256_from_generic(z2, &b->Z); + fiat_p256_point_add(x1, y1, z1, x1, y1, z1, x2, y2, z2); + fiat_p256_to_generic(&r->X, x1); + fiat_p256_to_generic(&r->Y, y1); + fiat_p256_to_generic(&r->Z, z1); +} + +static void ec_GFp_nistp256_dbl(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *a) { + fiat_p256_felem x, y, z; + fiat_p256_from_generic(x, &a->X); + fiat_p256_from_generic(y, &a->Y); + fiat_p256_from_generic(z, &a->Z); + fiat_p256_point_double(x, y, z, x, y, z); + fiat_p256_to_generic(&r->X, x); + fiat_p256_to_generic(&r->Y, y); + fiat_p256_to_generic(&r->Z, z); +} + +static void ec_GFp_nistp256_point_mul(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *p, + const EC_SCALAR *scalar) { + fiat_p256_felem p_pre_comp[17][3]; + OPENSSL_memset(&p_pre_comp, 0, sizeof(p_pre_comp)); + // Precompute multiples. + fiat_p256_from_generic(p_pre_comp[1][0], &p->X); + fiat_p256_from_generic(p_pre_comp[1][1], &p->Y); + fiat_p256_from_generic(p_pre_comp[1][2], &p->Z); + for (size_t j = 2; j <= 16; ++j) { + if (j & 1) { + fiat_p256_point_add(p_pre_comp[j][0], p_pre_comp[j][1], p_pre_comp[j][2], + p_pre_comp[1][0], p_pre_comp[1][1], p_pre_comp[1][2], + p_pre_comp[j - 1][0], p_pre_comp[j - 1][1], + p_pre_comp[j - 1][2]); + } else { + fiat_p256_point_double(p_pre_comp[j][0], p_pre_comp[j][1], + p_pre_comp[j][2], p_pre_comp[j / 2][0], + p_pre_comp[j / 2][1], p_pre_comp[j / 2][2]); + } + } + + // Set nq to the point at infinity. + fiat_p256_felem nq[3] = {{0}, {0}, {0}}, ftmp, tmp[3]; + + // Loop over `scalar` msb-to-lsb, incorporating `p_pre_comp` every 5th round. + int skip = 1; // Save two point operations in the first round. + for (size_t i = 255; i < 256; i--) { + // double + if (!skip) { + fiat_p256_point_double(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2]); + } + + // do other additions every 5 doublings + if (i % 5 == 0) { + crypto_word_t bits = fiat_p256_get_bit(scalar, i + 4) << 5; + bits |= fiat_p256_get_bit(scalar, i + 3) << 4; + bits |= fiat_p256_get_bit(scalar, i + 2) << 3; + bits |= fiat_p256_get_bit(scalar, i + 1) << 2; + bits |= fiat_p256_get_bit(scalar, i) << 1; + bits |= fiat_p256_get_bit(scalar, i - 1); + crypto_word_t sign, digit; + ec_GFp_nistp_recode_scalar_bits(&sign, &digit, bits); + + // select the point to add or subtract, in constant time. + fiat_p256_select_point((fiat_p256_limb_t)digit, 17, + (const fiat_p256_felem(*)[3])p_pre_comp, tmp); + fiat_p256_opp(ftmp, tmp[1]); // (X, -Y, Z) is the negative point. + fiat_p256_cmovznz(tmp[1], (fiat_p256_limb_t)sign, tmp[1], ftmp); + + if (!skip) { + fiat_p256_point_add(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2], tmp[0], + tmp[1], tmp[2]); + } else { + fiat_p256_copy(nq[0], tmp[0]); + fiat_p256_copy(nq[1], tmp[1]); + fiat_p256_copy(nq[2], tmp[2]); + skip = 0; + } + } + } + + fiat_p256_to_generic(&r->X, nq[0]); + fiat_p256_to_generic(&r->Y, nq[1]); + fiat_p256_to_generic(&r->Z, nq[2]); +} + +static void ec_GFp_nistp256_point_mul_base(const EC_GROUP *group, + EC_JACOBIAN *r, + const EC_SCALAR *scalar) { + // Set nq to the point at infinity. + fiat_p256_felem nq[3] = {{0}, {0}, {0}}, tmp[3]; + + int skip = 1; // Save two point operations in the first round. + for (size_t i = 31; i < 32; i--) { + if (!skip) { + fiat_p256_point_double(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2]); + } + + // First, look 32 bits upwards. + crypto_word_t bits = fiat_p256_get_bit(scalar, i + 224) << 3; + bits |= fiat_p256_get_bit(scalar, i + 160) << 2; + bits |= fiat_p256_get_bit(scalar, i + 96) << 1; + bits |= fiat_p256_get_bit(scalar, i + 32); + // Select the point to add, in constant time. + fiat_p256_select_point_affine((fiat_p256_limb_t)bits, 15, + fiat_p256_g_pre_comp[1], tmp); + + if (!skip) { + fiat_p256_point_add(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2], tmp[0], + tmp[1], tmp[2]); + } else { + fiat_p256_copy(nq[0], tmp[0]); + fiat_p256_copy(nq[1], tmp[1]); + fiat_p256_copy(nq[2], tmp[2]); + skip = 0; + } + + // Second, look at the current position. + bits = fiat_p256_get_bit(scalar, i + 192) << 3; + bits |= fiat_p256_get_bit(scalar, i + 128) << 2; + bits |= fiat_p256_get_bit(scalar, i + 64) << 1; + bits |= fiat_p256_get_bit(scalar, i); + // Select the point to add, in constant time. + fiat_p256_select_point_affine((fiat_p256_limb_t)bits, 15, + fiat_p256_g_pre_comp[0], tmp); + fiat_p256_point_add(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2], tmp[0], + tmp[1], tmp[2]); + } + + fiat_p256_to_generic(&r->X, nq[0]); + fiat_p256_to_generic(&r->Y, nq[1]); + fiat_p256_to_generic(&r->Z, nq[2]); +} + +static void ec_GFp_nistp256_point_mul_public(const EC_GROUP *group, + EC_JACOBIAN *r, + const EC_SCALAR *g_scalar, + const EC_JACOBIAN *p, + const EC_SCALAR *p_scalar) { +#define P256_WSIZE_PUBLIC 4 + // Precompute multiples of `p`. p_pre_comp[i] is (2*i+1) * `p`. + fiat_p256_felem p_pre_comp[1 << (P256_WSIZE_PUBLIC - 1)][3]; + fiat_p256_from_generic(p_pre_comp[0][0], &p->X); + fiat_p256_from_generic(p_pre_comp[0][1], &p->Y); + fiat_p256_from_generic(p_pre_comp[0][2], &p->Z); + fiat_p256_felem p2[3]; + fiat_p256_point_double(p2[0], p2[1], p2[2], p_pre_comp[0][0], + p_pre_comp[0][1], p_pre_comp[0][2]); + for (size_t i = 1; i < std::size(p_pre_comp); i++) { + fiat_p256_point_add(p_pre_comp[i][0], p_pre_comp[i][1], p_pre_comp[i][2], + p_pre_comp[i - 1][0], p_pre_comp[i - 1][1], + p_pre_comp[i - 1][2], p2[0], p2[1], p2[2]); + } + + // Set up the coefficients for `p_scalar`. + int8_t p_wNAF[257]; + ec_compute_wNAF(group, p_wNAF, p_scalar, 256, P256_WSIZE_PUBLIC); + + // Set `ret` to the point at infinity. + int skip = 1; // Save some point operations. + fiat_p256_felem ret[3] = {{0}, {0}, {0}}; + for (int i = 256; i >= 0; i--) { + if (!skip) { + fiat_p256_point_double(ret[0], ret[1], ret[2], ret[0], ret[1], ret[2]); + } + + // For the `g_scalar`, we use the precomputed table without the + // constant-time lookup. + if (i <= 31) { + // First, look 32 bits upwards. + crypto_word_t bits = fiat_p256_get_bit(g_scalar, i + 224) << 3; + bits |= fiat_p256_get_bit(g_scalar, i + 160) << 2; + bits |= fiat_p256_get_bit(g_scalar, i + 96) << 1; + bits |= fiat_p256_get_bit(g_scalar, i + 32); + if (bits != 0) { + size_t index = (size_t)(bits - 1); + fiat_p256_point_add(ret[0], ret[1], ret[2], ret[0], ret[1], ret[2], + fiat_p256_g_pre_comp[1][index][0], + fiat_p256_g_pre_comp[1][index][1], fiat_p256_one); + skip = 0; + } + + // Second, look at the current position. + bits = fiat_p256_get_bit(g_scalar, i + 192) << 3; + bits |= fiat_p256_get_bit(g_scalar, i + 128) << 2; + bits |= fiat_p256_get_bit(g_scalar, i + 64) << 1; + bits |= fiat_p256_get_bit(g_scalar, i); + if (bits != 0) { + size_t index = (size_t)(bits - 1); + fiat_p256_point_add(ret[0], ret[1], ret[2], ret[0], ret[1], ret[2], + fiat_p256_g_pre_comp[0][index][0], + fiat_p256_g_pre_comp[0][index][1], fiat_p256_one); + skip = 0; + } + } + + int digit = p_wNAF[i]; + if (digit != 0) { + assert(digit & 1); + size_t idx = (size_t)(digit < 0 ? (-digit) >> 1 : digit >> 1); + fiat_p256_felem *y = &p_pre_comp[idx][1], tmp; + if (digit < 0) { + fiat_p256_opp(tmp, p_pre_comp[idx][1]); + y = &tmp; + } + if (!skip) { + fiat_p256_point_add(ret[0], ret[1], ret[2], ret[0], ret[1], ret[2], + p_pre_comp[idx][0], *y, p_pre_comp[idx][2]); + } else { + fiat_p256_copy(ret[0], p_pre_comp[idx][0]); + fiat_p256_copy(ret[1], *y); + fiat_p256_copy(ret[2], p_pre_comp[idx][2]); + skip = 0; + } + } + } + + fiat_p256_to_generic(&r->X, ret[0]); + fiat_p256_to_generic(&r->Y, ret[1]); + fiat_p256_to_generic(&r->Z, ret[2]); +} + static int ec_GFp_nistp256_cmp_x_coordinate(const EC_GROUP *group, const EC_JACOBIAN *p, const EC_SCALAR *r) { @@ -588,11 +525,11 @@ // not. fiat_p256_felem Z2_mont; fiat_p256_from_generic(Z2_mont, &p->Z); - p256_coord_mul(Z2_mont, Z2_mont, Z2_mont); + fiat_p256_mul(Z2_mont, Z2_mont, Z2_mont); fiat_p256_felem r_Z2; - OPENSSL_memcpy(r_Z2, r->words, 32); // r < order < p, so this is valid. - p256_coord_mul(r_Z2, r_Z2, Z2_mont); + fiat_p256_from_words(r_Z2, r->words); // r < order < p, so this is valid. + fiat_p256_mul(r_Z2, r_Z2, Z2_mont); fiat_p256_felem X; fiat_p256_from_generic(X, &p->X); @@ -613,7 +550,7 @@ if (carry == 0 && bn_less_than_words(tmp.words, group->field.N.d, group->field.N.width)) { fiat_p256_from_generic(r_Z2, &tmp); - p256_coord_mul(r_Z2, r_Z2, Z2_mont); + fiat_p256_mul(r_Z2, r_Z2, Z2_mont); if (OPENSSL_memcmp(&r_Z2, &X, sizeof(r_Z2)) == 0) { return 1; } @@ -622,56 +559,6 @@ return 0; } -static void ec_GFp_nistp256_point_mul(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *p, - const EC_SCALAR *scalar) { - fiat_p256_felem t[3]; - fiat_p256_from_generic(t[0], &p->X); - fiat_p256_from_generic(t[1], &p->Y); - fiat_p256_from_generic(t[2], &p->Z); - p256_point_mul(t, t, (uint8_t *)scalar->words); - - fiat_p256_to_generic(&r->X, t[0]); - fiat_p256_to_generic(&r->Y, t[1]); - fiat_p256_to_generic(&r->Z, t[2]); -} - -static void ec_GFp_nistp256_point_mul_base(const EC_GROUP *group, - EC_JACOBIAN *r, const EC_SCALAR *s) { - alignas(32) fiat_p256_felem ret[3]; - p256_point_mul_base(ret, (uint8_t *)s->words); - fiat_p256_to_generic(&r->X, ret[0]); - fiat_p256_to_generic(&r->Y, ret[1]); - fiat_p256_to_generic(&r->Z, ret[2]); -} - -static void ec_GFp_nistp256_add(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *a, const EC_JACOBIAN *b) { - fiat_p256_felem p[3], q[3]; - fiat_p256_from_generic(p[0], &a->X); - fiat_p256_from_generic(p[1], &a->Y); - fiat_p256_from_generic(p[2], &a->Z); - fiat_p256_from_generic(q[0], &b->X); - fiat_p256_from_generic(q[1], &b->Y); - fiat_p256_from_generic(q[2], &b->Z); - p256_point_add_vartime_if_doubling((uintptr_t)p, (uintptr_t)p, (uintptr_t)q); - fiat_p256_to_generic(&r->X, p[0]); - fiat_p256_to_generic(&r->Y, p[1]); - fiat_p256_to_generic(&r->Z, p[2]); -} - -static void ec_GFp_nistp256_dbl(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *a) { - fiat_p256_felem p[3]; - fiat_p256_from_generic(p[0], &a->X); - fiat_p256_from_generic(p[1], &a->Y); - fiat_p256_from_generic(p[2], &a->Z); - p256_point_double((uintptr_t)p, (uintptr_t)p); - fiat_p256_to_generic(&r->X, p[0]); - fiat_p256_to_generic(&r->Y, p[1]); - fiat_p256_to_generic(&r->Z, p[2]); -} - BSSL_NAMESPACE_BEGIN DEFINE_METHOD_FUNCTION(EC_METHOD, EC_GFp_nistp256_method) { @@ -682,8 +569,9 @@ out->mul = ec_GFp_nistp256_point_mul; out->mul_base = ec_GFp_nistp256_point_mul_base; out->mul_public = ec_GFp_nistp256_point_mul_public; - out->scalar_inv0_montgomery = p256_order_inv0; - out->scalar_to_montgomery_inv_vartime = p256_order_mont_inv_vartime; + out->scalar_inv0_montgomery = ec_simple_scalar_inv0_montgomery; + out->scalar_to_montgomery_inv_vartime = + ec_simple_scalar_to_montgomery_inv_vartime; out->cmp_x_coordinate = ec_GFp_nistp256_cmp_x_coordinate; }
diff --git a/crypto/fipsmodule/ec/p256_internal.h b/crypto/fipsmodule/ec/p256_internal.h deleted file mode 100644 index 20b0ce7..0000000 --- a/crypto/fipsmodule/ec/p256_internal.h +++ /dev/null
@@ -1,193 +0,0 @@ -// Copyright 2026 The BoringSSL Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef OPENSSL_HEADER_EC_P256_INTERNAL_H -#define OPENSSL_HEADER_EC_P256_INTERNAL_H - -#include <openssl/base.h> -#include <openssl/bn.h> - -#include "../../internal.h" -#include "../../../third_party/fiat/bedrock_unverified_platform.c.inc" - -#define P256_LIMBS (32 / sizeof(bssl::crypto_word_t)) -typedef bssl::crypto_word_t fiat_p256_felem[P256_LIMBS]; - - -// Choose implementation of arithmetic in the coordinate field. - -#if defined(BORINGSSL_HAS_UINT128) -#include "../../../third_party/fiat/p256_field_64.br.c.inc" - -#include "../../../third_party/fiat/p256_64.h" -#elif defined(OPENSSL_64_BIT) -#include "../../../third_party/fiat/p256_field_64.br.c.inc" - -#include "../../../third_party/fiat/p256_64_msvc.h" -#else -#include "../../../third_party/fiat/p256_field_32.br.c.inc" - -#include "../../../third_party/fiat/p256_32.h" -// Add Bedrock versions of p256_32.h functions for p256_point.br.c.inc to call. -static inline void p256_coord_add(br_word_t out, br_word_t x, br_word_t y) { - fiat_p256_add((uint32_t *)out, (const uint32_t *)x, (const uint32_t *)y); -} -static inline void p256_coord_sub(br_word_t out, br_word_t x, br_word_t y) { - fiat_p256_sub((uint32_t *)out, (const uint32_t *)x, (const uint32_t *)y); -} -#endif - -extern "C" { -#if !defined(OPENSSL_NO_ASM) && (defined(__ELF__) || defined(__APPLE__)) && \ - defined(OPENSSL_X86_64) && !defined(OPENSSL_NANOLIBC) -// These functions are only available with gas and SysV ABI, used by Apple and -// ELF-based platforms. Unlike most of our SysV assembly, they currently rely on -// the SysV redzone. This trips one target which looks like it targets SysV but -// has no redzone. This happens to define `OPENSSL_NANOLIBC`, so gate on that. -// -// TODO(crbug.com/522255483): Come up with a clearer story for the redzone -// situation. -#define FIAT_P256_ADX_ASM -void fiat_p256_adx_mul(uint64_t x0[4], const uint64_t x1[4], - const uint64_t x2[4]); -void fiat_p256_adx_sqr(uint64_t x0[4], const uint64_t x1[4]); -#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) -void ecp_nistz256_mul_mont(uint64_t pr[4], const uint64_t py[4], uint64_t y0, - uint64_t x0, uint64_t x1, uint64_t x2, uint64_t x3); -void ecp_nistz256_sqr_mont(uint64_t pr[4], const uint64_t py[4], uint64_t y0, - uint64_t x0, uint64_t x1, uint64_t x2, uint64_t x3); -#endif -} - -static inline void p256_coord_mul(fiat_p256_felem out, const fiat_p256_felem x, - const fiat_p256_felem y) { -#if defined(FIAT_P256_ADX_ASM) - if (bssl::CRYPTO_is_BMI1_capable() && bssl::CRYPTO_is_BMI2_capable() && - bssl::CRYPTO_is_ADX_capable()) { - return fiat_p256_adx_mul(out, x, y); - } - fiat_p256_mul(out, x, y); -#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) - ecp_nistz256_mul_mont(out, y, y[0], x[0], x[1], x[2], x[3]); -#else - fiat_p256_mul(out, x, y); -#endif -} - -static inline void p256_coord_sqr(fiat_p256_felem out, - const fiat_p256_felem x) { -#if defined(FIAT_P256_ADX_ASM) - if (bssl::CRYPTO_is_BMI1_capable() && bssl::CRYPTO_is_BMI2_capable() && - bssl::CRYPTO_is_ADX_capable()) { - return fiat_p256_adx_sqr(out, x); - } - fiat_p256_square(out, x); -#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) - ecp_nistz256_sqr_mont(out, x, x[0], x[0], x[1], x[2], x[3]); -#else - fiat_p256_square(out, x); -#endif -} - -// Add Bedrock versions of these functions for p256_point.br.c.inc to call. -static inline void p256_coord_mul(br_word_t out, br_word_t x, br_word_t y) { - p256_coord_mul((br_word_t *)out, (const br_word_t *)x, (const br_word_t *)y); -} - -static inline void p256_coord_sqr(br_word_t out, br_word_t x) { - p256_coord_sqr((br_word_t *)out, (const br_word_t *)x); -} - - -// Choose implementation of arithmetic in the field modulo curve order. - -extern "C" { - -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) -// beeu_mod_inverse_vartime sets out = a^-1 mod p using a Euclidean algorithm. -// Assumption: 0 < a < p < 2^(256) and p is odd. -int beeu_mod_inverse_vartime(BN_ULONG out[4], const BN_ULONG a[4], - const BN_ULONG p[4]); -#endif - -// P-256 scalar operations. -// -// The following functions compute modulo N, where N is the order of P-256. They -// take fully-reduced inputs and give fully-reduced outputs. - -#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) -// ecp_nistz256_ord_mul_mont sets `res` to `a` * `b` where inputs and outputs -// are in Montgomery form. That is, `res` is `a` * `b` * 2^-256 mod N. -void ecp_nistz256_ord_mul_mont(BN_ULONG res[4], const BN_ULONG a[4], - const BN_ULONG b[4]); - -// ecp_nistz256_ord_sqr_mont sets `res` to `a`^(2*`rep`) where inputs and -// outputs are in Montgomery form. That is, `res` is -// (`a` * 2^-256)^(2*`rep`) * 2^256 mod N. -void ecp_nistz256_ord_sqr_mont(BN_ULONG res[4], const BN_ULONG a[4], - BN_ULONG rep); - -#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) -void ecp_nistz256_ord_mul_mont_nohw(BN_ULONG res[4], const BN_ULONG a[4], - const BN_ULONG b[4]); -void ecp_nistz256_ord_mul_mont_adx(BN_ULONG res[4], const BN_ULONG a[4], - const BN_ULONG b[4]); - -void ecp_nistz256_ord_sqr_mont_nohw(BN_ULONG res[4], const BN_ULONG a[4], - BN_ULONG rep); -void ecp_nistz256_ord_sqr_mont_adx(BN_ULONG res[4], const BN_ULONG a[4], - BN_ULONG rep); -#endif - -} // extern C - -static inline void p256_order_mul(const EC_GROUP *group, - BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]) { -#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) - return ecp_nistz256_ord_mul_mont(res, a, b); -#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) - if (bssl::CRYPTO_is_BMI2_capable() && bssl::CRYPTO_is_ADX_capable()) { - return ecp_nistz256_ord_mul_mont_adx(res, a, b); - } - return ecp_nistz256_ord_mul_mont_nohw(res, a, b); -#else - const BIGNUM *order = &group->order.N; - bssl::bn_mod_mul_montgomery_small(res, a, b, order->width, &group->order); -#endif -} - -static inline void p256_order_sqr(const EC_GROUP *group, - BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], BN_ULONG rep) { -#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) - return ecp_nistz256_ord_sqr_mont(res, a, rep); -#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) - if (bssl::CRYPTO_is_BMI2_capable() && bssl::CRYPTO_is_ADX_capable()) { - return ecp_nistz256_ord_sqr_mont_adx(res, a, rep); - } - return ecp_nistz256_ord_sqr_mont_nohw(res, a, rep); -#else - bssl::OPENSSL_memmove(res, a, sizeof(BN_ULONG) * P256_LIMBS); - for (BN_ULONG i = 0; i < rep; i++) { - p256_order_mul(group, res, res, res); - } -#endif -} - -#include "../../../third_party/fiat/p256_point.br.c.inc" - -#endif // OPENSSL_HEADER_EC_P256_INTERNAL_H
diff --git a/crypto/fipsmodule/ec/p256_test.cc b/crypto/fipsmodule/ec/p256_test.cc index 8a686b1..a8039c3 100644 --- a/crypto/fipsmodule/ec/p256_test.cc +++ b/crypto/fipsmodule/ec/p256_test.cc
@@ -12,26 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -#if !defined(BORINGSSL_SHARED_LIBRARY) // These tests use p256_internal.h - -#include <openssl/base.h> -#include <openssl/bn.h> -#include <openssl/ec.h> -#include <openssl/mem.h> -#include <openssl/nid.h> - +#include <gtest/gtest.h> #include "../../internal.h" #include "../../test/abi_test.h" -#include "../../test/file_test.h" -#include "internal.h" -#include "p256_internal.h" + BSSL_NAMESPACE_BEGIN namespace { -#if defined(FIAT_P256_ADX_ASM) && defined(SUPPORTS_ABI_TEST) +#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(__x86_64__) && \ + defined(SUPPORTS_ABI_TEST) +extern "C" { +#include "../../../third_party/fiat/p256_field.c.inc" +} -TEST(P256Test, CoordMulAdxABI) { +TEST(P256Test, AdxMulABI) { static const uint64_t in1[4] = {0}, in2[4] = {0}; uint64_t out[4]; if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() && @@ -42,7 +37,8 @@ } } -TEST(P256Test, CoordSqrAdxABI) { +#include <assert.h> +TEST(P256Test, AdxSquareABI) { static const uint64_t in[4] = {0}; uint64_t out[4]; if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() && @@ -52,505 +48,7 @@ GTEST_SKIP() << "Can't test ABI of ADX code without ADX"; } } - #endif -#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && \ - defined(SUPPORTS_ABI_TEST) - -TEST(P256Test, CoordMulABI) { - static const uint64_t x[4] = {0}, y[4] = {0}; - uint64_t out[4]; - CHECK_ABI(ecp_nistz256_mul_mont, out, y, y[0], x[0], x[1], x[2], x[3]); -} - -TEST(P256Test, CoordSqrABI) { - static const uint64_t x[4] = {0}; - uint64_t out[4]; - CHECK_ABI(ecp_nistz256_sqr_mont, out, x, x[0], x[0], x[1], x[2], x[3]); -} - -#endif - -#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && \ - defined(SUPPORTS_ABI_TEST) - -TEST(P256Test, OrdMulAdxABI) { - static const uint64_t in1[4] = {0}, in2[4] = {0}; - uint64_t out[4]; - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - CHECK_ABI(ecp_nistz256_ord_mul_mont_adx, out, in1, in2); - } else { - GTEST_SKIP() << "Can't test ABI of ADX code without ADX"; - } -} - -TEST(P256Test, OrdSqrAdxABI) { - static const uint64_t in[4] = {0}; - uint64_t out[4]; - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - CHECK_ABI(ecp_nistz256_ord_sqr_mont_adx, out, in, 1); - } else { - GTEST_SKIP() << "Can't test ABI of ADX code without ADX"; - } -} - -TEST(P256Test, OrdMulNohwABI) { - static const uint64_t in1[4] = {0}, in2[4] = {0}; - uint64_t out[4]; - CHECK_ABI(ecp_nistz256_ord_mul_mont_nohw, out, in1, in2); -} - -TEST(P256Test, OrdSqrNohwABI) { - static const uint64_t in[4] = {0}; - uint64_t out[4]; - CHECK_ABI(ecp_nistz256_ord_sqr_mont_nohw, out, in, 1); -} - -#endif - -typedef fiat_p256_felem P256_POINT_AFFINE[2]; -typedef fiat_p256_felem P256_POINT[3]; - -static inline void point_add(P256_POINT ret, P256_POINT p, P256_POINT q) { - p256_point_add_vartime_if_doubling((uintptr_t)ret, (uintptr_t)p, - (uintptr_t)q); -} - -static inline void point_add_affine(P256_POINT ret, P256_POINT a, - P256_POINT b_affine) { - uintptr_t bnz = 0; - bnz |= b_affine[0][0] | b_affine[0][1] | b_affine[0][2] | b_affine[0][3]; - bnz |= b_affine[1][0] | b_affine[1][1] | b_affine[1][2] | b_affine[1][3]; - p256_point_add_affinenz_conditional_vartime_if_doubling( - (uintptr_t)ret, (uintptr_t)a, (uintptr_t)b_affine, (uintptr_t)bnz); -} - -static inline void point_double(P256_POINT ret, P256_POINT a) { - p256_point_double((uintptr_t)ret, (uintptr_t)a); -} - -static bool GetFieldElement(FileTest *t, BN_ULONG out[P256_LIMBS], - const char *name) { - std::vector<uint8_t> bytes; - if (!t->GetBytes(&bytes, name)) { - return false; - } - - if (bytes.size() != BN_BYTES * P256_LIMBS) { - ADD_FAILURE() << "Invalid length: " << name; - return false; - } - - // `byte` contains bytes in big-endian while `out` should contain `BN_ULONG`s - // in little-endian. - OPENSSL_memset(out, 0, P256_LIMBS * sizeof(BN_ULONG)); - for (size_t i = 0; i < bytes.size(); i++) { - out[P256_LIMBS - 1 - (i / BN_BYTES)] <<= 8; - out[P256_LIMBS - 1 - (i / BN_BYTES)] |= bytes[i]; - } - - return true; -} - -static std::string FieldElementToString(const BN_ULONG a[P256_LIMBS]) { - std::string ret; - for (size_t i = P256_LIMBS - 1; i < P256_LIMBS; i--) { - char buf[2 * BN_BYTES + 1]; - snprintf(buf, sizeof(buf), BN_HEX_FMT2, a[i]); - ret += buf; - } - return ret; -} - -static testing::AssertionResult ExpectFieldElementsEqual( - const char *expected_expr, const char *actual_expr, - const BN_ULONG expected[P256_LIMBS], const BN_ULONG actual[P256_LIMBS]) { - if (OPENSSL_memcmp(expected, actual, sizeof(BN_ULONG) * P256_LIMBS) == 0) { - return testing::AssertionSuccess(); - } - - return testing::AssertionFailure() - << "Expected: " << FieldElementToString(expected) << " (" - << expected_expr << ")\n" - << "Actual: " << FieldElementToString(actual) << " (" << actual_expr - << ")"; -} - -#define EXPECT_FIELD_ELEMENTS_EQUAL(a, b) \ - EXPECT_PRED_FORMAT2(ExpectFieldElementsEqual, a, b) - -// P-256 scalars and field elements are the same size. -#define EXPECT_SCALARS_EQUAL(a, b) EXPECT_FIELD_ELEMENTS_EQUAL(a, b) - -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) -TEST(P256Test, BEEU) { -#if defined(OPENSSL_X86_64) - if (!CRYPTO_is_AVX_capable()) { - // No AVX support; cannot run the BEEU code. - return; - } -#endif - - const EC_GROUP *group = EC_group_p256(); - BN_ULONG order_words[P256_LIMBS]; - ASSERT_TRUE( - bn_copy_words(order_words, P256_LIMBS, EC_GROUP_get0_order(group))); - - BN_ULONG in[P256_LIMBS], out[P256_LIMBS]; - EC_SCALAR in_scalar, out_scalar, result; - OPENSSL_memset(in, 0, sizeof(in)); - - // Trying to find the inverse of zero should fail. - ASSERT_FALSE(beeu_mod_inverse_vartime(out, in, order_words)); - // This is not a constant-time function, so instrument both zero and a few - // inputs below. - ASSERT_FALSE(CHECK_ABI(beeu_mod_inverse_vartime, out, in, order_words)); - - BN_ULONG kOne[P256_LIMBS] = {}; - kOne[0] = 1; - - auto beeu_test = [&] { - EXPECT_TRUE(bn_less_than_words(in, order_words, P256_LIMBS)); - ASSERT_TRUE(beeu_mod_inverse_vartime(out, in, order_words)); - EXPECT_TRUE(bn_less_than_words(out, order_words, P256_LIMBS)); - - // Calculate out*in and confirm that it equals one, modulo the order. We - // only have Montgomery multiplication, but if we convert one input, and not - // the other, this gives the result outside the Montgomery domain. - OPENSSL_memcpy(in_scalar.words, in, sizeof(in)); - OPENSSL_memcpy(out_scalar.words, out, sizeof(out)); - ec_scalar_to_montgomery(group, &in_scalar, &in_scalar); - ec_scalar_mul_montgomery(group, &result, &in_scalar, &out_scalar); - EXPECT_SCALARS_EQUAL(kOne, result.words); - - // Invert the result and expect to get back to the original value. - ASSERT_TRUE(beeu_mod_inverse_vartime(out, out, order_words)); - EXPECT_SCALARS_EQUAL(in, out); - }; - - for (BN_ULONG i = 1; i < 2000; i++) { - SCOPED_TRACE(i); - in[0] = i; - if (i >= 1000) { - in[1] = i << 8; - in[2] = i << 32; - in[3] = i << 48; - } else { - in[1] = in[2] = in[3] = 0; - } - - beeu_test(); - if (i < 5) { - EXPECT_TRUE(CHECK_ABI(beeu_mod_inverse_vartime, out, in, order_words)); - } - } - - for (int i = 0; i < 255; i++) { - SCOPED_TRACE(i); - // Test `in` = 2^i. - OPENSSL_memset(in, 0, sizeof(in)); - in[i / BN_BITS2] = BN_ULONG{1} << (i % BN_BITS2); - beeu_test(); - - // Test `in` = N - 2^i. - bn_sub_words(in, order_words, in, P256_LIMBS); - beeu_test(); - } -} -#endif - -static void TestNegate(FileTest *t) { - fiat_p256_felem a, b; - ASSERT_TRUE(GetFieldElement(t, a, "A")); - ASSERT_TRUE(GetFieldElement(t, b, "B")); - - // Test that -A = B. - BN_ULONG ret[P256_LIMBS]; - fiat_p256_opp(ret, a); - EXPECT_FIELD_ELEMENTS_EQUAL(b, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - fiat_p256_opp(ret, ret /* a */); - EXPECT_FIELD_ELEMENTS_EQUAL(b, ret); - - // Test that -B = A. - fiat_p256_opp(ret, b); - EXPECT_FIELD_ELEMENTS_EQUAL(a, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - fiat_p256_opp(ret, ret /* b */); - EXPECT_FIELD_ELEMENTS_EQUAL(a, ret); -} - -static void TestMulMont(FileTest *t) { - BN_ULONG a[P256_LIMBS], b[P256_LIMBS], result[P256_LIMBS]; - ASSERT_TRUE(GetFieldElement(t, a, "A")); - ASSERT_TRUE(GetFieldElement(t, b, "B")); - ASSERT_TRUE(GetFieldElement(t, result, "Result")); - - BN_ULONG ret[P256_LIMBS]; - p256_coord_mul(ret, a, b); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - p256_coord_mul(ret, b, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - p256_coord_mul(ret, ret /* a */, b); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - p256_coord_mul(ret, b, ret); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - p256_coord_mul(ret, a, ret /* b */); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - p256_coord_mul(ret, ret /* b */, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { - p256_coord_sqr(ret, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - p256_coord_sqr(ret, ret /* a */); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - } -} - -static void TestFromMont(FileTest *t) { - BN_ULONG a[P256_LIMBS], result[P256_LIMBS]; - ASSERT_TRUE(GetFieldElement(t, a, "A")); - ASSERT_TRUE(GetFieldElement(t, result, "Result")); - - BN_ULONG ret[P256_LIMBS]; - fiat_p256_from_montgomery(ret, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - fiat_p256_from_montgomery(ret, ret /* a */); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); -} - -static void TestOrdMulMont(FileTest *t) { - // This test works on scalars rather than field elements, but the - // representation is the same. - BN_ULONG a[P256_LIMBS], b[P256_LIMBS], result[P256_LIMBS]; - ASSERT_TRUE(GetFieldElement(t, a, "A")); - ASSERT_TRUE(GetFieldElement(t, b, "B")); - ASSERT_TRUE(GetFieldElement(t, result, "Result")); - SCOPED_TRACE(FieldElementToString(a)); - SCOPED_TRACE(FieldElementToString(b)); - - const EC_GROUP *group = EC_group_p256(); - BN_ULONG ret[P256_LIMBS]; - p256_order_mul(group, ret, a, b); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - p256_order_mul(group, ret, b, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - p256_order_mul(group, ret, ret /* a */, b); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - p256_order_mul(group, ret, b, ret); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - p256_order_mul(group, ret, a, ret /* b */); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - p256_order_mul(group, ret, ret /* b */, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { - p256_order_sqr(group, ret, a, 1); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - p256_order_sqr(group, ret, ret /* a */, 1); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - } -} - -static bool PointToAffine(P256_POINT_AFFINE out, const P256_POINT in) { - static const uint8_t kP[] = { - 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - }; - - UniquePtr<BIGNUM> x(BN_new()), y(BN_new()), z(BN_new()); - UniquePtr<BIGNUM> p(BN_bin2bn(kP, sizeof(kP), nullptr)); - if (!x || !y || !z || !p || !bn_set_words(x.get(), in[0], P256_LIMBS) || - !bn_set_words(y.get(), in[1], P256_LIMBS) || - !bn_set_words(z.get(), in[2], P256_LIMBS)) { - return false; - } - - // Coordinates must be fully-reduced. - if (BN_cmp(x.get(), p.get()) >= 0 || BN_cmp(y.get(), p.get()) >= 0 || - BN_cmp(z.get(), p.get()) >= 0) { - return false; - } - - if (BN_is_zero(z.get())) { - // The point at infinity is represented as (0, 0). - OPENSSL_memset(out, 0, sizeof(P256_POINT_AFFINE)); - return true; - } - - UniquePtr<BN_CTX> ctx(BN_CTX_new()); - UniquePtr<BN_MONT_CTX> mont(BN_MONT_CTX_new_for_modulus(p.get(), ctx.get())); - if (!ctx || !mont || - // Invert Z. - !BN_from_montgomery(z.get(), z.get(), mont.get(), ctx.get()) || - !BN_mod_inverse(z.get(), z.get(), p.get(), ctx.get()) || - !BN_to_montgomery(z.get(), z.get(), mont.get(), ctx.get()) || - // Convert (X, Y, Z) to (X/Z^2, Y/Z^3). - !BN_mod_mul_montgomery(x.get(), x.get(), z.get(), mont.get(), - ctx.get()) || - !BN_mod_mul_montgomery(x.get(), x.get(), z.get(), mont.get(), - ctx.get()) || - !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), - ctx.get()) || - !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), - ctx.get()) || - !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), - ctx.get()) || - !bn_copy_words(out[0], P256_LIMBS, x.get()) || - !bn_copy_words(out[1], P256_LIMBS, y.get())) { - return false; - } - return true; -} - -static testing::AssertionResult ExpectPointsEqual( - const char *expected_expr, const char *actual_expr, - const P256_POINT_AFFINE expected, const P256_POINT actual) { - // There are multiple representations of the same `P256_POINT`, so convert to - // `P256_POINT_AFFINE` and compare. - P256_POINT_AFFINE affine; - if (!PointToAffine(affine, actual)) { - return testing::AssertionFailure() - << "Could not convert " << actual_expr << " to affine: (" - << FieldElementToString(actual[0]) << ", " - << FieldElementToString(actual[1]) << ", " - << FieldElementToString(actual[2]) << ")"; - } - - if (OPENSSL_memcmp(expected, affine, sizeof(P256_POINT_AFFINE)) != 0) { - return testing::AssertionFailure() - << "Expected: (" << FieldElementToString(expected[0]) << ", " - << FieldElementToString(expected[1]) << ") (" << expected_expr - << "; affine)\n" - << "Actual: (" << FieldElementToString(affine[0]) << ", " - << FieldElementToString(affine[1]) << ") (affine(" << actual_expr - << "))\n" - << "ActualXYZ:(" << FieldElementToString(actual[0]) << ", " - << FieldElementToString(actual[1]) << ", " - << FieldElementToString(actual[2]) << ") (" << actual_expr << ")"; - } - - return testing::AssertionSuccess(); -} - -#define EXPECT_POINTS_EQUAL(a, b) EXPECT_PRED_FORMAT2(ExpectPointsEqual, a, b) - -static void TestPointAdd(FileTest *t) { - P256_POINT a, b; - P256_POINT_AFFINE result; - ASSERT_TRUE(GetFieldElement(t, a[0], "A.X")); - ASSERT_TRUE(GetFieldElement(t, a[1], "A.Y")); - ASSERT_TRUE(GetFieldElement(t, a[2], "A.Z")); - ASSERT_TRUE(GetFieldElement(t, b[0], "B.X")); - ASSERT_TRUE(GetFieldElement(t, b[1], "B.Y")); - ASSERT_TRUE(GetFieldElement(t, b[2], "B.Z")); - ASSERT_TRUE(GetFieldElement(t, result[0], "Result.X")); - ASSERT_TRUE(GetFieldElement(t, result[1], "Result.Y")); - - P256_POINT ret; - point_add(ret, a, b); - EXPECT_POINTS_EQUAL(result, ret); - - point_add(ret, b, a); - EXPECT_POINTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - point_add(ret, ret /* a */, b); - EXPECT_POINTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - point_add(ret, b, ret /* a */); - EXPECT_POINTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - point_add(ret, a, ret /* b */); - EXPECT_POINTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - point_add(ret, ret /* b */, a); - EXPECT_POINTS_EQUAL(result, ret); - - P256_POINT a_affine, b_affine; - ASSERT_TRUE(PointToAffine(a_affine, a)); - ASSERT_TRUE(PointToAffine(b_affine, b)); - static const uint64_t p256_fe_one[4] = {0x1, 0xffffffff00000000, - 0xffffffffffffffff, 0xfffffffe}; - OPENSSL_memcpy(a_affine[2], p256_fe_one, sizeof(a_affine[2])); - OPENSSL_memcpy(b_affine[2], p256_fe_one, sizeof(b_affine[2])); - - point_add_affine(ret, a, b_affine); - EXPECT_POINTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - point_add_affine(ret, ret /* a */, b_affine); - EXPECT_POINTS_EQUAL(result, ret); - - point_add_affine(ret, b, a_affine); - EXPECT_POINTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - point_add_affine(ret, ret /* b */, a_affine); - EXPECT_POINTS_EQUAL(result, ret); - - if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { - point_double(ret, a); - EXPECT_POINTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - point_double(ret, ret /* a */); - EXPECT_POINTS_EQUAL(result, ret); - } -} - -TEST(P256Test, TestVectors) { - return FileTestGTest("crypto/fipsmodule/ec/p256-tests.txt", [](FileTest *t) { - if (t->GetParameter() == "Negate") { - TestNegate(t); - } else if (t->GetParameter() == "MulMont") { - TestMulMont(t); - } else if (t->GetParameter() == "FromMont") { - TestFromMont(t); - } else if (t->GetParameter() == "PointAdd") { - TestPointAdd(t); - } else if (t->GetParameter() == "OrdMulMont") { - TestOrdMulMont(t); - } else { - FAIL() << "Unknown test type:" << t->GetParameter(); - } - }); -} - } // namespace BSSL_NAMESPACE_END - -#endif // !defined(BORINGSSL_SHARED_LIBRARY)
diff --git a/gen/bcm/p256-armv8-asm-apple.S b/gen/bcm/p256-armv8-asm-apple.S index 60cba77..5c206c4 100644 --- a/gen/bcm/p256-armv8-asm-apple.S +++ b/gen/bcm/p256-armv8-asm-apple.S
@@ -4,6 +4,27 @@ #include <openssl/asm_base.h> #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__APPLE__) +.section __TEXT,__const +.align 5 +p256_constants: +Lpoly: +.quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 +LRR: // 2^512 mod P precomputed for NIST P256 polynomial +.quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd +Lone_mont: +.quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe +Lone: +.quad 1,0,0,0 +Lord: +.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 +LordK: +.quad 0xccd1c8aaee00bc4f +.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 +.align 2 +.text + +// void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4], +// const BN_ULONG x2[4]); .globl _ecp_nistz256_mul_mont .private_extern _ecp_nistz256_mul_mont @@ -14,135 +35,15 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - mul x13,x3,x2 // a[0]*b[0] - umulh x8,x3,x2 + ldr x3,[x2] // bp[0] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] - mul x14,x4,x2 // a[1]*b[0] - umulh x9,x4,x2 - - mul x15,x5,x2 // a[2]*b[0] - umulh x10,x5,x2 - - mul x16,x6,x2 // a[3]*b[0] - umulh x11,x6,x2 - ldr x2,[x1,#8] // b[1] - - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adc x17,xzr,x11 - mov x19,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - ldr x2,[x1,#8*(1+1)] // b[1+1] - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - ldr x2,[x1,#8*(2+1)] // b[2+1] - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - mov w7, #-1 // poly1 = 0x00000000ffffffff - // last reduction - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - adcs x16,x17,x11 - adc x17,x19,xzr - - neg x12,x7 // poly3 = 0xffffffff00000001 - adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus - sbcs x9,x14,x7 - sbcs x10,x15,xzr - sbcs x11,x16,x12 - sbcs xzr,x17,xzr // did it borrow? - - csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus - csel x14,x14,x9,lo - csel x15,x15,x10,lo - stp x13,x14,[x0] - csel x16,x16,x11,lo - stp x15,x16,[x0,#16] + bl __ecp_nistz256_mul_mont ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 @@ -150,6 +51,7 @@ ret +// void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); .globl _ecp_nistz256_sqr_mont .private_extern _ecp_nistz256_sqr_mont @@ -161,6 +63,312 @@ add x29,sp,#0 stp x19,x20,[sp,#16] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sqr_mont + + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl _ecp_nistz256_div_by_2 +.private_extern _ecp_nistz256_div_by_2 + +.align 4 +.alt_entry _ecp_nistz256_div_by_2 +_ecp_nistz256_div_by_2: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_div_by_2 + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl _ecp_nistz256_mul_by_2 +.private_extern _ecp_nistz256_mul_by_2 + +.align 4 +.alt_entry _ecp_nistz256_mul_by_2 +_ecp_nistz256_mul_by_2: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + + bl __ecp_nistz256_add_to // ret = a+a // 2*a + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl _ecp_nistz256_mul_by_3 +.private_extern _ecp_nistz256_mul_by_3 + +.align 4 +.alt_entry _ecp_nistz256_mul_by_3 +_ecp_nistz256_mul_by_3: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + mov x4,x14 + mov x5,x15 + mov x6,x16 + mov x7,x17 + + bl __ecp_nistz256_add_to // ret = a+a // 2*a + + mov x8,x4 + mov x9,x5 + mov x10,x6 + mov x11,x7 + + bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4], +// const BN_ULONG x2[4]); +.globl _ecp_nistz256_sub +.private_extern _ecp_nistz256_sub + +.align 4 +.alt_entry _ecp_nistz256_sub +_ecp_nistz256_sub: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sub_from + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl _ecp_nistz256_neg +.private_extern _ecp_nistz256_neg + +.align 4 +.alt_entry _ecp_nistz256_neg +_ecp_nistz256_neg: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x2,x1 + mov x14,xzr // a = 0 + mov x15,xzr + mov x16,xzr + mov x17,xzr + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sub_from + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded +// to x4-x7 and b[0] - to x3 + +.align 4 +.alt_entry __ecp_nistz256_mul_mont +__ecp_nistz256_mul_mont: + mul x14,x4,x3 // a[0]*b[0] + umulh x8,x4,x3 + + mul x15,x5,x3 // a[1]*b[0] + umulh x9,x5,x3 + + mul x16,x6,x3 // a[2]*b[0] + umulh x10,x6,x3 + + mul x17,x7,x3 // a[3]*b[0] + umulh x11,x7,x3 + ldr x3,[x2,#8] // b[1] + + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adc x19,xzr,x11 + mov x20,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + ldr x3,[x2,#8*(1+1)] // b[1+1] + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + ldr x3,[x2,#8*(2+1)] // b[2+1] + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + // last reduction + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + adcs x17,x19,x11 + adc x19,x20,xzr + + adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x19,xzr // did it borrow? + + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] + + ret + + +// note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded +// to x4-x7 + +.align 4 +.alt_entry __ecp_nistz256_sqr_mont +__ecp_nistz256_sqr_mont: // | | | | | |a1*a0| | // | | | | |a2*a0| | | // | |a3*a2|a3*a0| | | | @@ -175,132 +383,848 @@ // multiplication result, which can't overflow, because it // can never be all ones. - mul x14,x4,x3 // a[1]*a[0] - umulh x9,x4,x3 - mul x15,x5,x3 // a[2]*a[0] - umulh x10,x5,x3 - mul x16,x6,x3 // a[3]*a[0] - umulh x17,x6,x3 - - adds x15,x15,x9 // accumulate high parts of multiplication - mul x8,x5,x4 // a[2]*a[1] + mul x15,x5,x4 // a[1]*a[0] umulh x9,x5,x4 - adcs x16,x16,x10 - mul x10,x6,x4 // a[3]*a[1] - umulh x11,x6,x4 - adc x17,x17,xzr // can't overflow + mul x16,x6,x4 // a[2]*a[0] + umulh x10,x6,x4 + mul x17,x7,x4 // a[3]*a[0] + umulh x19,x7,x4 - mul x19,x6,x5 // a[3]*a[2] - umulh x20,x6,x5 + adds x16,x16,x9 // accumulate high parts of multiplication + mul x8,x6,x5 // a[2]*a[1] + umulh x9,x6,x5 + adcs x17,x17,x10 + mul x10,x7,x5 // a[3]*a[1] + umulh x11,x7,x5 + adc x19,x19,xzr // can't overflow + + mul x20,x7,x6 // a[3]*a[2] + umulh x1,x7,x6 adds x9,x9,x10 // accumulate high parts of multiplication - mul x13,x3,x3 // a[0]*a[0] + mul x14,x4,x4 // a[0]*a[0] adc x10,x11,xzr // can't overflow - adds x16,x16,x8 // accumulate low parts of multiplication - umulh x3,x3,x3 - adcs x17,x17,x9 - mul x9,x4,x4 // a[1]*a[1] - adcs x19,x19,x10 + adds x17,x17,x8 // accumulate low parts of multiplication umulh x4,x4,x4 - adc x20,x20,xzr // can't overflow - - adds x14,x14,x14 // acc[1-6]*=2 - mul x10,x5,x5 // a[2]*a[2] - adcs x15,x15,x15 + adcs x19,x19,x9 + mul x9,x5,x5 // a[1]*a[1] + adcs x20,x20,x10 umulh x5,x5,x5 + adc x1,x1,xzr // can't overflow + + adds x15,x15,x15 // acc[1-6]*=2 + mul x10,x6,x6 // a[2]*a[2] adcs x16,x16,x16 - mul x11,x6,x6 // a[3]*a[3] - adcs x17,x17,x17 umulh x6,x6,x6 + adcs x17,x17,x17 + mul x11,x7,x7 // a[3]*a[3] adcs x19,x19,x19 + umulh x7,x7,x7 adcs x20,x20,x20 + adcs x1,x1,x1 adc x2,xzr,xzr - adds x14,x14,x3 // +a[i]*a[i] - adcs x15,x15,x9 - adcs x16,x16,x4 - adcs x17,x17,x10 - adcs x19,x19,x5 - lsl x8,x13,#32 - adcs x20,x20,x11 - lsr x9,x13,#32 - adc x2,x2,x6 - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - adc x16,x11,xzr // can't overflow + adds x15,x15,x4 // +a[i]*a[i] + adcs x16,x16,x9 + adcs x17,x17,x5 + adcs x19,x19,x10 + adcs x20,x20,x6 + lsl x8,x14,#32 + adcs x1,x1,x11 + lsr x9,x14,#32 + adc x2,x2,x7 + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + adc x17,x11,xzr // can't overflow - mov w7, #-1 // poly1 = 0x00000000ffffffff - adds x13,x13,x17 // accumulate upper half - adcs x14,x14,x19 + adds x14,x14,x19 // accumulate upper half adcs x15,x15,x20 - adcs x16,x16,x2 - adc x17,xzr,xzr + adcs x16,x16,x1 + adcs x17,x17,x2 + adc x19,xzr,xzr - neg x12,x7 // poly3 = 0xffffffff00000001 - adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus - sbcs x9,x14,x7 - sbcs x10,x15,xzr - sbcs x11,x16,x12 - sbcs xzr,x17,xzr // did it borrow? + adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x19,xzr // did it borrow? - csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus - csel x14,x14,x9,lo - csel x15,x15,x10,lo - stp x13,x14,[x0] - csel x16,x16,x11,lo - stp x15,x16,[x0,#16] + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 + ret + + +// Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to +// x4-x7 and x8-x11. This is done because it's used in multiple +// contexts, e.g. in multiplication by 2 and 3... + +.align 4 +.alt_entry __ecp_nistz256_add_to +__ecp_nistz256_add_to: + adds x14,x14,x8 // ret = a+b + adcs x15,x15,x9 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x1,xzr,xzr // zap x1 + + adds x8,x14,#1 // subs x8,x4,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x1,xzr // did subtraction borrow? + + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] + + ret + + + +.align 4 +.alt_entry __ecp_nistz256_sub_from +__ecp_nistz256_sub_from: + ldp x8,x9,[x2] + ldp x10,x11,[x2,#16] + subs x14,x14,x8 // ret = a-b + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbcs x17,x17,x11 + sbc x1,xzr,xzr // zap x1 + + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adc x11,x17,x13 + cmp x1,xzr // did subtraction borrow? + + csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret + csel x15,x15,x9,eq + csel x16,x16,x10,eq + stp x14,x15,[x0] + csel x17,x17,x11,eq + stp x16,x17,[x0,#16] + + ret + + + +.align 4 +.alt_entry __ecp_nistz256_sub_morf +__ecp_nistz256_sub_morf: + ldp x8,x9,[x2] + ldp x10,x11,[x2,#16] + subs x14,x8,x14 // ret = b-a + sbcs x15,x9,x15 + sbcs x16,x10,x16 + sbcs x17,x11,x17 + sbc x1,xzr,xzr // zap x1 + + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adc x11,x17,x13 + cmp x1,xzr // did subtraction borrow? + + csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret + csel x15,x15,x9,eq + csel x16,x16,x10,eq + stp x14,x15,[x0] + csel x17,x17,x11,eq + stp x16,x17,[x0,#16] + + ret + + + +.align 4 +.alt_entry __ecp_nistz256_div_by_2 +__ecp_nistz256_div_by_2: + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = a+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adcs x11,x17,x13 + adc x1,xzr,xzr // zap x1 + tst x14,#1 // is a even? + + csel x14,x14,x8,eq // ret = even ? a : a+modulus + csel x15,x15,x9,eq + csel x16,x16,x10,eq + csel x17,x17,x11,eq + csel x1,xzr,x1,eq + + lsr x14,x14,#1 // ret >>= 1 + orr x14,x14,x15,lsl#63 + lsr x15,x15,#1 + orr x15,x15,x16,lsl#63 + lsr x16,x16,#1 + orr x16,x16,x17,lsl#63 + lsr x17,x17,#1 + stp x14,x15,[x0] + orr x17,x17,x1,lsl#63 + stp x16,x17,[x0,#16] + + ret + +.globl _ecp_nistz256_point_double +.private_extern _ecp_nistz256_point_double + +.align 5 +.alt_entry _ecp_nistz256_point_double +_ecp_nistz256_point_double: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + sub sp,sp,#32*4 + +Ldouble_shortcut: + ldp x14,x15,[x1,#32] + mov x21,x0 + ldp x16,x17,[x1,#48] + mov x22,x1 + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + mov x8,x14 + ldr x13,[x13,#24] + mov x9,x15 + ldp x4,x5,[x22,#64] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[x22,#64+16] + add x0,sp,#0 + bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y); + + add x0,sp,#64 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z); + + ldp x8,x9,[x22] + ldp x10,x11,[x22,#16] + mov x4,x14 // put Zsqr aside for p256_sub + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x0,sp,#32 + bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x); + + add x2,x22,#0 + mov x14,x4 // restore Zsqr + mov x15,x5 + ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont + mov x16,x6 + mov x17,x7 + ldp x6,x7,[sp,#0+16] + add x0,sp,#64 + bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr); + + add x0,sp,#0 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S); + + ldr x3,[x22,#32] + ldp x4,x5,[x22,#64] + ldp x6,x7,[x22,#64+16] + add x2,x22,#32 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y); + + mov x8,x14 + mov x9,x15 + ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[sp,#0+16] + add x0,x21,#64 + bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0); + + add x0,sp,#96 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S); + + ldr x3,[sp,#64] // forward load for p256_mul_mont + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x0,x21,#32 + bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0); + + add x2,sp,#64 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr); + + mov x8,x14 // duplicate M + mov x9,x15 + mov x10,x16 + mov x11,x17 + mov x4,x14 // put M aside + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x0,sp,#32 + bl __ecp_nistz256_add_to + mov x8,x4 // restore M + mov x9,x5 + ldr x3,[x22] // forward load for p256_mul_mont + mov x10,x6 + ldp x4,x5,[sp,#0] + mov x11,x7 + ldp x6,x7,[sp,#0+16] + bl __ecp_nistz256_add_to // p256_mul_by_3(M, M); + + add x2,x22,#0 + add x0,sp,#0 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x); + + mov x8,x14 + mov x9,x15 + ldp x4,x5,[sp,#32] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[sp,#32+16] + add x0,sp,#96 + bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S); + + add x0,x21,#0 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M); + + add x2,sp,#96 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0); + + add x2,sp,#0 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x); + + ldr x3,[sp,#32] + mov x4,x14 // copy S + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x2,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M); + + add x2,x21,#32 + add x0,x21,#32 + bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y); + + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x29,x30,[sp],#96 AARCH64_VALIDATE_LINK_REGISTER ret -.section __TEXT,__const +.globl _ecp_nistz256_point_add +.private_extern _ecp_nistz256_point_add .align 5 -p256_constants: -Lord: -.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 -LordK: -.quad 0xccd1c8aaee00bc4f -.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 -.align 2 -.text +.alt_entry _ecp_nistz256_point_add +_ecp_nistz256_point_add: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#32*12 + ldp x4,x5,[x2,#64] // in2_z + ldp x6,x7,[x2,#64+16] + mov x21,x0 + mov x22,x1 + mov x23,x2 + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] + orr x8,x4,x5 + orr x10,x6,x7 + orr x25,x8,x10 + cmp x25,#0 + csetm x25,ne // ~in2infty + add x0,sp,#192 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z); + + ldp x4,x5,[x22,#64] // in1_z + ldp x6,x7,[x22,#64+16] + orr x8,x4,x5 + orr x10,x6,x7 + orr x24,x8,x10 + cmp x24,#0 + csetm x24,ne // ~in1infty + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); + + ldr x3,[x23,#64] + ldp x4,x5,[sp,#192] + ldp x6,x7,[sp,#192+16] + add x2,x23,#64 + add x0,sp,#320 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,x22,#64 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); + + ldr x3,[x22,#32] + ldp x4,x5,[sp,#320] + ldp x6,x7,[sp,#320+16] + add x2,x22,#32 + add x0,sp,#320 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y); + + ldr x3,[x23,#32] + ldp x4,x5,[sp,#352] + ldp x6,x7,[sp,#352+16] + add x2,x23,#32 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); + + add x2,sp,#320 + ldr x3,[sp,#192] // forward load for p256_mul_mont + ldp x4,x5,[x22] + ldp x6,x7,[x22,#16] + add x0,sp,#160 + bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1); + + orr x14,x14,x15 // see if result is zero + orr x16,x16,x17 + orr x26,x14,x16 // ~is_equal(S1,S2) + + add x2,sp,#192 + add x0,sp,#256 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr); + + ldr x3,[sp,#128] + ldp x4,x5,[x23] + ldp x6,x7,[x23,#16] + add x2,sp,#128 + add x0,sp,#288 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr); + + add x2,sp,#256 + ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont + ldp x6,x7,[sp,#160+16] + add x0,sp,#96 + bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1); + + orr x14,x14,x15 // see if result is zero + orr x16,x16,x17 + orr x14,x14,x16 // ~is_equal(U1,U2) + + mvn x27,x24 // -1/0 -> 0/-1 + mvn x28,x25 // -1/0 -> 0/-1 + orr x14,x14,x27 + orr x14,x14,x28 + orr x14,x14,x26 + cbnz x14,Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2)) + +Ladd_double: + mov x1,x22 + mov x0,x21 + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames + b Ldouble_shortcut + +.align 4 +Ladd_proceed: + add x0,sp,#192 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#96] + ldp x6,x7,[sp,#96+16] + add x2,x22,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); + + ldp x4,x5,[sp,#96] + ldp x6,x7,[sp,#96+16] + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); + + ldr x3,[x23,#64] + ldp x4,x5,[sp,#64] + ldp x6,x7,[sp,#64+16] + add x2,x23,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z); + + ldr x3,[sp,#96] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,sp,#96 + add x0,sp,#224 + bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); + + ldr x3,[sp,#128] + ldp x4,x5,[sp,#256] + ldp x6,x7,[sp,#256+16] + add x2,sp,#128 + add x0,sp,#288 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr); + + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + add x0,sp,#128 + bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); + + add x2,sp,#192 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); + + add x2,sp,#224 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); + + add x2,sp,#288 + ldr x3,[sp,#224] // forward load for p256_mul_mont + ldp x4,x5,[sp,#320] + ldp x6,x7,[sp,#320+16] + add x0,sp,#32 + bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); + + add x2,sp,#224 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub); + + ldr x3,[sp,#160] + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x2,sp,#160 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); + + add x2,sp,#352 + bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); + + ldp x4,x5,[sp,#0] // res + ldp x6,x7,[sp,#0+16] + ldp x8,x9,[x23] // in2 + ldp x10,x11,[x23,#16] + ldp x14,x15,[x22,#0] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#0+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+0+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+0+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#0+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#0+48] + stp x14,x15,[x21,#0] + stp x16,x17,[x21,#0+16] + ldp x14,x15,[x22,#32] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#32+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+32+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+32+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#32+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#32+48] + stp x14,x15,[x21,#32] + stp x16,x17,[x21,#32+16] + ldp x14,x15,[x22,#64] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#64+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + csel x14,x8,x14,ne + csel x15,x9,x15,ne + csel x16,x10,x16,ne + csel x17,x11,x17,ne + stp x14,x15,[x21,#64] + stp x16,x17,[x21,#64+16] + +Ladd_done: + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +.globl _ecp_nistz256_point_add_affine +.private_extern _ecp_nistz256_point_add_affine + +.align 5 +.alt_entry _ecp_nistz256_point_add_affine +_ecp_nistz256_point_add_affine: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + sub sp,sp,#32*10 + + mov x21,x0 + mov x22,x1 + mov x23,x2 + adrp x13,Lpoly@PAGE + add x13,x13,Lpoly@PAGEOFF + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + ldp x4,x5,[x1,#64] // in1_z + ldp x6,x7,[x1,#64+16] + orr x8,x4,x5 + orr x10,x6,x7 + orr x24,x8,x10 + cmp x24,#0 + csetm x24,ne // ~in1infty + + ldp x14,x15,[x2] // in2_x + ldp x16,x17,[x2,#16] + ldp x8,x9,[x2,#32] // in2_y + ldp x10,x11,[x2,#48] + orr x14,x14,x15 + orr x16,x16,x17 + orr x8,x8,x9 + orr x10,x10,x11 + orr x14,x14,x16 + orr x8,x8,x10 + orr x25,x14,x8 + cmp x25,#0 + csetm x25,ne // ~in2infty + + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); + + mov x4,x14 + mov x5,x15 + mov x6,x16 + mov x7,x17 + ldr x3,[x23] + add x2,x23,#0 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x); + + add x2,x22,#0 + ldr x3,[x22,#64] // forward load for p256_mul_mont + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x0,sp,#160 + bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x); + + add x2,x22,#64 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#160] + ldp x6,x7,[sp,#160+16] + add x2,x22,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); + + ldr x3,[x23,#32] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,x23,#32 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); + + add x2,x22,#32 + ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont + ldp x6,x7,[sp,#160+16] + add x0,sp,#192 + bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y); + + add x0,sp,#224 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); + + ldp x4,x5,[sp,#192] + ldp x6,x7,[sp,#192+16] + add x0,sp,#288 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); + + ldr x3,[sp,#160] + ldp x4,x5,[sp,#224] + ldp x6,x7,[sp,#224+16] + add x2,sp,#160 + add x0,sp,#256 + bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); + + ldr x3,[x22] + ldp x4,x5,[sp,#224] + ldp x6,x7,[sp,#224+16] + add x2,x22,#0 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr); + + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + add x0,sp,#224 + bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); + + add x2,sp,#288 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); + + add x2,sp,#256 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); + + add x2,sp,#96 + ldr x3,[x22,#32] // forward load for p256_mul_mont + ldp x4,x5,[sp,#256] + ldp x6,x7,[sp,#256+16] + add x0,sp,#32 + bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); + + add x2,x22,#32 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub); + + ldr x3,[sp,#192] + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x2,sp,#192 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); + + add x2,sp,#128 + bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); + + ldp x4,x5,[sp,#0] // res + ldp x6,x7,[sp,#0+16] + ldp x8,x9,[x23] // in2 + ldp x10,x11,[x23,#16] + ldp x14,x15,[x22,#0] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#0+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+0+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+0+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#0+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#0+48] + stp x14,x15,[x21,#0] + stp x16,x17,[x21,#0+16] + adrp x23,Lone_mont@PAGE-64 + add x23,x23,Lone_mont@PAGEOFF-64 + ldp x14,x15,[x22,#32] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#32+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+32+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+32+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#32+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#32+48] + stp x14,x15,[x21,#32] + stp x16,x17,[x21,#32+16] + ldp x14,x15,[x22,#64] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#64+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + csel x14,x8,x14,ne + csel x15,x9,x15,ne + csel x16,x10,x16,ne + csel x17,x11,x17,ne + stp x14,x15,[x21,#64] + stp x16,x17,[x21,#64+16] + + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x29,x30,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret + +//////////////////////////////////////////////////////////////////////// // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4], // uint64_t b[4]); .globl _ecp_nistz256_ord_mul_mont .private_extern _ecp_nistz256_ord_mul_mont .align 4 +.alt_entry _ecp_nistz256_ord_mul_mont _ecp_nistz256_ord_mul_mont: AARCH64_VALID_CALL_TARGET // Armv8.3-A PAuth: even though x30 is pushed to stack it is not popped later. @@ -318,7 +1242,7 @@ ldp x12,x13,[x23,#0] ldp x21,x22,[x23,#16] - ldr x23,[x23,#32] // LordK + ldr x23,[x23,#32] mul x14,x4,x3 // a[0]*b[0] umulh x8,x4,x3 @@ -700,4 +1624,121 @@ ldr x29,[sp],#64 ret +//////////////////////////////////////////////////////////////////////// +// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); +.globl _ecp_nistz256_select_w5 +.private_extern _ecp_nistz256_select_w5 + +.align 4 +.alt_entry _ecp_nistz256_select_w5 +_ecp_nistz256_select_w5: + AARCH64_VALID_CALL_TARGET + + // x10 := x0 + // w9 := 0; loop counter and incremented internal index + mov x10, x0 + mov w9, #0 + + // [v16-v21] := 0 + movi v16.16b, #0 + movi v17.16b, #0 + movi v18.16b, #0 + movi v19.16b, #0 + movi v20.16b, #0 + movi v21.16b, #0 + +Lselect_w5_loop: + // Loop 16 times. + + // Increment index (loop counter); tested at the end of the loop + add w9, w9, #1 + + // [v22-v27] := Load a (3*256-bit = 6*128-bit) table entry starting at x1 + // and advance x1 to point to the next entry + ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 + + // x11 := (w9 == w2)? All 1s : All 0s + cmp w9, w2 + csetm x11, eq + + // continue loading ... + ld1 {v26.2d, v27.2d}, [x1],#32 + + // duplicate mask_64 into Mask (all 0s or all 1s) + dup v3.2d, x11 + + // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] + // i.e., values in output registers will remain the same if w9 != w2 + bit v16.16b, v22.16b, v3.16b + bit v17.16b, v23.16b, v3.16b + + bit v18.16b, v24.16b, v3.16b + bit v19.16b, v25.16b, v3.16b + + bit v20.16b, v26.16b, v3.16b + bit v21.16b, v27.16b, v3.16b + + // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back + tbz w9, #4, Lselect_w5_loop + + // Write [v16-v21] to memory at the output pointer + st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x10],#64 + st1 {v20.2d, v21.2d}, [x10] + + ret + + + +//////////////////////////////////////////////////////////////////////// +// void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); +.globl _ecp_nistz256_select_w7 +.private_extern _ecp_nistz256_select_w7 + +.align 4 +.alt_entry _ecp_nistz256_select_w7 +_ecp_nistz256_select_w7: + AARCH64_VALID_CALL_TARGET + + // w9 := 0; loop counter and incremented internal index + mov w9, #0 + + // [v16-v21] := 0 + movi v16.16b, #0 + movi v17.16b, #0 + movi v18.16b, #0 + movi v19.16b, #0 + +Lselect_w7_loop: + // Loop 64 times. + + // Increment index (loop counter); tested at the end of the loop + add w9, w9, #1 + + // [v22-v25] := Load a (2*256-bit = 4*128-bit) table entry starting at x1 + // and advance x1 to point to the next entry + ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 + + // x11 := (w9 == w2)? All 1s : All 0s + cmp w9, w2 + csetm x11, eq + + // duplicate mask_64 into Mask (all 0s or all 1s) + dup v3.2d, x11 + + // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] + // i.e., values in output registers will remain the same if w9 != w2 + bit v16.16b, v22.16b, v3.16b + bit v17.16b, v23.16b, v3.16b + + bit v18.16b, v24.16b, v3.16b + bit v19.16b, v25.16b, v3.16b + + // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back + tbz w9, #6, Lselect_w7_loop + + // Write [v16-v19] to memory at the output pointer + st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x0] + + ret + #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__APPLE__)
diff --git a/gen/bcm/p256-armv8-asm-linux.S b/gen/bcm/p256-armv8-asm-linux.S index e7c83bc..56d5f00 100644 --- a/gen/bcm/p256-armv8-asm-linux.S +++ b/gen/bcm/p256-armv8-asm-linux.S
@@ -4,6 +4,27 @@ #include <openssl/asm_base.h> #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__ELF__) +.section .rodata +.align 5 +p256_constants: +.Lpoly: +.quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 +.LRR: // 2^512 mod P precomputed for NIST P256 polynomial +.quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd +.Lone_mont: +.quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe +.Lone: +.quad 1,0,0,0 +.Lord: +.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 +.LordK: +.quad 0xccd1c8aaee00bc4f +.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 +.align 2 +.text + +// void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4], +// const BN_ULONG x2[4]); .globl ecp_nistz256_mul_mont .hidden ecp_nistz256_mul_mont .type ecp_nistz256_mul_mont,%function @@ -14,135 +35,15 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - mul x13,x3,x2 // a[0]*b[0] - umulh x8,x3,x2 + ldr x3,[x2] // bp[0] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] - mul x14,x4,x2 // a[1]*b[0] - umulh x9,x4,x2 - - mul x15,x5,x2 // a[2]*b[0] - umulh x10,x5,x2 - - mul x16,x6,x2 // a[3]*b[0] - umulh x11,x6,x2 - ldr x2,[x1,#8] // b[1] - - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adc x17,xzr,x11 - mov x19,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - ldr x2,[x1,#8*(1+1)] // b[1+1] - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - ldr x2,[x1,#8*(2+1)] // b[2+1] - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - mov w7, #-1 // poly1 = 0x00000000ffffffff - // last reduction - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - adcs x16,x17,x11 - adc x17,x19,xzr - - neg x12,x7 // poly3 = 0xffffffff00000001 - adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus - sbcs x9,x14,x7 - sbcs x10,x15,xzr - sbcs x11,x16,x12 - sbcs xzr,x17,xzr // did it borrow? - - csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus - csel x14,x14,x9,lo - csel x15,x15,x10,lo - stp x13,x14,[x0] - csel x16,x16,x11,lo - stp x15,x16,[x0,#16] + bl __ecp_nistz256_mul_mont ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 @@ -150,6 +51,7 @@ ret .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont +// void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); .globl ecp_nistz256_sqr_mont .hidden ecp_nistz256_sqr_mont .type ecp_nistz256_sqr_mont,%function @@ -160,6 +62,305 @@ add x29,sp,#0 stp x19,x20,[sp,#16] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sqr_mont + + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont + +// void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_div_by_2 +.hidden ecp_nistz256_div_by_2 +.type ecp_nistz256_div_by_2,%function +.align 4 +ecp_nistz256_div_by_2: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_div_by_2 + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2 + +// void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_mul_by_2 +.hidden ecp_nistz256_mul_by_2 +.type ecp_nistz256_mul_by_2,%function +.align 4 +ecp_nistz256_mul_by_2: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + + bl __ecp_nistz256_add_to // ret = a+a // 2*a + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2 + +// void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_mul_by_3 +.hidden ecp_nistz256_mul_by_3 +.type ecp_nistz256_mul_by_3,%function +.align 4 +ecp_nistz256_mul_by_3: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + mov x4,x14 + mov x5,x15 + mov x6,x16 + mov x7,x17 + + bl __ecp_nistz256_add_to // ret = a+a // 2*a + + mov x8,x4 + mov x9,x5 + mov x10,x6 + mov x11,x7 + + bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3 + +// void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4], +// const BN_ULONG x2[4]); +.globl ecp_nistz256_sub +.hidden ecp_nistz256_sub +.type ecp_nistz256_sub,%function +.align 4 +ecp_nistz256_sub: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sub_from + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_sub,.-ecp_nistz256_sub + +// void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_neg +.hidden ecp_nistz256_neg +.type ecp_nistz256_neg,%function +.align 4 +ecp_nistz256_neg: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x2,x1 + mov x14,xzr // a = 0 + mov x15,xzr + mov x16,xzr + mov x17,xzr + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sub_from + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_neg,.-ecp_nistz256_neg + +// note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded +// to x4-x7 and b[0] - to x3 +.type __ecp_nistz256_mul_mont,%function +.align 4 +__ecp_nistz256_mul_mont: + mul x14,x4,x3 // a[0]*b[0] + umulh x8,x4,x3 + + mul x15,x5,x3 // a[1]*b[0] + umulh x9,x5,x3 + + mul x16,x6,x3 // a[2]*b[0] + umulh x10,x6,x3 + + mul x17,x7,x3 // a[3]*b[0] + umulh x11,x7,x3 + ldr x3,[x2,#8] // b[1] + + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adc x19,xzr,x11 + mov x20,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + ldr x3,[x2,#8*(1+1)] // b[1+1] + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + ldr x3,[x2,#8*(2+1)] // b[2+1] + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + // last reduction + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + adcs x17,x19,x11 + adc x19,x20,xzr + + adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x19,xzr // did it borrow? + + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] + + ret +.size __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont + +// note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded +// to x4-x7 +.type __ecp_nistz256_sqr_mont,%function +.align 4 +__ecp_nistz256_sqr_mont: // | | | | | |a1*a0| | // | | | | |a2*a0| | | // | |a3*a2|a3*a0| | | | @@ -174,126 +375,834 @@ // multiplication result, which can't overflow, because it // can never be all ones. - mul x14,x4,x3 // a[1]*a[0] - umulh x9,x4,x3 - mul x15,x5,x3 // a[2]*a[0] - umulh x10,x5,x3 - mul x16,x6,x3 // a[3]*a[0] - umulh x17,x6,x3 - - adds x15,x15,x9 // accumulate high parts of multiplication - mul x8,x5,x4 // a[2]*a[1] + mul x15,x5,x4 // a[1]*a[0] umulh x9,x5,x4 - adcs x16,x16,x10 - mul x10,x6,x4 // a[3]*a[1] - umulh x11,x6,x4 - adc x17,x17,xzr // can't overflow + mul x16,x6,x4 // a[2]*a[0] + umulh x10,x6,x4 + mul x17,x7,x4 // a[3]*a[0] + umulh x19,x7,x4 - mul x19,x6,x5 // a[3]*a[2] - umulh x20,x6,x5 + adds x16,x16,x9 // accumulate high parts of multiplication + mul x8,x6,x5 // a[2]*a[1] + umulh x9,x6,x5 + adcs x17,x17,x10 + mul x10,x7,x5 // a[3]*a[1] + umulh x11,x7,x5 + adc x19,x19,xzr // can't overflow + + mul x20,x7,x6 // a[3]*a[2] + umulh x1,x7,x6 adds x9,x9,x10 // accumulate high parts of multiplication - mul x13,x3,x3 // a[0]*a[0] + mul x14,x4,x4 // a[0]*a[0] adc x10,x11,xzr // can't overflow - adds x16,x16,x8 // accumulate low parts of multiplication - umulh x3,x3,x3 - adcs x17,x17,x9 - mul x9,x4,x4 // a[1]*a[1] - adcs x19,x19,x10 + adds x17,x17,x8 // accumulate low parts of multiplication umulh x4,x4,x4 - adc x20,x20,xzr // can't overflow - - adds x14,x14,x14 // acc[1-6]*=2 - mul x10,x5,x5 // a[2]*a[2] - adcs x15,x15,x15 + adcs x19,x19,x9 + mul x9,x5,x5 // a[1]*a[1] + adcs x20,x20,x10 umulh x5,x5,x5 + adc x1,x1,xzr // can't overflow + + adds x15,x15,x15 // acc[1-6]*=2 + mul x10,x6,x6 // a[2]*a[2] adcs x16,x16,x16 - mul x11,x6,x6 // a[3]*a[3] - adcs x17,x17,x17 umulh x6,x6,x6 + adcs x17,x17,x17 + mul x11,x7,x7 // a[3]*a[3] adcs x19,x19,x19 + umulh x7,x7,x7 adcs x20,x20,x20 + adcs x1,x1,x1 adc x2,xzr,xzr - adds x14,x14,x3 // +a[i]*a[i] - adcs x15,x15,x9 - adcs x16,x16,x4 - adcs x17,x17,x10 - adcs x19,x19,x5 - lsl x8,x13,#32 - adcs x20,x20,x11 - lsr x9,x13,#32 - adc x2,x2,x6 - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - adc x16,x11,xzr // can't overflow + adds x15,x15,x4 // +a[i]*a[i] + adcs x16,x16,x9 + adcs x17,x17,x5 + adcs x19,x19,x10 + adcs x20,x20,x6 + lsl x8,x14,#32 + adcs x1,x1,x11 + lsr x9,x14,#32 + adc x2,x2,x7 + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + adc x17,x11,xzr // can't overflow - mov w7, #-1 // poly1 = 0x00000000ffffffff - adds x13,x13,x17 // accumulate upper half - adcs x14,x14,x19 + adds x14,x14,x19 // accumulate upper half adcs x15,x15,x20 - adcs x16,x16,x2 - adc x17,xzr,xzr + adcs x16,x16,x1 + adcs x17,x17,x2 + adc x19,xzr,xzr - neg x12,x7 // poly3 = 0xffffffff00000001 - adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus - sbcs x9,x14,x7 - sbcs x10,x15,xzr - sbcs x11,x16,x12 - sbcs xzr,x17,xzr // did it borrow? + adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x19,xzr // did it borrow? - csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus - csel x14,x14,x9,lo - csel x15,x15,x10,lo - stp x13,x14,[x0] - csel x16,x16,x11,lo - stp x15,x16,[x0,#16] + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 + ret +.size __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont + +// Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to +// x4-x7 and x8-x11. This is done because it's used in multiple +// contexts, e.g. in multiplication by 2 and 3... +.type __ecp_nistz256_add_to,%function +.align 4 +__ecp_nistz256_add_to: + adds x14,x14,x8 // ret = a+b + adcs x15,x15,x9 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x1,xzr,xzr // zap x1 + + adds x8,x14,#1 // subs x8,x4,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x1,xzr // did subtraction borrow? + + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] + + ret +.size __ecp_nistz256_add_to,.-__ecp_nistz256_add_to + +.type __ecp_nistz256_sub_from,%function +.align 4 +__ecp_nistz256_sub_from: + ldp x8,x9,[x2] + ldp x10,x11,[x2,#16] + subs x14,x14,x8 // ret = a-b + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbcs x17,x17,x11 + sbc x1,xzr,xzr // zap x1 + + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adc x11,x17,x13 + cmp x1,xzr // did subtraction borrow? + + csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret + csel x15,x15,x9,eq + csel x16,x16,x10,eq + stp x14,x15,[x0] + csel x17,x17,x11,eq + stp x16,x17,[x0,#16] + + ret +.size __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from + +.type __ecp_nistz256_sub_morf,%function +.align 4 +__ecp_nistz256_sub_morf: + ldp x8,x9,[x2] + ldp x10,x11,[x2,#16] + subs x14,x8,x14 // ret = b-a + sbcs x15,x9,x15 + sbcs x16,x10,x16 + sbcs x17,x11,x17 + sbc x1,xzr,xzr // zap x1 + + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adc x11,x17,x13 + cmp x1,xzr // did subtraction borrow? + + csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret + csel x15,x15,x9,eq + csel x16,x16,x10,eq + stp x14,x15,[x0] + csel x17,x17,x11,eq + stp x16,x17,[x0,#16] + + ret +.size __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf + +.type __ecp_nistz256_div_by_2,%function +.align 4 +__ecp_nistz256_div_by_2: + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = a+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adcs x11,x17,x13 + adc x1,xzr,xzr // zap x1 + tst x14,#1 // is a even? + + csel x14,x14,x8,eq // ret = even ? a : a+modulus + csel x15,x15,x9,eq + csel x16,x16,x10,eq + csel x17,x17,x11,eq + csel x1,xzr,x1,eq + + lsr x14,x14,#1 // ret >>= 1 + orr x14,x14,x15,lsl#63 + lsr x15,x15,#1 + orr x15,x15,x16,lsl#63 + lsr x16,x16,#1 + orr x16,x16,x17,lsl#63 + lsr x17,x17,#1 + stp x14,x15,[x0] + orr x17,x17,x1,lsl#63 + stp x16,x17,[x0,#16] + + ret +.size __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2 +.globl ecp_nistz256_point_double +.hidden ecp_nistz256_point_double +.type ecp_nistz256_point_double,%function +.align 5 +ecp_nistz256_point_double: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + sub sp,sp,#32*4 + +.Ldouble_shortcut: + ldp x14,x15,[x1,#32] + mov x21,x0 + ldp x16,x17,[x1,#48] + mov x22,x1 + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + mov x8,x14 + ldr x13,[x13,#24] + mov x9,x15 + ldp x4,x5,[x22,#64] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[x22,#64+16] + add x0,sp,#0 + bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y); + + add x0,sp,#64 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z); + + ldp x8,x9,[x22] + ldp x10,x11,[x22,#16] + mov x4,x14 // put Zsqr aside for p256_sub + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x0,sp,#32 + bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x); + + add x2,x22,#0 + mov x14,x4 // restore Zsqr + mov x15,x5 + ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont + mov x16,x6 + mov x17,x7 + ldp x6,x7,[sp,#0+16] + add x0,sp,#64 + bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr); + + add x0,sp,#0 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S); + + ldr x3,[x22,#32] + ldp x4,x5,[x22,#64] + ldp x6,x7,[x22,#64+16] + add x2,x22,#32 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y); + + mov x8,x14 + mov x9,x15 + ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[sp,#0+16] + add x0,x21,#64 + bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0); + + add x0,sp,#96 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S); + + ldr x3,[sp,#64] // forward load for p256_mul_mont + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x0,x21,#32 + bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0); + + add x2,sp,#64 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr); + + mov x8,x14 // duplicate M + mov x9,x15 + mov x10,x16 + mov x11,x17 + mov x4,x14 // put M aside + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x0,sp,#32 + bl __ecp_nistz256_add_to + mov x8,x4 // restore M + mov x9,x5 + ldr x3,[x22] // forward load for p256_mul_mont + mov x10,x6 + ldp x4,x5,[sp,#0] + mov x11,x7 + ldp x6,x7,[sp,#0+16] + bl __ecp_nistz256_add_to // p256_mul_by_3(M, M); + + add x2,x22,#0 + add x0,sp,#0 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x); + + mov x8,x14 + mov x9,x15 + ldp x4,x5,[sp,#32] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[sp,#32+16] + add x0,sp,#96 + bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S); + + add x0,x21,#0 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M); + + add x2,sp,#96 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0); + + add x2,sp,#0 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x); + + ldr x3,[sp,#32] + mov x4,x14 // copy S + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x2,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M); + + add x2,x21,#32 + add x0,x21,#32 + bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y); + + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x29,x30,[sp],#96 AARCH64_VALIDATE_LINK_REGISTER ret -.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont -.section .rodata - +.size ecp_nistz256_point_double,.-ecp_nistz256_point_double +.globl ecp_nistz256_point_add +.hidden ecp_nistz256_point_add +.type ecp_nistz256_point_add,%function .align 5 -p256_constants: -.Lord: -.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 -.LordK: -.quad 0xccd1c8aaee00bc4f -.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 -.align 2 -.text +ecp_nistz256_point_add: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#32*12 + ldp x4,x5,[x2,#64] // in2_z + ldp x6,x7,[x2,#64+16] + mov x21,x0 + mov x22,x1 + mov x23,x2 + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + orr x8,x4,x5 + orr x10,x6,x7 + orr x25,x8,x10 + cmp x25,#0 + csetm x25,ne // ~in2infty + add x0,sp,#192 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z); + + ldp x4,x5,[x22,#64] // in1_z + ldp x6,x7,[x22,#64+16] + orr x8,x4,x5 + orr x10,x6,x7 + orr x24,x8,x10 + cmp x24,#0 + csetm x24,ne // ~in1infty + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); + + ldr x3,[x23,#64] + ldp x4,x5,[sp,#192] + ldp x6,x7,[sp,#192+16] + add x2,x23,#64 + add x0,sp,#320 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,x22,#64 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); + + ldr x3,[x22,#32] + ldp x4,x5,[sp,#320] + ldp x6,x7,[sp,#320+16] + add x2,x22,#32 + add x0,sp,#320 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y); + + ldr x3,[x23,#32] + ldp x4,x5,[sp,#352] + ldp x6,x7,[sp,#352+16] + add x2,x23,#32 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); + + add x2,sp,#320 + ldr x3,[sp,#192] // forward load for p256_mul_mont + ldp x4,x5,[x22] + ldp x6,x7,[x22,#16] + add x0,sp,#160 + bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1); + + orr x14,x14,x15 // see if result is zero + orr x16,x16,x17 + orr x26,x14,x16 // ~is_equal(S1,S2) + + add x2,sp,#192 + add x0,sp,#256 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr); + + ldr x3,[sp,#128] + ldp x4,x5,[x23] + ldp x6,x7,[x23,#16] + add x2,sp,#128 + add x0,sp,#288 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr); + + add x2,sp,#256 + ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont + ldp x6,x7,[sp,#160+16] + add x0,sp,#96 + bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1); + + orr x14,x14,x15 // see if result is zero + orr x16,x16,x17 + orr x14,x14,x16 // ~is_equal(U1,U2) + + mvn x27,x24 // -1/0 -> 0/-1 + mvn x28,x25 // -1/0 -> 0/-1 + orr x14,x14,x27 + orr x14,x14,x28 + orr x14,x14,x26 + cbnz x14,.Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2)) + +.Ladd_double: + mov x1,x22 + mov x0,x21 + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames + b .Ldouble_shortcut + +.align 4 +.Ladd_proceed: + add x0,sp,#192 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#96] + ldp x6,x7,[sp,#96+16] + add x2,x22,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); + + ldp x4,x5,[sp,#96] + ldp x6,x7,[sp,#96+16] + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); + + ldr x3,[x23,#64] + ldp x4,x5,[sp,#64] + ldp x6,x7,[sp,#64+16] + add x2,x23,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z); + + ldr x3,[sp,#96] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,sp,#96 + add x0,sp,#224 + bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); + + ldr x3,[sp,#128] + ldp x4,x5,[sp,#256] + ldp x6,x7,[sp,#256+16] + add x2,sp,#128 + add x0,sp,#288 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr); + + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + add x0,sp,#128 + bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); + + add x2,sp,#192 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); + + add x2,sp,#224 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); + + add x2,sp,#288 + ldr x3,[sp,#224] // forward load for p256_mul_mont + ldp x4,x5,[sp,#320] + ldp x6,x7,[sp,#320+16] + add x0,sp,#32 + bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); + + add x2,sp,#224 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub); + + ldr x3,[sp,#160] + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x2,sp,#160 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); + + add x2,sp,#352 + bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); + + ldp x4,x5,[sp,#0] // res + ldp x6,x7,[sp,#0+16] + ldp x8,x9,[x23] // in2 + ldp x10,x11,[x23,#16] + ldp x14,x15,[x22,#0] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#0+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+0+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+0+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#0+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#0+48] + stp x14,x15,[x21,#0] + stp x16,x17,[x21,#0+16] + ldp x14,x15,[x22,#32] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#32+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+32+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+32+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#32+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#32+48] + stp x14,x15,[x21,#32] + stp x16,x17,[x21,#32+16] + ldp x14,x15,[x22,#64] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#64+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + csel x14,x8,x14,ne + csel x15,x9,x15,ne + csel x16,x10,x16,ne + csel x17,x11,x17,ne + stp x14,x15,[x21,#64] + stp x16,x17,[x21,#64+16] + +.Ladd_done: + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_point_add,.-ecp_nistz256_point_add +.globl ecp_nistz256_point_add_affine +.hidden ecp_nistz256_point_add_affine +.type ecp_nistz256_point_add_affine,%function +.align 5 +ecp_nistz256_point_add_affine: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + sub sp,sp,#32*10 + + mov x21,x0 + mov x22,x1 + mov x23,x2 + adrp x13,.Lpoly + add x13,x13,:lo12:.Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + ldp x4,x5,[x1,#64] // in1_z + ldp x6,x7,[x1,#64+16] + orr x8,x4,x5 + orr x10,x6,x7 + orr x24,x8,x10 + cmp x24,#0 + csetm x24,ne // ~in1infty + + ldp x14,x15,[x2] // in2_x + ldp x16,x17,[x2,#16] + ldp x8,x9,[x2,#32] // in2_y + ldp x10,x11,[x2,#48] + orr x14,x14,x15 + orr x16,x16,x17 + orr x8,x8,x9 + orr x10,x10,x11 + orr x14,x14,x16 + orr x8,x8,x10 + orr x25,x14,x8 + cmp x25,#0 + csetm x25,ne // ~in2infty + + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); + + mov x4,x14 + mov x5,x15 + mov x6,x16 + mov x7,x17 + ldr x3,[x23] + add x2,x23,#0 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x); + + add x2,x22,#0 + ldr x3,[x22,#64] // forward load for p256_mul_mont + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x0,sp,#160 + bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x); + + add x2,x22,#64 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#160] + ldp x6,x7,[sp,#160+16] + add x2,x22,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); + + ldr x3,[x23,#32] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,x23,#32 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); + + add x2,x22,#32 + ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont + ldp x6,x7,[sp,#160+16] + add x0,sp,#192 + bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y); + + add x0,sp,#224 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); + + ldp x4,x5,[sp,#192] + ldp x6,x7,[sp,#192+16] + add x0,sp,#288 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); + + ldr x3,[sp,#160] + ldp x4,x5,[sp,#224] + ldp x6,x7,[sp,#224+16] + add x2,sp,#160 + add x0,sp,#256 + bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); + + ldr x3,[x22] + ldp x4,x5,[sp,#224] + ldp x6,x7,[sp,#224+16] + add x2,x22,#0 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr); + + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + add x0,sp,#224 + bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); + + add x2,sp,#288 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); + + add x2,sp,#256 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); + + add x2,sp,#96 + ldr x3,[x22,#32] // forward load for p256_mul_mont + ldp x4,x5,[sp,#256] + ldp x6,x7,[sp,#256+16] + add x0,sp,#32 + bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); + + add x2,x22,#32 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub); + + ldr x3,[sp,#192] + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x2,sp,#192 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); + + add x2,sp,#128 + bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); + + ldp x4,x5,[sp,#0] // res + ldp x6,x7,[sp,#0+16] + ldp x8,x9,[x23] // in2 + ldp x10,x11,[x23,#16] + ldp x14,x15,[x22,#0] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#0+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+0+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+0+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#0+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#0+48] + stp x14,x15,[x21,#0] + stp x16,x17,[x21,#0+16] + adrp x23,.Lone_mont-64 + add x23,x23,:lo12:.Lone_mont-64 + ldp x14,x15,[x22,#32] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#32+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+32+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+32+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#32+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#32+48] + stp x14,x15,[x21,#32] + stp x16,x17,[x21,#32+16] + ldp x14,x15,[x22,#64] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#64+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + csel x14,x8,x14,ne + csel x15,x9,x15,ne + csel x16,x10,x16,ne + csel x17,x11,x17,ne + stp x14,x15,[x21,#64] + stp x16,x17,[x21,#64+16] + + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x29,x30,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret +.size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine +//////////////////////////////////////////////////////////////////////// // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4], // uint64_t b[4]); .globl ecp_nistz256_ord_mul_mont @@ -317,7 +1226,7 @@ ldp x12,x13,[x23,#0] ldp x21,x22,[x23,#16] - ldr x23,[x23,#32] // .LordK + ldr x23,[x23,#32] mul x14,x4,x3 // a[0]*b[0] umulh x8,x4,x3 @@ -698,4 +1607,119 @@ ldr x29,[sp],#64 ret .size ecp_nistz256_ord_sqr_mont,.-ecp_nistz256_ord_sqr_mont +//////////////////////////////////////////////////////////////////////// +// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w5 +.hidden ecp_nistz256_select_w5 +.type ecp_nistz256_select_w5,%function +.align 4 +ecp_nistz256_select_w5: + AARCH64_VALID_CALL_TARGET + + // x10 := x0 + // w9 := 0; loop counter and incremented internal index + mov x10, x0 + mov w9, #0 + + // [v16-v21] := 0 + movi v16.16b, #0 + movi v17.16b, #0 + movi v18.16b, #0 + movi v19.16b, #0 + movi v20.16b, #0 + movi v21.16b, #0 + +.Lselect_w5_loop: + // Loop 16 times. + + // Increment index (loop counter); tested at the end of the loop + add w9, w9, #1 + + // [v22-v27] := Load a (3*256-bit = 6*128-bit) table entry starting at x1 + // and advance x1 to point to the next entry + ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 + + // x11 := (w9 == w2)? All 1s : All 0s + cmp w9, w2 + csetm x11, eq + + // continue loading ... + ld1 {v26.2d, v27.2d}, [x1],#32 + + // duplicate mask_64 into Mask (all 0s or all 1s) + dup v3.2d, x11 + + // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] + // i.e., values in output registers will remain the same if w9 != w2 + bit v16.16b, v22.16b, v3.16b + bit v17.16b, v23.16b, v3.16b + + bit v18.16b, v24.16b, v3.16b + bit v19.16b, v25.16b, v3.16b + + bit v20.16b, v26.16b, v3.16b + bit v21.16b, v27.16b, v3.16b + + // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back + tbz w9, #4, .Lselect_w5_loop + + // Write [v16-v21] to memory at the output pointer + st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x10],#64 + st1 {v20.2d, v21.2d}, [x10] + + ret +.size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5 + + +//////////////////////////////////////////////////////////////////////// +// void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w7 +.hidden ecp_nistz256_select_w7 +.type ecp_nistz256_select_w7,%function +.align 4 +ecp_nistz256_select_w7: + AARCH64_VALID_CALL_TARGET + + // w9 := 0; loop counter and incremented internal index + mov w9, #0 + + // [v16-v21] := 0 + movi v16.16b, #0 + movi v17.16b, #0 + movi v18.16b, #0 + movi v19.16b, #0 + +.Lselect_w7_loop: + // Loop 64 times. + + // Increment index (loop counter); tested at the end of the loop + add w9, w9, #1 + + // [v22-v25] := Load a (2*256-bit = 4*128-bit) table entry starting at x1 + // and advance x1 to point to the next entry + ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 + + // x11 := (w9 == w2)? All 1s : All 0s + cmp w9, w2 + csetm x11, eq + + // duplicate mask_64 into Mask (all 0s or all 1s) + dup v3.2d, x11 + + // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] + // i.e., values in output registers will remain the same if w9 != w2 + bit v16.16b, v22.16b, v3.16b + bit v17.16b, v23.16b, v3.16b + + bit v18.16b, v24.16b, v3.16b + bit v19.16b, v25.16b, v3.16b + + // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back + tbz w9, #6, .Lselect_w7_loop + + // Write [v16-v19] to memory at the output pointer + st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x0] + + ret +.size ecp_nistz256_select_w7,.-ecp_nistz256_select_w7 #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__ELF__)
diff --git a/gen/bcm/p256-armv8-asm-win.S b/gen/bcm/p256-armv8-asm-win.S index 71f0d94..fd80d77 100644 --- a/gen/bcm/p256-armv8-asm-win.S +++ b/gen/bcm/p256-armv8-asm-win.S
@@ -4,6 +4,27 @@ #include <openssl/asm_base.h> #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(_WIN32) +.section .rodata +.align 5 +p256_constants: +Lpoly: +.quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 +LRR: // 2^512 mod P precomputed for NIST P256 polynomial +.quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd +Lone_mont: +.quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe +Lone: +.quad 1,0,0,0 +Lord: +.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 +LordK: +.quad 0xccd1c8aaee00bc4f +.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 +.align 2 +.text + +// void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4], +// const BN_ULONG x2[4]); .globl ecp_nistz256_mul_mont .def ecp_nistz256_mul_mont @@ -16,135 +37,15 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - mul x13,x3,x2 // a[0]*b[0] - umulh x8,x3,x2 + ldr x3,[x2] // bp[0] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] - mul x14,x4,x2 // a[1]*b[0] - umulh x9,x4,x2 - - mul x15,x5,x2 // a[2]*b[0] - umulh x10,x5,x2 - - mul x16,x6,x2 // a[3]*b[0] - umulh x11,x6,x2 - ldr x2,[x1,#8] // b[1] - - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adc x17,xzr,x11 - mov x19,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - ldr x2,[x1,#8*(1+1)] // b[1+1] - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - ldr x2,[x1,#8*(2+1)] // b[2+1] - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x3,x2 // lo(a[0]*b[i]) - adcs x14,x15,x9 - mul x9,x4,x2 // lo(a[1]*b[i]) - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - mul x10,x5,x2 // lo(a[2]*b[i]) - adcs x16,x17,x11 - mul x11,x6,x2 // lo(a[3]*b[i]) - adc x17,x19,xzr - - adds x13,x13,x8 // accumulate low parts of multiplication - umulh x8,x3,x2 // hi(a[0]*b[i]) - adcs x14,x14,x9 - umulh x9,x4,x2 // hi(a[1]*b[i]) - adcs x15,x15,x10 - umulh x10,x5,x2 // hi(a[2]*b[i]) - adcs x16,x16,x11 - umulh x11,x6,x2 // hi(a[3]*b[i]) - adc x17,x17,xzr - adds x14,x14,x8 // accumulate high parts of multiplication - lsl x8,x13,#32 - adcs x15,x15,x9 - lsr x9,x13,#32 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x19,xzr,xzr - mov w7, #-1 // poly1 = 0x00000000ffffffff - // last reduction - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - adcs x16,x17,x11 - adc x17,x19,xzr - - neg x12,x7 // poly3 = 0xffffffff00000001 - adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus - sbcs x9,x14,x7 - sbcs x10,x15,xzr - sbcs x11,x16,x12 - sbcs xzr,x17,xzr // did it borrow? - - csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus - csel x14,x14,x9,lo - csel x15,x15,x10,lo - stp x13,x14,[x0] - csel x16,x16,x11,lo - stp x15,x16,[x0,#16] + bl __ecp_nistz256_mul_mont ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 @@ -152,6 +53,7 @@ ret +// void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); .globl ecp_nistz256_sqr_mont .def ecp_nistz256_sqr_mont @@ -164,6 +66,319 @@ add x29,sp,#0 stp x19,x20,[sp,#16] + ldp x4,x5,[x1] + ldp x6,x7,[x1,#16] + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sqr_mont + + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_div_by_2 + +.def ecp_nistz256_div_by_2 + .type 32 +.endef +.align 4 +ecp_nistz256_div_by_2: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_div_by_2 + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_mul_by_2 + +.def ecp_nistz256_mul_by_2 + .type 32 +.endef +.align 4 +ecp_nistz256_mul_by_2: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + + bl __ecp_nistz256_add_to // ret = a+a // 2*a + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_mul_by_3 + +.def ecp_nistz256_mul_by_3 + .type 32 +.endef +.align 4 +ecp_nistz256_mul_by_3: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + mov x4,x14 + mov x5,x15 + mov x6,x16 + mov x7,x17 + + bl __ecp_nistz256_add_to // ret = a+a // 2*a + + mov x8,x4 + mov x9,x5 + mov x10,x6 + mov x11,x7 + + bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4], +// const BN_ULONG x2[4]); +.globl ecp_nistz256_sub + +.def ecp_nistz256_sub + .type 32 +.endef +.align 4 +ecp_nistz256_sub: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + ldp x14,x15,[x1] + ldp x16,x17,[x1,#16] + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sub_from + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]); +.globl ecp_nistz256_neg + +.def ecp_nistz256_neg + .type 32 +.endef +.align 4 +ecp_nistz256_neg: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + + mov x2,x1 + mov x14,xzr // a = 0 + mov x15,xzr + mov x16,xzr + mov x17,xzr + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + bl __ecp_nistz256_sub_from + + ldp x29,x30,[sp],#16 + AARCH64_VALIDATE_LINK_REGISTER + ret + + +// note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded +// to x4-x7 and b[0] - to x3 +.def __ecp_nistz256_mul_mont + .type 32 +.endef +.align 4 +__ecp_nistz256_mul_mont: + mul x14,x4,x3 // a[0]*b[0] + umulh x8,x4,x3 + + mul x15,x5,x3 // a[1]*b[0] + umulh x9,x5,x3 + + mul x16,x6,x3 // a[2]*b[0] + umulh x10,x6,x3 + + mul x17,x7,x3 // a[3]*b[0] + umulh x11,x7,x3 + ldr x3,[x2,#8] // b[1] + + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adc x19,xzr,x11 + mov x20,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + ldr x3,[x2,#8*(1+1)] // b[1+1] + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + ldr x3,[x2,#8*(2+1)] // b[2+1] + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x4,x3 // lo(a[0]*b[i]) + adcs x15,x16,x9 + mul x9,x5,x3 // lo(a[1]*b[i]) + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + mul x10,x6,x3 // lo(a[2]*b[i]) + adcs x17,x19,x11 + mul x11,x7,x3 // lo(a[3]*b[i]) + adc x19,x20,xzr + + adds x14,x14,x8 // accumulate low parts of multiplication + umulh x8,x4,x3 // hi(a[0]*b[i]) + adcs x15,x15,x9 + umulh x9,x5,x3 // hi(a[1]*b[i]) + adcs x16,x16,x10 + umulh x10,x6,x3 // hi(a[2]*b[i]) + adcs x17,x17,x11 + umulh x11,x7,x3 // hi(a[3]*b[i]) + adc x19,x19,xzr + adds x15,x15,x8 // accumulate high parts of multiplication + lsl x8,x14,#32 + adcs x16,x16,x9 + lsr x9,x14,#32 + adcs x17,x17,x10 + adcs x19,x19,x11 + adc x20,xzr,xzr + // last reduction + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + adcs x17,x19,x11 + adc x19,x20,xzr + + adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x19,xzr // did it borrow? + + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] + + ret + + +// note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded +// to x4-x7 +.def __ecp_nistz256_sqr_mont + .type 32 +.endef +.align 4 +__ecp_nistz256_sqr_mont: // | | | | | |a1*a0| | // | | | | |a2*a0| | | // | |a3*a2|a3*a0| | | | @@ -178,126 +393,848 @@ // multiplication result, which can't overflow, because it // can never be all ones. - mul x14,x4,x3 // a[1]*a[0] - umulh x9,x4,x3 - mul x15,x5,x3 // a[2]*a[0] - umulh x10,x5,x3 - mul x16,x6,x3 // a[3]*a[0] - umulh x17,x6,x3 - - adds x15,x15,x9 // accumulate high parts of multiplication - mul x8,x5,x4 // a[2]*a[1] + mul x15,x5,x4 // a[1]*a[0] umulh x9,x5,x4 - adcs x16,x16,x10 - mul x10,x6,x4 // a[3]*a[1] - umulh x11,x6,x4 - adc x17,x17,xzr // can't overflow + mul x16,x6,x4 // a[2]*a[0] + umulh x10,x6,x4 + mul x17,x7,x4 // a[3]*a[0] + umulh x19,x7,x4 - mul x19,x6,x5 // a[3]*a[2] - umulh x20,x6,x5 + adds x16,x16,x9 // accumulate high parts of multiplication + mul x8,x6,x5 // a[2]*a[1] + umulh x9,x6,x5 + adcs x17,x17,x10 + mul x10,x7,x5 // a[3]*a[1] + umulh x11,x7,x5 + adc x19,x19,xzr // can't overflow + + mul x20,x7,x6 // a[3]*a[2] + umulh x1,x7,x6 adds x9,x9,x10 // accumulate high parts of multiplication - mul x13,x3,x3 // a[0]*a[0] + mul x14,x4,x4 // a[0]*a[0] adc x10,x11,xzr // can't overflow - adds x16,x16,x8 // accumulate low parts of multiplication - umulh x3,x3,x3 - adcs x17,x17,x9 - mul x9,x4,x4 // a[1]*a[1] - adcs x19,x19,x10 + adds x17,x17,x8 // accumulate low parts of multiplication umulh x4,x4,x4 - adc x20,x20,xzr // can't overflow - - adds x14,x14,x14 // acc[1-6]*=2 - mul x10,x5,x5 // a[2]*a[2] - adcs x15,x15,x15 + adcs x19,x19,x9 + mul x9,x5,x5 // a[1]*a[1] + adcs x20,x20,x10 umulh x5,x5,x5 + adc x1,x1,xzr // can't overflow + + adds x15,x15,x15 // acc[1-6]*=2 + mul x10,x6,x6 // a[2]*a[2] adcs x16,x16,x16 - mul x11,x6,x6 // a[3]*a[3] - adcs x17,x17,x17 umulh x6,x6,x6 + adcs x17,x17,x17 + mul x11,x7,x7 // a[3]*a[3] adcs x19,x19,x19 + umulh x7,x7,x7 adcs x20,x20,x20 + adcs x1,x1,x1 adc x2,xzr,xzr - adds x14,x14,x3 // +a[i]*a[i] - adcs x15,x15,x9 - adcs x16,x16,x4 - adcs x17,x17,x10 - adcs x19,x19,x5 - lsl x8,x13,#32 - adcs x20,x20,x11 - lsr x9,x13,#32 - adc x2,x2,x6 - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - lsl x8,x13,#32 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - lsr x9,x13,#32 - adc x16,x11,xzr // can't overflow - subs x10,x13,x8 // "*0xffff0001" - sbc x11,x13,x9 - adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] - adcs x14,x15,x9 - adcs x15,x16,x10 // +=acc[0]*0xffff0001 - adc x16,x11,xzr // can't overflow + adds x15,x15,x4 // +a[i]*a[i] + adcs x16,x16,x9 + adcs x17,x17,x5 + adcs x19,x19,x10 + adcs x20,x20,x6 + lsl x8,x14,#32 + adcs x1,x1,x11 + lsr x9,x14,#32 + adc x2,x2,x7 + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + lsl x8,x14,#32 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + lsr x9,x14,#32 + adc x17,x11,xzr // can't overflow + subs x10,x14,x8 // "*0xffff0001" + sbc x11,x14,x9 + adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] + adcs x15,x16,x9 + adcs x16,x17,x10 // +=acc[0]*0xffff0001 + adc x17,x11,xzr // can't overflow - mov w7, #-1 // poly1 = 0x00000000ffffffff - adds x13,x13,x17 // accumulate upper half - adcs x14,x14,x19 + adds x14,x14,x19 // accumulate upper half adcs x15,x15,x20 - adcs x16,x16,x2 - adc x17,xzr,xzr + adcs x16,x16,x1 + adcs x17,x17,x2 + adc x19,xzr,xzr - neg x12,x7 // poly3 = 0xffffffff00000001 - adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus - sbcs x9,x14,x7 - sbcs x10,x15,xzr - sbcs x11,x16,x12 - sbcs xzr,x17,xzr // did it borrow? + adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x19,xzr // did it borrow? - csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus - csel x14,x14,x9,lo - csel x15,x15,x10,lo - stp x13,x14,[x0] - csel x16,x16,x11,lo - stp x15,x16,[x0,#16] + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 + ret + + +// Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to +// x4-x7 and x8-x11. This is done because it's used in multiple +// contexts, e.g. in multiplication by 2 and 3... +.def __ecp_nistz256_add_to + .type 32 +.endef +.align 4 +__ecp_nistz256_add_to: + adds x14,x14,x8 // ret = a+b + adcs x15,x15,x9 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x1,xzr,xzr // zap x1 + + adds x8,x14,#1 // subs x8,x4,#-1 // tmp = ret-modulus + sbcs x9,x15,x12 + sbcs x10,x16,xzr + sbcs x11,x17,x13 + sbcs xzr,x1,xzr // did subtraction borrow? + + csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus + csel x15,x15,x9,lo + csel x16,x16,x10,lo + stp x14,x15,[x0] + csel x17,x17,x11,lo + stp x16,x17,[x0,#16] + + ret + + +.def __ecp_nistz256_sub_from + .type 32 +.endef +.align 4 +__ecp_nistz256_sub_from: + ldp x8,x9,[x2] + ldp x10,x11,[x2,#16] + subs x14,x14,x8 // ret = a-b + sbcs x15,x15,x9 + sbcs x16,x16,x10 + sbcs x17,x17,x11 + sbc x1,xzr,xzr // zap x1 + + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adc x11,x17,x13 + cmp x1,xzr // did subtraction borrow? + + csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret + csel x15,x15,x9,eq + csel x16,x16,x10,eq + stp x14,x15,[x0] + csel x17,x17,x11,eq + stp x16,x17,[x0,#16] + + ret + + +.def __ecp_nistz256_sub_morf + .type 32 +.endef +.align 4 +__ecp_nistz256_sub_morf: + ldp x8,x9,[x2] + ldp x10,x11,[x2,#16] + subs x14,x8,x14 // ret = b-a + sbcs x15,x9,x15 + sbcs x16,x10,x16 + sbcs x17,x11,x17 + sbc x1,xzr,xzr // zap x1 + + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adc x11,x17,x13 + cmp x1,xzr // did subtraction borrow? + + csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret + csel x15,x15,x9,eq + csel x16,x16,x10,eq + stp x14,x15,[x0] + csel x17,x17,x11,eq + stp x16,x17,[x0,#16] + + ret + + +.def __ecp_nistz256_div_by_2 + .type 32 +.endef +.align 4 +__ecp_nistz256_div_by_2: + subs x8,x14,#1 // adds x8,x4,#-1 // tmp = a+modulus + adcs x9,x15,x12 + adcs x10,x16,xzr + adcs x11,x17,x13 + adc x1,xzr,xzr // zap x1 + tst x14,#1 // is a even? + + csel x14,x14,x8,eq // ret = even ? a : a+modulus + csel x15,x15,x9,eq + csel x16,x16,x10,eq + csel x17,x17,x11,eq + csel x1,xzr,x1,eq + + lsr x14,x14,#1 // ret >>= 1 + orr x14,x14,x15,lsl#63 + lsr x15,x15,#1 + orr x15,x15,x16,lsl#63 + lsr x16,x16,#1 + orr x16,x16,x17,lsl#63 + lsr x17,x17,#1 + stp x14,x15,[x0] + orr x17,x17,x1,lsl#63 + stp x16,x17,[x0,#16] + + ret + +.globl ecp_nistz256_point_double + +.def ecp_nistz256_point_double + .type 32 +.endef +.align 5 +ecp_nistz256_point_double: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + sub sp,sp,#32*4 + +Ldouble_shortcut: + ldp x14,x15,[x1,#32] + mov x21,x0 + ldp x16,x17,[x1,#48] + mov x22,x1 + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + mov x8,x14 + ldr x13,[x13,#24] + mov x9,x15 + ldp x4,x5,[x22,#64] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[x22,#64+16] + add x0,sp,#0 + bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y); + + add x0,sp,#64 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z); + + ldp x8,x9,[x22] + ldp x10,x11,[x22,#16] + mov x4,x14 // put Zsqr aside for p256_sub + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x0,sp,#32 + bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x); + + add x2,x22,#0 + mov x14,x4 // restore Zsqr + mov x15,x5 + ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont + mov x16,x6 + mov x17,x7 + ldp x6,x7,[sp,#0+16] + add x0,sp,#64 + bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr); + + add x0,sp,#0 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S); + + ldr x3,[x22,#32] + ldp x4,x5,[x22,#64] + ldp x6,x7,[x22,#64+16] + add x2,x22,#32 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y); + + mov x8,x14 + mov x9,x15 + ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[sp,#0+16] + add x0,x21,#64 + bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0); + + add x0,sp,#96 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S); + + ldr x3,[sp,#64] // forward load for p256_mul_mont + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x0,x21,#32 + bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0); + + add x2,sp,#64 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr); + + mov x8,x14 // duplicate M + mov x9,x15 + mov x10,x16 + mov x11,x17 + mov x4,x14 // put M aside + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x0,sp,#32 + bl __ecp_nistz256_add_to + mov x8,x4 // restore M + mov x9,x5 + ldr x3,[x22] // forward load for p256_mul_mont + mov x10,x6 + ldp x4,x5,[sp,#0] + mov x11,x7 + ldp x6,x7,[sp,#0+16] + bl __ecp_nistz256_add_to // p256_mul_by_3(M, M); + + add x2,x22,#0 + add x0,sp,#0 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x); + + mov x8,x14 + mov x9,x15 + ldp x4,x5,[sp,#32] // forward load for p256_sqr_mont + mov x10,x16 + mov x11,x17 + ldp x6,x7,[sp,#32+16] + add x0,sp,#96 + bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S); + + add x0,x21,#0 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M); + + add x2,sp,#96 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0); + + add x2,sp,#0 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x); + + ldr x3,[sp,#32] + mov x4,x14 // copy S + mov x5,x15 + mov x6,x16 + mov x7,x17 + add x2,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M); + + add x2,x21,#32 + add x0,x21,#32 + bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y); + + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x29,x30,[sp],#96 AARCH64_VALIDATE_LINK_REGISTER ret -.section .rodata +.globl ecp_nistz256_point_add +.def ecp_nistz256_point_add + .type 32 +.endef .align 5 -p256_constants: -Lord: -.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 -LordK: -.quad 0xccd1c8aaee00bc4f -.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 -.align 2 -.text +ecp_nistz256_point_add: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-96]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + stp x27,x28,[sp,#80] + sub sp,sp,#32*12 + ldp x4,x5,[x2,#64] // in2_z + ldp x6,x7,[x2,#64+16] + mov x21,x0 + mov x22,x1 + mov x23,x2 + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + orr x8,x4,x5 + orr x10,x6,x7 + orr x25,x8,x10 + cmp x25,#0 + csetm x25,ne // ~in2infty + add x0,sp,#192 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z); + + ldp x4,x5,[x22,#64] // in1_z + ldp x6,x7,[x22,#64+16] + orr x8,x4,x5 + orr x10,x6,x7 + orr x24,x8,x10 + cmp x24,#0 + csetm x24,ne // ~in1infty + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); + + ldr x3,[x23,#64] + ldp x4,x5,[sp,#192] + ldp x6,x7,[sp,#192+16] + add x2,x23,#64 + add x0,sp,#320 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,x22,#64 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); + + ldr x3,[x22,#32] + ldp x4,x5,[sp,#320] + ldp x6,x7,[sp,#320+16] + add x2,x22,#32 + add x0,sp,#320 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y); + + ldr x3,[x23,#32] + ldp x4,x5,[sp,#352] + ldp x6,x7,[sp,#352+16] + add x2,x23,#32 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); + + add x2,sp,#320 + ldr x3,[sp,#192] // forward load for p256_mul_mont + ldp x4,x5,[x22] + ldp x6,x7,[x22,#16] + add x0,sp,#160 + bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1); + + orr x14,x14,x15 // see if result is zero + orr x16,x16,x17 + orr x26,x14,x16 // ~is_equal(S1,S2) + + add x2,sp,#192 + add x0,sp,#256 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr); + + ldr x3,[sp,#128] + ldp x4,x5,[x23] + ldp x6,x7,[x23,#16] + add x2,sp,#128 + add x0,sp,#288 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr); + + add x2,sp,#256 + ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont + ldp x6,x7,[sp,#160+16] + add x0,sp,#96 + bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1); + + orr x14,x14,x15 // see if result is zero + orr x16,x16,x17 + orr x14,x14,x16 // ~is_equal(U1,U2) + + mvn x27,x24 // -1/0 -> 0/-1 + mvn x28,x25 // -1/0 -> 0/-1 + orr x14,x14,x27 + orr x14,x14,x28 + orr x14,x14,x26 + cbnz x14,Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2)) + +Ladd_double: + mov x1,x22 + mov x0,x21 + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames + b Ldouble_shortcut + +.align 4 +Ladd_proceed: + add x0,sp,#192 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#96] + ldp x6,x7,[sp,#96+16] + add x2,x22,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); + + ldp x4,x5,[sp,#96] + ldp x6,x7,[sp,#96+16] + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); + + ldr x3,[x23,#64] + ldp x4,x5,[sp,#64] + ldp x6,x7,[sp,#64+16] + add x2,x23,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z); + + ldr x3,[sp,#96] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,sp,#96 + add x0,sp,#224 + bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); + + ldr x3,[sp,#128] + ldp x4,x5,[sp,#256] + ldp x6,x7,[sp,#256+16] + add x2,sp,#128 + add x0,sp,#288 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr); + + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + add x0,sp,#128 + bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); + + add x2,sp,#192 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); + + add x2,sp,#224 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); + + add x2,sp,#288 + ldr x3,[sp,#224] // forward load for p256_mul_mont + ldp x4,x5,[sp,#320] + ldp x6,x7,[sp,#320+16] + add x0,sp,#32 + bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); + + add x2,sp,#224 + add x0,sp,#352 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub); + + ldr x3,[sp,#160] + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x2,sp,#160 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); + + add x2,sp,#352 + bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); + + ldp x4,x5,[sp,#0] // res + ldp x6,x7,[sp,#0+16] + ldp x8,x9,[x23] // in2 + ldp x10,x11,[x23,#16] + ldp x14,x15,[x22,#0] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#0+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+0+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+0+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#0+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#0+48] + stp x14,x15,[x21,#0] + stp x16,x17,[x21,#0+16] + ldp x14,x15,[x22,#32] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#32+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+32+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+32+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#32+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#32+48] + stp x14,x15,[x21,#32] + stp x16,x17,[x21,#32+16] + ldp x14,x15,[x22,#64] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#64+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + csel x14,x8,x14,ne + csel x15,x9,x15,ne + csel x16,x10,x16,ne + csel x17,x11,x17,ne + stp x14,x15,[x21,#64] + stp x16,x17,[x21,#64+16] + +Ladd_done: + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x27,x28,[x29,#80] + ldp x29,x30,[sp],#96 + AARCH64_VALIDATE_LINK_REGISTER + ret + +.globl ecp_nistz256_point_add_affine + +.def ecp_nistz256_point_add_affine + .type 32 +.endef +.align 5 +ecp_nistz256_point_add_affine: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-80]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + stp x21,x22,[sp,#32] + stp x23,x24,[sp,#48] + stp x25,x26,[sp,#64] + sub sp,sp,#32*10 + + mov x21,x0 + mov x22,x1 + mov x23,x2 + adrp x13,Lpoly + add x13,x13,:lo12:Lpoly + ldr x12,[x13,#8] + ldr x13,[x13,#24] + + ldp x4,x5,[x1,#64] // in1_z + ldp x6,x7,[x1,#64+16] + orr x8,x4,x5 + orr x10,x6,x7 + orr x24,x8,x10 + cmp x24,#0 + csetm x24,ne // ~in1infty + + ldp x14,x15,[x2] // in2_x + ldp x16,x17,[x2,#16] + ldp x8,x9,[x2,#32] // in2_y + ldp x10,x11,[x2,#48] + orr x14,x14,x15 + orr x16,x16,x17 + orr x8,x8,x9 + orr x10,x10,x11 + orr x14,x14,x16 + orr x8,x8,x10 + orr x25,x14,x8 + cmp x25,#0 + csetm x25,ne // ~in2infty + + add x0,sp,#128 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); + + mov x4,x14 + mov x5,x15 + mov x6,x16 + mov x7,x17 + ldr x3,[x23] + add x2,x23,#0 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x); + + add x2,x22,#0 + ldr x3,[x22,#64] // forward load for p256_mul_mont + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x0,sp,#160 + bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x); + + add x2,x22,#64 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); + + ldr x3,[x22,#64] + ldp x4,x5,[sp,#160] + ldp x6,x7,[sp,#160+16] + add x2,x22,#64 + add x0,sp,#64 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); + + ldr x3,[x23,#32] + ldp x4,x5,[sp,#128] + ldp x6,x7,[sp,#128+16] + add x2,x23,#32 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); + + add x2,x22,#32 + ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont + ldp x6,x7,[sp,#160+16] + add x0,sp,#192 + bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y); + + add x0,sp,#224 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); + + ldp x4,x5,[sp,#192] + ldp x6,x7,[sp,#192+16] + add x0,sp,#288 + bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); + + ldr x3,[sp,#160] + ldp x4,x5,[sp,#224] + ldp x6,x7,[sp,#224+16] + add x2,sp,#160 + add x0,sp,#256 + bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); + + ldr x3,[x22] + ldp x4,x5,[sp,#224] + ldp x6,x7,[sp,#224+16] + add x2,x22,#0 + add x0,sp,#96 + bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr); + + mov x8,x14 + mov x9,x15 + mov x10,x16 + mov x11,x17 + add x0,sp,#224 + bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); + + add x2,sp,#288 + add x0,sp,#0 + bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); + + add x2,sp,#256 + bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); + + add x2,sp,#96 + ldr x3,[x22,#32] // forward load for p256_mul_mont + ldp x4,x5,[sp,#256] + ldp x6,x7,[sp,#256+16] + add x0,sp,#32 + bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); + + add x2,x22,#32 + add x0,sp,#128 + bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub); + + ldr x3,[sp,#192] + ldp x4,x5,[sp,#32] + ldp x6,x7,[sp,#32+16] + add x2,sp,#192 + add x0,sp,#32 + bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); + + add x2,sp,#128 + bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); + + ldp x4,x5,[sp,#0] // res + ldp x6,x7,[sp,#0+16] + ldp x8,x9,[x23] // in2 + ldp x10,x11,[x23,#16] + ldp x14,x15,[x22,#0] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#0+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+0+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+0+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#0+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#0+48] + stp x14,x15,[x21,#0] + stp x16,x17,[x21,#0+16] + adrp x23,Lone_mont-64 + add x23,x23,:lo12:Lone_mont-64 + ldp x14,x15,[x22,#32] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#32+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + ldp x4,x5,[sp,#0+32+32] // res + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + ldp x6,x7,[sp,#0+32+48] + csel x14,x8,x14,ne + csel x15,x9,x15,ne + ldp x8,x9,[x23,#32+32] // in2 + csel x16,x10,x16,ne + csel x17,x11,x17,ne + ldp x10,x11,[x23,#32+48] + stp x14,x15,[x21,#32] + stp x16,x17,[x21,#32+16] + ldp x14,x15,[x22,#64] // in1 + cmp x24,#0 // ~, remember? + ldp x16,x17,[x22,#64+16] + csel x8,x4,x8,ne + csel x9,x5,x9,ne + csel x10,x6,x10,ne + csel x11,x7,x11,ne + cmp x25,#0 // ~, remember? + csel x14,x8,x14,ne + csel x15,x9,x15,ne + csel x16,x10,x16,ne + csel x17,x11,x17,ne + stp x14,x15,[x21,#64] + stp x16,x17,[x21,#64+16] + + add sp,x29,#0 // destroy frame + ldp x19,x20,[x29,#16] + ldp x21,x22,[x29,#32] + ldp x23,x24,[x29,#48] + ldp x25,x26,[x29,#64] + ldp x29,x30,[sp],#80 + AARCH64_VALIDATE_LINK_REGISTER + ret + +//////////////////////////////////////////////////////////////////////// // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4], // uint64_t b[4]); .globl ecp_nistz256_ord_mul_mont @@ -323,7 +1260,7 @@ ldp x12,x13,[x23,#0] ldp x21,x22,[x23,#16] - ldr x23,[x23,#32] // LordK + ldr x23,[x23,#32] mul x14,x4,x3 // a[0]*b[0] umulh x8,x4,x3 @@ -706,4 +1643,123 @@ ldr x29,[sp],#64 ret +//////////////////////////////////////////////////////////////////////// +// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w5 + +.def ecp_nistz256_select_w5 + .type 32 +.endef +.align 4 +ecp_nistz256_select_w5: + AARCH64_VALID_CALL_TARGET + + // x10 := x0 + // w9 := 0; loop counter and incremented internal index + mov x10, x0 + mov w9, #0 + + // [v16-v21] := 0 + movi v16.16b, #0 + movi v17.16b, #0 + movi v18.16b, #0 + movi v19.16b, #0 + movi v20.16b, #0 + movi v21.16b, #0 + +Lselect_w5_loop: + // Loop 16 times. + + // Increment index (loop counter); tested at the end of the loop + add w9, w9, #1 + + // [v22-v27] := Load a (3*256-bit = 6*128-bit) table entry starting at x1 + // and advance x1 to point to the next entry + ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 + + // x11 := (w9 == w2)? All 1s : All 0s + cmp w9, w2 + csetm x11, eq + + // continue loading ... + ld1 {v26.2d, v27.2d}, [x1],#32 + + // duplicate mask_64 into Mask (all 0s or all 1s) + dup v3.2d, x11 + + // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] + // i.e., values in output registers will remain the same if w9 != w2 + bit v16.16b, v22.16b, v3.16b + bit v17.16b, v23.16b, v3.16b + + bit v18.16b, v24.16b, v3.16b + bit v19.16b, v25.16b, v3.16b + + bit v20.16b, v26.16b, v3.16b + bit v21.16b, v27.16b, v3.16b + + // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back + tbz w9, #4, Lselect_w5_loop + + // Write [v16-v21] to memory at the output pointer + st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x10],#64 + st1 {v20.2d, v21.2d}, [x10] + + ret + + + +//////////////////////////////////////////////////////////////////////// +// void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); +.globl ecp_nistz256_select_w7 + +.def ecp_nistz256_select_w7 + .type 32 +.endef +.align 4 +ecp_nistz256_select_w7: + AARCH64_VALID_CALL_TARGET + + // w9 := 0; loop counter and incremented internal index + mov w9, #0 + + // [v16-v21] := 0 + movi v16.16b, #0 + movi v17.16b, #0 + movi v18.16b, #0 + movi v19.16b, #0 + +Lselect_w7_loop: + // Loop 64 times. + + // Increment index (loop counter); tested at the end of the loop + add w9, w9, #1 + + // [v22-v25] := Load a (2*256-bit = 4*128-bit) table entry starting at x1 + // and advance x1 to point to the next entry + ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 + + // x11 := (w9 == w2)? All 1s : All 0s + cmp w9, w2 + csetm x11, eq + + // duplicate mask_64 into Mask (all 0s or all 1s) + dup v3.2d, x11 + + // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] + // i.e., values in output registers will remain the same if w9 != w2 + bit v16.16b, v22.16b, v3.16b + bit v17.16b, v23.16b, v3.16b + + bit v18.16b, v24.16b, v3.16b + bit v19.16b, v25.16b, v3.16b + + // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back + tbz w9, #6, Lselect_w7_loop + + // Write [v16-v19] to memory at the output pointer + st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x0] + + ret + #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(_WIN32)
diff --git a/gen/bcm/p256-x86_64-asm-apple.S b/gen/bcm/p256-x86_64-asm-apple.S index ab368c9..255a34d 100644 --- a/gen/bcm/p256-x86_64-asm-apple.S +++ b/gen/bcm/p256-x86_64-asm-apple.S
@@ -7,7 +7,22 @@ .text +.section __DATA,__const +.p2align 6 p256_constants: +L$poly: +.quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 + +L$One: +.long 1,1,1,1,1,1,1,1 +L$Two: +.long 2,2,2,2,2,2,2,2 +L$Three: +.long 3,3,3,3,3,3,3,3 +L$ONE_mont: +.quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe + + L$ord: .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 L$ordK: @@ -16,6 +31,64 @@ +.globl _ecp_nistz256_neg +.private_extern _ecp_nistz256_neg + +.p2align 5 +_ecp_nistz256_neg: + +_CET_ENDBR + pushq %r12 + + pushq %r13 + +L$neg_body: + + xorq %r8,%r8 + xorq %r9,%r9 + xorq %r10,%r10 + xorq %r11,%r11 + xorq %r13,%r13 + + subq 0(%rsi),%r8 + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + movq %r8,%rax + sbbq 24(%rsi),%r11 + leaq L$poly(%rip),%rsi + movq %r9,%rdx + sbbq $0,%r13 + + addq 0(%rsi),%r8 + movq %r10,%rcx + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + movq %r11,%r12 + adcq 24(%rsi),%r11 + testq %r13,%r13 + + cmovzq %rax,%r8 + cmovzq %rdx,%r9 + movq %r8,0(%rdi) + cmovzq %rcx,%r10 + movq %r9,8(%rdi) + cmovzq %r12,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 0(%rsp),%r13 + + movq 8(%rsp),%r12 + + leaq 16(%rsp),%rsp + +L$neg_epilogue: + ret + + + + + @@ -1084,4 +1157,3388 @@ ret + + + + + + +.globl _ecp_nistz256_mul_mont_nohw +.private_extern _ecp_nistz256_mul_mont_nohw + +.p2align 5 +.alt_entry _ecp_nistz256_mul_mont_nohw +_ecp_nistz256_mul_mont_nohw: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + +L$mul_body: + movq %rdx,%rbx + movq 0(%rdx),%rax + movq 0(%rsi),%r9 + movq 8(%rsi),%r10 + movq 16(%rsi),%r11 + movq 24(%rsi),%r12 + + call __ecp_nistz256_mul_montq + + movq 0(%rsp),%r15 + + movq 8(%rsp),%r14 + + movq 16(%rsp),%r13 + + movq 24(%rsp),%r12 + + movq 32(%rsp),%rbx + + movq 40(%rsp),%rbp + + leaq 48(%rsp),%rsp + +L$mul_epilogue: + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_mul_montq +__ecp_nistz256_mul_montq: + + + + movq %rax,%rbp + mulq %r9 + movq L$poly+8(%rip),%r14 + movq %rax,%r8 + movq %rbp,%rax + movq %rdx,%r9 + + mulq %r10 + movq L$poly+24(%rip),%r15 + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq %r11 + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq %r12 + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + xorq %r13,%r13 + movq %rdx,%r12 + + + + + + + + + + + movq %r8,%rbp + shlq $32,%r8 + mulq %r15 + shrq $32,%rbp + addq %r8,%r9 + adcq %rbp,%r10 + adcq %rax,%r11 + movq 8(%rbx),%rax + adcq %rdx,%r12 + adcq $0,%r13 + xorq %r8,%r8 + + + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rcx,%r10 + adcq $0,%rdx + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 16(%rsi) + addq %rcx,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 24(%rsi) + addq %rcx,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %r9,%rax + adcq %rdx,%r13 + adcq $0,%r8 + + + + movq %r9,%rbp + shlq $32,%r9 + mulq %r15 + shrq $32,%rbp + addq %r9,%r10 + adcq %rbp,%r11 + adcq %rax,%r12 + movq 16(%rbx),%rax + adcq %rdx,%r13 + adcq $0,%r8 + xorq %r9,%r9 + + + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rcx,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 16(%rsi) + addq %rcx,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 24(%rsi) + addq %rcx,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %r10,%rax + adcq %rdx,%r8 + adcq $0,%r9 + + + + movq %r10,%rbp + shlq $32,%r10 + mulq %r15 + shrq $32,%rbp + addq %r10,%r11 + adcq %rbp,%r12 + adcq %rax,%r13 + movq 24(%rbx),%rax + adcq %rdx,%r8 + adcq $0,%r9 + xorq %r10,%r10 + + + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rcx,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 16(%rsi) + addq %rcx,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 24(%rsi) + addq %rcx,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r11,%rax + adcq %rdx,%r9 + adcq $0,%r10 + + + + movq %r11,%rbp + shlq $32,%r11 + mulq %r15 + shrq $32,%rbp + addq %r11,%r12 + adcq %rbp,%r13 + movq %r12,%rcx + adcq %rax,%r8 + adcq %rdx,%r9 + movq %r13,%rbp + adcq $0,%r10 + + + + subq $-1,%r12 + movq %r8,%rbx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%rdx + sbbq %r15,%r9 + sbbq $0,%r10 + + cmovcq %rcx,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rbx,%r8 + movq %r13,8(%rdi) + cmovcq %rdx,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret + + + + + + + + + + +.globl _ecp_nistz256_sqr_mont_nohw +.private_extern _ecp_nistz256_sqr_mont_nohw + +.p2align 5 +.alt_entry _ecp_nistz256_sqr_mont_nohw +_ecp_nistz256_sqr_mont_nohw: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + +L$sqr_body: + movq 0(%rsi),%rax + movq 8(%rsi),%r14 + movq 16(%rsi),%r15 + movq 24(%rsi),%r8 + + call __ecp_nistz256_sqr_montq + + movq 0(%rsp),%r15 + + movq 8(%rsp),%r14 + + movq 16(%rsp),%r13 + + movq 24(%rsp),%r12 + + movq 32(%rsp),%rbx + + movq 40(%rsp),%rbp + + leaq 48(%rsp),%rsp + +L$sqr_epilogue: + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_sqr_montq +__ecp_nistz256_sqr_montq: + + movq %rax,%r13 + mulq %r14 + movq %rax,%r9 + movq %r15,%rax + movq %rdx,%r10 + + mulq %r13 + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq %r13 + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r12 + + + mulq %r14 + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq %r14 + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r12 + movq %rdx,%r13 + adcq $0,%r13 + + + mulq %r15 + xorq %r15,%r15 + addq %rax,%r13 + movq 0(%rsi),%rax + movq %rdx,%r14 + adcq $0,%r14 + + addq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %r12,%r12 + adcq %r13,%r13 + adcq %r14,%r14 + adcq $0,%r15 + + mulq %rax + movq %rax,%r8 + movq 8(%rsi),%rax + movq %rdx,%rcx + + mulq %rax + addq %rcx,%r9 + adcq %rax,%r10 + movq 16(%rsi),%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq %rax + addq %rcx,%r11 + adcq %rax,%r12 + movq 24(%rsi),%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq %rax + addq %rcx,%r13 + adcq %rax,%r14 + movq %r8,%rax + adcq %rdx,%r15 + + movq L$poly+8(%rip),%rsi + movq L$poly+24(%rip),%rbp + + + + + movq %r8,%rcx + shlq $32,%r8 + mulq %rbp + shrq $32,%rcx + addq %r8,%r9 + adcq %rcx,%r10 + adcq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + + + + movq %r9,%rcx + shlq $32,%r9 + movq %rdx,%r8 + mulq %rbp + shrq $32,%rcx + addq %r9,%r10 + adcq %rcx,%r11 + adcq %rax,%r8 + movq %r10,%rax + adcq $0,%rdx + + + + movq %r10,%rcx + shlq $32,%r10 + movq %rdx,%r9 + mulq %rbp + shrq $32,%rcx + addq %r10,%r11 + adcq %rcx,%r8 + adcq %rax,%r9 + movq %r11,%rax + adcq $0,%rdx + + + + movq %r11,%rcx + shlq $32,%r11 + movq %rdx,%r10 + mulq %rbp + shrq $32,%rcx + addq %r11,%r8 + adcq %rcx,%r9 + adcq %rax,%r10 + adcq $0,%rdx + xorq %r11,%r11 + + + + addq %r8,%r12 + adcq %r9,%r13 + movq %r12,%r8 + adcq %r10,%r14 + adcq %rdx,%r15 + movq %r13,%r9 + adcq $0,%r11 + + subq $-1,%r12 + movq %r14,%r10 + sbbq %rsi,%r13 + sbbq $0,%r14 + movq %r15,%rcx + sbbq %rbp,%r15 + sbbq $0,%r11 + + cmovcq %r8,%r12 + cmovcq %r9,%r13 + movq %r12,0(%rdi) + cmovcq %r10,%r14 + movq %r13,8(%rdi) + cmovcq %rcx,%r15 + movq %r14,16(%rdi) + movq %r15,24(%rdi) + + ret + + +.globl _ecp_nistz256_mul_mont_adx +.private_extern _ecp_nistz256_mul_mont_adx + +.p2align 5 +.alt_entry _ecp_nistz256_mul_mont_adx +_ecp_nistz256_mul_mont_adx: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + +L$mulx_body: + movq %rdx,%rbx + movq 0(%rdx),%rdx + movq 0(%rsi),%r9 + movq 8(%rsi),%r10 + movq 16(%rsi),%r11 + movq 24(%rsi),%r12 + leaq -128(%rsi),%rsi + + call __ecp_nistz256_mul_montx + + movq 0(%rsp),%r15 + + movq 8(%rsp),%r14 + + movq 16(%rsp),%r13 + + movq 24(%rsp),%r12 + + movq 32(%rsp),%rbx + + movq 40(%rsp),%rbp + + leaq 48(%rsp),%rsp + +L$mulx_epilogue: + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_mul_montx +__ecp_nistz256_mul_montx: + + + + mulxq %r9,%r8,%r9 + mulxq %r10,%rcx,%r10 + movq $32,%r14 + xorq %r13,%r13 + mulxq %r11,%rbp,%r11 + movq L$poly+24(%rip),%r15 + adcq %rcx,%r9 + mulxq %r12,%rcx,%r12 + movq %r8,%rdx + adcq %rbp,%r10 + shlxq %r14,%r8,%rbp + adcq %rcx,%r11 + shrxq %r14,%r8,%rcx + adcq $0,%r12 + + + + addq %rbp,%r9 + adcq %rcx,%r10 + + mulxq %r15,%rcx,%rbp + movq 8(%rbx),%rdx + adcq %rcx,%r11 + adcq %rbp,%r12 + adcq $0,%r13 + xorq %r8,%r8 + + + + mulxq 0+128(%rsi),%rcx,%rbp + adcxq %rcx,%r9 + adoxq %rbp,%r10 + + mulxq 8+128(%rsi),%rcx,%rbp + adcxq %rcx,%r10 + adoxq %rbp,%r11 + + mulxq 16+128(%rsi),%rcx,%rbp + adcxq %rcx,%r11 + adoxq %rbp,%r12 + + mulxq 24+128(%rsi),%rcx,%rbp + movq %r9,%rdx + adcxq %rcx,%r12 + shlxq %r14,%r9,%rcx + adoxq %rbp,%r13 + shrxq %r14,%r9,%rbp + + adcxq %r8,%r13 + adoxq %r8,%r8 + adcq $0,%r8 + + + + addq %rcx,%r10 + adcq %rbp,%r11 + + mulxq %r15,%rcx,%rbp + movq 16(%rbx),%rdx + adcq %rcx,%r12 + adcq %rbp,%r13 + adcq $0,%r8 + xorq %r9,%r9 + + + + mulxq 0+128(%rsi),%rcx,%rbp + adcxq %rcx,%r10 + adoxq %rbp,%r11 + + mulxq 8+128(%rsi),%rcx,%rbp + adcxq %rcx,%r11 + adoxq %rbp,%r12 + + mulxq 16+128(%rsi),%rcx,%rbp + adcxq %rcx,%r12 + adoxq %rbp,%r13 + + mulxq 24+128(%rsi),%rcx,%rbp + movq %r10,%rdx + adcxq %rcx,%r13 + shlxq %r14,%r10,%rcx + adoxq %rbp,%r8 + shrxq %r14,%r10,%rbp + + adcxq %r9,%r8 + adoxq %r9,%r9 + adcq $0,%r9 + + + + addq %rcx,%r11 + adcq %rbp,%r12 + + mulxq %r15,%rcx,%rbp + movq 24(%rbx),%rdx + adcq %rcx,%r13 + adcq %rbp,%r8 + adcq $0,%r9 + xorq %r10,%r10 + + + + mulxq 0+128(%rsi),%rcx,%rbp + adcxq %rcx,%r11 + adoxq %rbp,%r12 + + mulxq 8+128(%rsi),%rcx,%rbp + adcxq %rcx,%r12 + adoxq %rbp,%r13 + + mulxq 16+128(%rsi),%rcx,%rbp + adcxq %rcx,%r13 + adoxq %rbp,%r8 + + mulxq 24+128(%rsi),%rcx,%rbp + movq %r11,%rdx + adcxq %rcx,%r8 + shlxq %r14,%r11,%rcx + adoxq %rbp,%r9 + shrxq %r14,%r11,%rbp + + adcxq %r10,%r9 + adoxq %r10,%r10 + adcq $0,%r10 + + + + addq %rcx,%r12 + adcq %rbp,%r13 + + mulxq %r15,%rcx,%rbp + movq %r12,%rbx + movq L$poly+8(%rip),%r14 + adcq %rcx,%r8 + movq %r13,%rdx + adcq %rbp,%r9 + adcq $0,%r10 + + + + xorl %eax,%eax + movq %r8,%rcx + sbbq $-1,%r12 + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%rbp + sbbq %r15,%r9 + sbbq $0,%r10 + + cmovcq %rbx,%r12 + cmovcq %rdx,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %rbp,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret + + + +.globl _ecp_nistz256_sqr_mont_adx +.private_extern _ecp_nistz256_sqr_mont_adx + +.p2align 5 +.alt_entry _ecp_nistz256_sqr_mont_adx +_ecp_nistz256_sqr_mont_adx: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + +L$sqrx_body: + movq 0(%rsi),%rdx + movq 8(%rsi),%r14 + movq 16(%rsi),%r15 + movq 24(%rsi),%r8 + leaq -128(%rsi),%rsi + + call __ecp_nistz256_sqr_montx + + movq 0(%rsp),%r15 + + movq 8(%rsp),%r14 + + movq 16(%rsp),%r13 + + movq 24(%rsp),%r12 + + movq 32(%rsp),%rbx + + movq 40(%rsp),%rbp + + leaq 48(%rsp),%rsp + +L$sqrx_epilogue: + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_sqr_montx +__ecp_nistz256_sqr_montx: + + mulxq %r14,%r9,%r10 + mulxq %r15,%rcx,%r11 + xorl %eax,%eax + adcq %rcx,%r10 + mulxq %r8,%rbp,%r12 + movq %r14,%rdx + adcq %rbp,%r11 + adcq $0,%r12 + xorq %r13,%r13 + + + mulxq %r15,%rcx,%rbp + adcxq %rcx,%r11 + adoxq %rbp,%r12 + + mulxq %r8,%rcx,%rbp + movq %r15,%rdx + adcxq %rcx,%r12 + adoxq %rbp,%r13 + adcq $0,%r13 + + + mulxq %r8,%rcx,%r14 + movq 0+128(%rsi),%rdx + xorq %r15,%r15 + adcxq %r9,%r9 + adoxq %rcx,%r13 + adcxq %r10,%r10 + adoxq %r15,%r14 + + mulxq %rdx,%r8,%rbp + movq 8+128(%rsi),%rdx + adcxq %r11,%r11 + adoxq %rbp,%r9 + adcxq %r12,%r12 + mulxq %rdx,%rcx,%rax + movq 16+128(%rsi),%rdx + adcxq %r13,%r13 + adoxq %rcx,%r10 + adcxq %r14,%r14 +.byte 0x67 + mulxq %rdx,%rcx,%rbp + movq 24+128(%rsi),%rdx + adoxq %rax,%r11 + adcxq %r15,%r15 + adoxq %rcx,%r12 + movq $32,%rsi + adoxq %rbp,%r13 +.byte 0x67,0x67 + mulxq %rdx,%rcx,%rax + movq L$poly+24(%rip),%rdx + adoxq %rcx,%r14 + shlxq %rsi,%r8,%rcx + adoxq %rax,%r15 + shrxq %rsi,%r8,%rax + movq %rdx,%rbp + + + addq %rcx,%r9 + adcq %rax,%r10 + + mulxq %r8,%rcx,%r8 + adcq %rcx,%r11 + shlxq %rsi,%r9,%rcx + adcq $0,%r8 + shrxq %rsi,%r9,%rax + + + addq %rcx,%r10 + adcq %rax,%r11 + + mulxq %r9,%rcx,%r9 + adcq %rcx,%r8 + shlxq %rsi,%r10,%rcx + adcq $0,%r9 + shrxq %rsi,%r10,%rax + + + addq %rcx,%r11 + adcq %rax,%r8 + + mulxq %r10,%rcx,%r10 + adcq %rcx,%r9 + shlxq %rsi,%r11,%rcx + adcq $0,%r10 + shrxq %rsi,%r11,%rax + + + addq %rcx,%r8 + adcq %rax,%r9 + + mulxq %r11,%rcx,%r11 + adcq %rcx,%r10 + adcq $0,%r11 + + xorq %rdx,%rdx + addq %r8,%r12 + movq L$poly+8(%rip),%rsi + adcq %r9,%r13 + movq %r12,%r8 + adcq %r10,%r14 + adcq %r11,%r15 + movq %r13,%r9 + adcq $0,%rdx + + subq $-1,%r12 + movq %r14,%r10 + sbbq %rsi,%r13 + sbbq $0,%r14 + movq %r15,%r11 + sbbq %rbp,%r15 + sbbq $0,%rdx + + cmovcq %r8,%r12 + cmovcq %r9,%r13 + movq %r12,0(%rdi) + cmovcq %r10,%r14 + movq %r13,8(%rdi) + cmovcq %r11,%r15 + movq %r14,16(%rdi) + movq %r15,24(%rdi) + + ret + + + + +.globl _ecp_nistz256_select_w5_nohw +.private_extern _ecp_nistz256_select_w5_nohw + +.p2align 5 +.alt_entry _ecp_nistz256_select_w5_nohw +_ecp_nistz256_select_w5_nohw: + +_CET_ENDBR + movdqa L$One(%rip),%xmm0 + movd %edx,%xmm1 + + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + pxor %xmm6,%xmm6 + pxor %xmm7,%xmm7 + + movdqa %xmm0,%xmm8 + pshufd $0,%xmm1,%xmm1 + + movq $16,%rax +L$select_loop_sse_w5: + + movdqa %xmm8,%xmm15 + paddd %xmm0,%xmm8 + pcmpeqd %xmm1,%xmm15 + + movdqa 0(%rsi),%xmm9 + movdqa 16(%rsi),%xmm10 + movdqa 32(%rsi),%xmm11 + movdqa 48(%rsi),%xmm12 + movdqa 64(%rsi),%xmm13 + movdqa 80(%rsi),%xmm14 + leaq 96(%rsi),%rsi + + pand %xmm15,%xmm9 + pand %xmm15,%xmm10 + por %xmm9,%xmm2 + pand %xmm15,%xmm11 + por %xmm10,%xmm3 + pand %xmm15,%xmm12 + por %xmm11,%xmm4 + pand %xmm15,%xmm13 + por %xmm12,%xmm5 + pand %xmm15,%xmm14 + por %xmm13,%xmm6 + por %xmm14,%xmm7 + + decq %rax + jnz L$select_loop_sse_w5 + + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + movdqu %xmm4,32(%rdi) + movdqu %xmm5,48(%rdi) + movdqu %xmm6,64(%rdi) + movdqu %xmm7,80(%rdi) + ret + +L$SEH_end_ecp_nistz256_select_w5_nohw: + + + + +.globl _ecp_nistz256_select_w7_nohw +.private_extern _ecp_nistz256_select_w7_nohw + +.p2align 5 +.alt_entry _ecp_nistz256_select_w7_nohw +_ecp_nistz256_select_w7_nohw: + +_CET_ENDBR + movdqa L$One(%rip),%xmm8 + movd %edx,%xmm1 + + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + + movdqa %xmm8,%xmm0 + pshufd $0,%xmm1,%xmm1 + movq $64,%rax + +L$select_loop_sse_w7: + movdqa %xmm8,%xmm15 + paddd %xmm0,%xmm8 + movdqa 0(%rsi),%xmm9 + movdqa 16(%rsi),%xmm10 + pcmpeqd %xmm1,%xmm15 + movdqa 32(%rsi),%xmm11 + movdqa 48(%rsi),%xmm12 + leaq 64(%rsi),%rsi + + pand %xmm15,%xmm9 + pand %xmm15,%xmm10 + por %xmm9,%xmm2 + pand %xmm15,%xmm11 + por %xmm10,%xmm3 + pand %xmm15,%xmm12 + por %xmm11,%xmm4 + prefetcht0 255(%rsi) + por %xmm12,%xmm5 + + decq %rax + jnz L$select_loop_sse_w7 + + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + movdqu %xmm4,32(%rdi) + movdqu %xmm5,48(%rdi) + ret + +L$SEH_end_ecp_nistz256_select_w7_nohw: + + + +.globl _ecp_nistz256_select_w5_avx2 +.private_extern _ecp_nistz256_select_w5_avx2 + +.p2align 5 +.alt_entry _ecp_nistz256_select_w5_avx2 +_ecp_nistz256_select_w5_avx2: + +_CET_ENDBR + vzeroupper + vmovdqa L$Two(%rip),%ymm0 + + vpxor %ymm2,%ymm2,%ymm2 + vpxor %ymm3,%ymm3,%ymm3 + vpxor %ymm4,%ymm4,%ymm4 + + vmovdqa L$One(%rip),%ymm5 + vmovdqa L$Two(%rip),%ymm10 + + vmovd %edx,%xmm1 + vpermd %ymm1,%ymm2,%ymm1 + + movq $8,%rax +L$select_loop_avx2_w5: + + vmovdqa 0(%rsi),%ymm6 + vmovdqa 32(%rsi),%ymm7 + vmovdqa 64(%rsi),%ymm8 + + vmovdqa 96(%rsi),%ymm11 + vmovdqa 128(%rsi),%ymm12 + vmovdqa 160(%rsi),%ymm13 + + vpcmpeqd %ymm1,%ymm5,%ymm9 + vpcmpeqd %ymm1,%ymm10,%ymm14 + + vpaddd %ymm0,%ymm5,%ymm5 + vpaddd %ymm0,%ymm10,%ymm10 + leaq 192(%rsi),%rsi + + vpand %ymm9,%ymm6,%ymm6 + vpand %ymm9,%ymm7,%ymm7 + vpand %ymm9,%ymm8,%ymm8 + vpand %ymm14,%ymm11,%ymm11 + vpand %ymm14,%ymm12,%ymm12 + vpand %ymm14,%ymm13,%ymm13 + + vpxor %ymm6,%ymm2,%ymm2 + vpxor %ymm7,%ymm3,%ymm3 + vpxor %ymm8,%ymm4,%ymm4 + vpxor %ymm11,%ymm2,%ymm2 + vpxor %ymm12,%ymm3,%ymm3 + vpxor %ymm13,%ymm4,%ymm4 + + decq %rax + jnz L$select_loop_avx2_w5 + + vmovdqu %ymm2,0(%rdi) + vmovdqu %ymm3,32(%rdi) + vmovdqu %ymm4,64(%rdi) + vzeroupper + ret + +L$SEH_end_ecp_nistz256_select_w5_avx2: + + + + +.globl _ecp_nistz256_select_w7_avx2 +.private_extern _ecp_nistz256_select_w7_avx2 + +.p2align 5 +.alt_entry _ecp_nistz256_select_w7_avx2 +_ecp_nistz256_select_w7_avx2: + +_CET_ENDBR + vzeroupper + vmovdqa L$Three(%rip),%ymm0 + + vpxor %ymm2,%ymm2,%ymm2 + vpxor %ymm3,%ymm3,%ymm3 + + vmovdqa L$One(%rip),%ymm4 + vmovdqa L$Two(%rip),%ymm8 + vmovdqa L$Three(%rip),%ymm12 + + vmovd %edx,%xmm1 + vpermd %ymm1,%ymm2,%ymm1 + + + movq $21,%rax +L$select_loop_avx2_w7: + + vmovdqa 0(%rsi),%ymm5 + vmovdqa 32(%rsi),%ymm6 + + vmovdqa 64(%rsi),%ymm9 + vmovdqa 96(%rsi),%ymm10 + + vmovdqa 128(%rsi),%ymm13 + vmovdqa 160(%rsi),%ymm14 + + vpcmpeqd %ymm1,%ymm4,%ymm7 + vpcmpeqd %ymm1,%ymm8,%ymm11 + vpcmpeqd %ymm1,%ymm12,%ymm15 + + vpaddd %ymm0,%ymm4,%ymm4 + vpaddd %ymm0,%ymm8,%ymm8 + vpaddd %ymm0,%ymm12,%ymm12 + leaq 192(%rsi),%rsi + + vpand %ymm7,%ymm5,%ymm5 + vpand %ymm7,%ymm6,%ymm6 + vpand %ymm11,%ymm9,%ymm9 + vpand %ymm11,%ymm10,%ymm10 + vpand %ymm15,%ymm13,%ymm13 + vpand %ymm15,%ymm14,%ymm14 + + vpxor %ymm5,%ymm2,%ymm2 + vpxor %ymm6,%ymm3,%ymm3 + vpxor %ymm9,%ymm2,%ymm2 + vpxor %ymm10,%ymm3,%ymm3 + vpxor %ymm13,%ymm2,%ymm2 + vpxor %ymm14,%ymm3,%ymm3 + + decq %rax + jnz L$select_loop_avx2_w7 + + + vmovdqa 0(%rsi),%ymm5 + vmovdqa 32(%rsi),%ymm6 + + vpcmpeqd %ymm1,%ymm4,%ymm7 + + vpand %ymm7,%ymm5,%ymm5 + vpand %ymm7,%ymm6,%ymm6 + + vpxor %ymm5,%ymm2,%ymm2 + vpxor %ymm6,%ymm3,%ymm3 + + vmovdqu %ymm2,0(%rdi) + vmovdqu %ymm3,32(%rdi) + vzeroupper + ret + +L$SEH_end_ecp_nistz256_select_w7_avx2: + + +.p2align 5 +.alt_entry __ecp_nistz256_add_toq +__ecp_nistz256_add_toq: + + xorq %r11,%r11 + addq 0(%rbx),%r12 + adcq 8(%rbx),%r13 + movq %r12,%rax + adcq 16(%rbx),%r8 + adcq 24(%rbx),%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_sub_fromq +__ecp_nistz256_sub_fromq: + + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r13 + movq %r12,%rax + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + movq %r13,%rbp + sbbq %r11,%r11 + + addq $-1,%r12 + movq %r8,%rcx + adcq %r14,%r13 + adcq $0,%r8 + movq %r9,%r10 + adcq %r15,%r9 + testq %r11,%r11 + + cmovzq %rax,%r12 + cmovzq %rbp,%r13 + movq %r12,0(%rdi) + cmovzq %rcx,%r8 + movq %r13,8(%rdi) + cmovzq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_subq +__ecp_nistz256_subq: + + subq %r12,%rax + sbbq %r13,%rbp + movq %rax,%r12 + sbbq %r8,%rcx + sbbq %r9,%r10 + movq %rbp,%r13 + sbbq %r11,%r11 + + addq $-1,%rax + movq %rcx,%r8 + adcq %r14,%rbp + adcq $0,%rcx + movq %r10,%r9 + adcq %r15,%r10 + testq %r11,%r11 + + cmovnzq %rax,%r12 + cmovnzq %rbp,%r13 + cmovnzq %rcx,%r8 + cmovnzq %r10,%r9 + + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_mul_by_2q +__ecp_nistz256_mul_by_2q: + + xorq %r11,%r11 + addq %r12,%r12 + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret + + +.globl _ecp_nistz256_point_double_nohw +.private_extern _ecp_nistz256_point_double_nohw + +.p2align 5 +.alt_entry _ecp_nistz256_point_double_nohw +_ecp_nistz256_point_double_nohw: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $160+8,%rsp + +L$point_doubleq_body: + +L$point_double_shortcutq: + movdqu 0(%rsi),%xmm0 + movq %rsi,%rbx + movdqu 16(%rsi),%xmm1 + movq 32+0(%rsi),%r12 + movq 32+8(%rsi),%r13 + movq 32+16(%rsi),%r8 + movq 32+24(%rsi),%r9 + movq L$poly+8(%rip),%r14 + movq L$poly+24(%rip),%r15 + movdqa %xmm0,96(%rsp) + movdqa %xmm1,96+16(%rsp) + leaq 32(%rdi),%r10 + leaq 64(%rdi),%r11 + movq %rdi,%xmm0 + movq %r10,%xmm1 + movq %r11,%xmm2 + + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_by_2q + + movq 64+0(%rsi),%rax + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + leaq 64-0(%rsi),%rsi + leaq 64(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 0+0(%rsp),%rax + movq 8+0(%rsp),%r14 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + leaq 0(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 32(%rbx),%rax + movq 64+0(%rbx),%r9 + movq 64+8(%rbx),%r10 + movq 64+16(%rbx),%r11 + movq 64+24(%rbx),%r12 + leaq 64-0(%rbx),%rsi + leaq 32(%rbx),%rbx + movq %xmm2,%rdi + call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_by_2q + + movq 96+0(%rsp),%r12 + movq 96+8(%rsp),%r13 + leaq 64(%rsp),%rbx + movq 96+16(%rsp),%r8 + movq 96+24(%rsp),%r9 + leaq 32(%rsp),%rdi + call __ecp_nistz256_add_toq + + movq 96+0(%rsp),%r12 + movq 96+8(%rsp),%r13 + leaq 64(%rsp),%rbx + movq 96+16(%rsp),%r8 + movq 96+24(%rsp),%r9 + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 0+0(%rsp),%rax + movq 8+0(%rsp),%r14 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + movq %xmm1,%rdi + call __ecp_nistz256_sqr_montq + xorq %r9,%r9 + movq %r12,%rax + addq $-1,%r12 + movq %r13,%r10 + adcq %rsi,%r13 + movq %r14,%rcx + adcq $0,%r14 + movq %r15,%r8 + adcq %rbp,%r15 + adcq $0,%r9 + xorq %rsi,%rsi + testq $1,%rax + + cmovzq %rax,%r12 + cmovzq %r10,%r13 + cmovzq %rcx,%r14 + cmovzq %r8,%r15 + cmovzq %rsi,%r9 + + movq %r13,%rax + shrq $1,%r12 + shlq $63,%rax + movq %r14,%r10 + shrq $1,%r13 + orq %rax,%r12 + shlq $63,%r10 + movq %r15,%rcx + shrq $1,%r14 + orq %r10,%r13 + shlq $63,%rcx + movq %r12,0(%rdi) + shrq $1,%r15 + movq %r13,8(%rdi) + shlq $63,%r9 + orq %rcx,%r14 + orq %r9,%r15 + movq %r14,16(%rdi) + movq %r15,24(%rdi) + movq 64(%rsp),%rax + leaq 64(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_by_2q + + leaq 32(%rsp),%rbx + leaq 32(%rsp),%rdi + call __ecp_nistz256_add_toq + + movq 96(%rsp),%rax + leaq 96(%rsp),%rbx + movq 0+0(%rsp),%r9 + movq 8+0(%rsp),%r10 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r11 + movq 24+0(%rsp),%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_by_2q + + movq 0+32(%rsp),%rax + movq 8+32(%rsp),%r14 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r15 + movq 24+32(%rsp),%r8 + movq %xmm0,%rdi + call __ecp_nistz256_sqr_montq + + leaq 128(%rsp),%rbx + movq %r14,%r8 + movq %r15,%r9 + movq %rsi,%r14 + movq %rbp,%r15 + call __ecp_nistz256_sub_fromq + + movq 0+0(%rsp),%rax + movq 0+8(%rsp),%rbp + movq 0+16(%rsp),%rcx + movq 0+24(%rsp),%r10 + leaq 0(%rsp),%rdi + call __ecp_nistz256_subq + + movq 32(%rsp),%rax + leaq 32(%rsp),%rbx + movq %r12,%r14 + xorl %ecx,%ecx + movq %r12,0+0(%rsp) + movq %r13,%r10 + movq %r13,0+8(%rsp) + cmovzq %r8,%r11 + movq %r8,0+16(%rsp) + leaq 0-0(%rsp),%rsi + cmovzq %r9,%r12 + movq %r9,0+24(%rsp) + movq %r14,%r9 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq %xmm1,%rbx + movq %xmm1,%rdi + call __ecp_nistz256_sub_fromq + + leaq 160+56(%rsp),%rsi + + movq -48(%rsi),%r15 + + movq -40(%rsi),%r14 + + movq -32(%rsi),%r13 + + movq -24(%rsi),%r12 + + movq -16(%rsi),%rbx + + movq -8(%rsi),%rbp + + leaq (%rsi),%rsp + +L$point_doubleq_epilogue: + ret + + +.globl _ecp_nistz256_point_add_nohw +.private_extern _ecp_nistz256_point_add_nohw + +.p2align 5 +.alt_entry _ecp_nistz256_point_add_nohw +_ecp_nistz256_point_add_nohw: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $576+8,%rsp + +L$point_addq_body: + + movdqu 0(%rsi),%xmm0 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + movdqu 64(%rsi),%xmm4 + movdqu 80(%rsi),%xmm5 + movq %rsi,%rbx + movq %rdx,%rsi + movdqa %xmm0,384(%rsp) + movdqa %xmm1,384+16(%rsp) + movdqa %xmm2,416(%rsp) + movdqa %xmm3,416+16(%rsp) + movdqa %xmm4,448(%rsp) + movdqa %xmm5,448+16(%rsp) + por %xmm4,%xmm5 + + movdqu 0(%rsi),%xmm0 + pshufd $0xb1,%xmm5,%xmm3 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + por %xmm3,%xmm5 + movdqu 48(%rsi),%xmm3 + movq 64+0(%rsi),%rax + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + movdqa %xmm0,480(%rsp) + pshufd $0x1e,%xmm5,%xmm4 + movdqa %xmm1,480+16(%rsp) + movdqu 64(%rsi),%xmm0 + movdqu 80(%rsi),%xmm1 + movdqa %xmm2,512(%rsp) + movdqa %xmm3,512+16(%rsp) + por %xmm4,%xmm5 + pxor %xmm4,%xmm4 + por %xmm0,%xmm1 + movq %rdi,%xmm0 + + leaq 64-0(%rsi),%rsi + movq %rax,544+0(%rsp) + movq %r14,544+8(%rsp) + movq %r15,544+16(%rsp) + movq %r8,544+24(%rsp) + leaq 96(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + pcmpeqd %xmm4,%xmm5 + pshufd $0xb1,%xmm1,%xmm4 + por %xmm1,%xmm4 + pshufd $0,%xmm5,%xmm5 + pshufd $0x1e,%xmm4,%xmm3 + por %xmm3,%xmm4 + pxor %xmm3,%xmm3 + pcmpeqd %xmm3,%xmm4 + pshufd $0,%xmm4,%xmm4 + movq 64+0(%rbx),%rax + movq 64+8(%rbx),%r14 + movq 64+16(%rbx),%r15 + movq 64+24(%rbx),%r8 + movq %rbx,%xmm1 + + leaq 64-0(%rbx),%rsi + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 544(%rsp),%rax + leaq 544(%rsp),%rbx + movq 0+96(%rsp),%r9 + movq 8+96(%rsp),%r10 + leaq 0+96(%rsp),%rsi + movq 16+96(%rsp),%r11 + movq 24+96(%rsp),%r12 + leaq 224(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 448(%rsp),%rax + leaq 448(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 416(%rsp),%rax + leaq 416(%rsp),%rbx + movq 0+224(%rsp),%r9 + movq 8+224(%rsp),%r10 + leaq 0+224(%rsp),%rsi + movq 16+224(%rsp),%r11 + movq 24+224(%rsp),%r12 + leaq 224(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 512(%rsp),%rax + leaq 512(%rsp),%rbx + movq 0+256(%rsp),%r9 + movq 8+256(%rsp),%r10 + leaq 0+256(%rsp),%rsi + movq 16+256(%rsp),%r11 + movq 24+256(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 224(%rsp),%rbx + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + orq %r13,%r12 + movdqa %xmm4,%xmm2 + orq %r8,%r12 + orq %r9,%r12 + por %xmm5,%xmm2 + movq %r12,%xmm3 + + movq 384(%rsp),%rax + leaq 384(%rsp),%rbx + movq 0+96(%rsp),%r9 + movq 8+96(%rsp),%r10 + leaq 0+96(%rsp),%rsi + movq 16+96(%rsp),%r11 + movq 24+96(%rsp),%r12 + leaq 160(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 480(%rsp),%rax + leaq 480(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 192(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 160(%rsp),%rbx + leaq 0(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + orq %r13,%r12 + orq %r8,%r12 + orq %r9,%r12 + + movq %xmm2,%r8 + movq %xmm3,%r9 + orq %r8,%r12 +.byte 0x3e + jnz L$add_proceedq + + + + testq %r9,%r9 + jz L$add_doubleq + + + + + + + movq %xmm0,%rdi + pxor %xmm0,%xmm0 + movdqu %xmm0,0(%rdi) + movdqu %xmm0,16(%rdi) + movdqu %xmm0,32(%rdi) + movdqu %xmm0,48(%rdi) + movdqu %xmm0,64(%rdi) + movdqu %xmm0,80(%rdi) + jmp L$add_doneq + +.p2align 5 +L$add_doubleq: + movq %xmm1,%rsi + movq %xmm0,%rdi + addq $416,%rsp + + jmp L$point_double_shortcutq + + +.p2align 5 +L$add_proceedq: + movq 0+64(%rsp),%rax + movq 8+64(%rsp),%r14 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r15 + movq 24+64(%rsp),%r8 + leaq 96(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 448(%rsp),%rax + leaq 448(%rsp),%rbx + movq 0+0(%rsp),%r9 + movq 8+0(%rsp),%r10 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r11 + movq 24+0(%rsp),%r12 + leaq 352(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 0+0(%rsp),%rax + movq 8+0(%rsp),%r14 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 544(%rsp),%rax + leaq 544(%rsp),%rbx + movq 0+352(%rsp),%r9 + movq 8+352(%rsp),%r10 + leaq 0+352(%rsp),%rsi + movq 16+352(%rsp),%r11 + movq 24+352(%rsp),%r12 + leaq 352(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 0(%rsp),%rax + leaq 0(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 160(%rsp),%rax + leaq 160(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 192(%rsp),%rdi + call __ecp_nistz256_mul_montq + + + + + xorq %r11,%r11 + addq %r12,%r12 + leaq 96(%rsp),%rsi + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + movq 0(%rsi),%rax + cmovcq %rbp,%r13 + movq 8(%rsi),%rbp + cmovcq %rcx,%r8 + movq 16(%rsi),%rcx + cmovcq %r10,%r9 + movq 24(%rsi),%r10 + + call __ecp_nistz256_subq + + leaq 128(%rsp),%rbx + leaq 288(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 192+0(%rsp),%rax + movq 192+8(%rsp),%rbp + movq 192+16(%rsp),%rcx + movq 192+24(%rsp),%r10 + leaq 320(%rsp),%rdi + + call __ecp_nistz256_subq + + movq %r12,0(%rdi) + movq %r13,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq 128(%rsp),%rax + leaq 128(%rsp),%rbx + movq 0+224(%rsp),%r9 + movq 8+224(%rsp),%r10 + leaq 0+224(%rsp),%rsi + movq 16+224(%rsp),%r11 + movq 24+224(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 320(%rsp),%rax + leaq 320(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 320(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 256(%rsp),%rbx + leaq 320(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq %xmm0,%rdi + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 352(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 352+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 544(%rsp),%xmm2 + pand 544+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 448(%rsp),%xmm2 + pand 448+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,64(%rdi) + movdqu %xmm3,80(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 288(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 288+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 480(%rsp),%xmm2 + pand 480+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 384(%rsp),%xmm2 + pand 384+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 320(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 320+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 512(%rsp),%xmm2 + pand 512+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 416(%rsp),%xmm2 + pand 416+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + +L$add_doneq: + leaq 576+56(%rsp),%rsi + + movq -48(%rsi),%r15 + + movq -40(%rsi),%r14 + + movq -32(%rsi),%r13 + + movq -24(%rsi),%r12 + + movq -16(%rsi),%rbx + + movq -8(%rsi),%rbp + + leaq (%rsi),%rsp + +L$point_addq_epilogue: + ret + + +.globl _ecp_nistz256_point_add_affine_nohw +.private_extern _ecp_nistz256_point_add_affine_nohw + +.p2align 5 +.alt_entry _ecp_nistz256_point_add_affine_nohw +_ecp_nistz256_point_add_affine_nohw: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $480+8,%rsp + +L$add_affineq_body: + + movdqu 0(%rsi),%xmm0 + movq %rdx,%rbx + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + movdqu 64(%rsi),%xmm4 + movdqu 80(%rsi),%xmm5 + movq 64+0(%rsi),%rax + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + movdqa %xmm0,320(%rsp) + movdqa %xmm1,320+16(%rsp) + movdqa %xmm2,352(%rsp) + movdqa %xmm3,352+16(%rsp) + movdqa %xmm4,384(%rsp) + movdqa %xmm5,384+16(%rsp) + por %xmm4,%xmm5 + + movdqu 0(%rbx),%xmm0 + pshufd $0xb1,%xmm5,%xmm3 + movdqu 16(%rbx),%xmm1 + movdqu 32(%rbx),%xmm2 + por %xmm3,%xmm5 + movdqu 48(%rbx),%xmm3 + movdqa %xmm0,416(%rsp) + pshufd $0x1e,%xmm5,%xmm4 + movdqa %xmm1,416+16(%rsp) + por %xmm0,%xmm1 + movq %rdi,%xmm0 + movdqa %xmm2,448(%rsp) + movdqa %xmm3,448+16(%rsp) + por %xmm2,%xmm3 + por %xmm4,%xmm5 + pxor %xmm4,%xmm4 + por %xmm1,%xmm3 + + leaq 64-0(%rsi),%rsi + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + pcmpeqd %xmm4,%xmm5 + pshufd $0xb1,%xmm3,%xmm4 + movq 0(%rbx),%rax + + movq %r12,%r9 + por %xmm3,%xmm4 + pshufd $0,%xmm5,%xmm5 + pshufd $0x1e,%xmm4,%xmm3 + movq %r13,%r10 + por %xmm3,%xmm4 + pxor %xmm3,%xmm3 + movq %r14,%r11 + pcmpeqd %xmm3,%xmm4 + pshufd $0,%xmm4,%xmm4 + + leaq 32-0(%rsp),%rsi + movq %r15,%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 320(%rsp),%rbx + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 384(%rsp),%rax + leaq 384(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 384(%rsp),%rax + leaq 384(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 288(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 448(%rsp),%rax + leaq 448(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 352(%rsp),%rbx + leaq 96(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 0+64(%rsp),%rax + movq 8+64(%rsp),%r14 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r15 + movq 24+64(%rsp),%r8 + leaq 128(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 0+96(%rsp),%rax + movq 8+96(%rsp),%r14 + leaq 0+96(%rsp),%rsi + movq 16+96(%rsp),%r15 + movq 24+96(%rsp),%r8 + leaq 192(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 128(%rsp),%rax + leaq 128(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 160(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 320(%rsp),%rax + leaq 320(%rsp),%rbx + movq 0+128(%rsp),%r9 + movq 8+128(%rsp),%r10 + leaq 0+128(%rsp),%rsi + movq 16+128(%rsp),%r11 + movq 24+128(%rsp),%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montq + + + + + xorq %r11,%r11 + addq %r12,%r12 + leaq 192(%rsp),%rsi + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + movq 0(%rsi),%rax + cmovcq %rbp,%r13 + movq 8(%rsi),%rbp + cmovcq %rcx,%r8 + movq 16(%rsi),%rcx + cmovcq %r10,%r9 + movq 24(%rsi),%r10 + + call __ecp_nistz256_subq + + leaq 160(%rsp),%rbx + leaq 224(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 0+0(%rsp),%rax + movq 0+8(%rsp),%rbp + movq 0+16(%rsp),%rcx + movq 0+24(%rsp),%r10 + leaq 64(%rsp),%rdi + + call __ecp_nistz256_subq + + movq %r12,0(%rdi) + movq %r13,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq 352(%rsp),%rax + leaq 352(%rsp),%rbx + movq 0+160(%rsp),%r9 + movq 8+160(%rsp),%r10 + leaq 0+160(%rsp),%rsi + movq 16+160(%rsp),%r11 + movq 24+160(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 96(%rsp),%rax + leaq 96(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 64(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 32(%rsp),%rbx + leaq 256(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq %xmm0,%rdi + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 288(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 288+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand L$ONE_mont(%rip),%xmm2 + pand L$ONE_mont+16(%rip),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 384(%rsp),%xmm2 + pand 384+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,64(%rdi) + movdqu %xmm3,80(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 224(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 224+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 416(%rsp),%xmm2 + pand 416+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 320(%rsp),%xmm2 + pand 320+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 256(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 256+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 448(%rsp),%xmm2 + pand 448+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 352(%rsp),%xmm2 + pand 352+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + + leaq 480+56(%rsp),%rsi + + movq -48(%rsi),%r15 + + movq -40(%rsi),%r14 + + movq -32(%rsi),%r13 + + movq -24(%rsi),%r12 + + movq -16(%rsi),%rbx + + movq -8(%rsi),%rbp + + leaq (%rsi),%rsp + +L$add_affineq_epilogue: + ret + + + +.p2align 5 +.alt_entry __ecp_nistz256_add_tox +__ecp_nistz256_add_tox: + + xorq %r11,%r11 + adcq 0(%rbx),%r12 + adcq 8(%rbx),%r13 + movq %r12,%rax + adcq 16(%rbx),%r8 + adcq 24(%rbx),%r9 + movq %r13,%rbp + adcq $0,%r11 + + xorq %r10,%r10 + sbbq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_sub_fromx +__ecp_nistz256_sub_fromx: + + xorq %r11,%r11 + sbbq 0(%rbx),%r12 + sbbq 8(%rbx),%r13 + movq %r12,%rax + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + movq %r13,%rbp + sbbq $0,%r11 + + xorq %r10,%r10 + adcq $-1,%r12 + movq %r8,%rcx + adcq %r14,%r13 + adcq $0,%r8 + movq %r9,%r10 + adcq %r15,%r9 + + btq $0,%r11 + cmovncq %rax,%r12 + cmovncq %rbp,%r13 + movq %r12,0(%rdi) + cmovncq %rcx,%r8 + movq %r13,8(%rdi) + cmovncq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_subx +__ecp_nistz256_subx: + + xorq %r11,%r11 + sbbq %r12,%rax + sbbq %r13,%rbp + movq %rax,%r12 + sbbq %r8,%rcx + sbbq %r9,%r10 + movq %rbp,%r13 + sbbq $0,%r11 + + xorq %r9,%r9 + adcq $-1,%rax + movq %rcx,%r8 + adcq %r14,%rbp + adcq $0,%rcx + movq %r10,%r9 + adcq %r15,%r10 + + btq $0,%r11 + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + cmovcq %rcx,%r8 + cmovcq %r10,%r9 + + ret + + + + +.p2align 5 +.alt_entry __ecp_nistz256_mul_by_2x +__ecp_nistz256_mul_by_2x: + + xorq %r11,%r11 + adcq %r12,%r12 + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + xorq %r10,%r10 + sbbq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret + + +.globl _ecp_nistz256_point_double_adx +.private_extern _ecp_nistz256_point_double_adx + +.p2align 5 +.alt_entry _ecp_nistz256_point_double_adx +_ecp_nistz256_point_double_adx: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $160+8,%rsp + +L$point_doublex_body: + +L$point_double_shortcutx: + movdqu 0(%rsi),%xmm0 + movq %rsi,%rbx + movdqu 16(%rsi),%xmm1 + movq 32+0(%rsi),%r12 + movq 32+8(%rsi),%r13 + movq 32+16(%rsi),%r8 + movq 32+24(%rsi),%r9 + movq L$poly+8(%rip),%r14 + movq L$poly+24(%rip),%r15 + movdqa %xmm0,96(%rsp) + movdqa %xmm1,96+16(%rsp) + leaq 32(%rdi),%r10 + leaq 64(%rdi),%r11 + movq %rdi,%xmm0 + movq %r10,%xmm1 + movq %r11,%xmm2 + + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_by_2x + + movq 64+0(%rsi),%rdx + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + leaq 64-128(%rsi),%rsi + leaq 64(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 0+0(%rsp),%rdx + movq 8+0(%rsp),%r14 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + leaq 0(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 32(%rbx),%rdx + movq 64+0(%rbx),%r9 + movq 64+8(%rbx),%r10 + movq 64+16(%rbx),%r11 + movq 64+24(%rbx),%r12 + leaq 64-128(%rbx),%rsi + leaq 32(%rbx),%rbx + movq %xmm2,%rdi + call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_by_2x + + movq 96+0(%rsp),%r12 + movq 96+8(%rsp),%r13 + leaq 64(%rsp),%rbx + movq 96+16(%rsp),%r8 + movq 96+24(%rsp),%r9 + leaq 32(%rsp),%rdi + call __ecp_nistz256_add_tox + + movq 96+0(%rsp),%r12 + movq 96+8(%rsp),%r13 + leaq 64(%rsp),%rbx + movq 96+16(%rsp),%r8 + movq 96+24(%rsp),%r9 + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 0+0(%rsp),%rdx + movq 8+0(%rsp),%r14 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + movq %xmm1,%rdi + call __ecp_nistz256_sqr_montx + xorq %r9,%r9 + movq %r12,%rax + addq $-1,%r12 + movq %r13,%r10 + adcq %rsi,%r13 + movq %r14,%rcx + adcq $0,%r14 + movq %r15,%r8 + adcq %rbp,%r15 + adcq $0,%r9 + xorq %rsi,%rsi + testq $1,%rax + + cmovzq %rax,%r12 + cmovzq %r10,%r13 + cmovzq %rcx,%r14 + cmovzq %r8,%r15 + cmovzq %rsi,%r9 + + movq %r13,%rax + shrq $1,%r12 + shlq $63,%rax + movq %r14,%r10 + shrq $1,%r13 + orq %rax,%r12 + shlq $63,%r10 + movq %r15,%rcx + shrq $1,%r14 + orq %r10,%r13 + shlq $63,%rcx + movq %r12,0(%rdi) + shrq $1,%r15 + movq %r13,8(%rdi) + shlq $63,%r9 + orq %rcx,%r14 + orq %r9,%r15 + movq %r14,16(%rdi) + movq %r15,24(%rdi) + movq 64(%rsp),%rdx + leaq 64(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_by_2x + + leaq 32(%rsp),%rbx + leaq 32(%rsp),%rdi + call __ecp_nistz256_add_tox + + movq 96(%rsp),%rdx + leaq 96(%rsp),%rbx + movq 0+0(%rsp),%r9 + movq 8+0(%rsp),%r10 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r11 + movq 24+0(%rsp),%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_by_2x + + movq 0+32(%rsp),%rdx + movq 8+32(%rsp),%r14 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r15 + movq 24+32(%rsp),%r8 + movq %xmm0,%rdi + call __ecp_nistz256_sqr_montx + + leaq 128(%rsp),%rbx + movq %r14,%r8 + movq %r15,%r9 + movq %rsi,%r14 + movq %rbp,%r15 + call __ecp_nistz256_sub_fromx + + movq 0+0(%rsp),%rax + movq 0+8(%rsp),%rbp + movq 0+16(%rsp),%rcx + movq 0+24(%rsp),%r10 + leaq 0(%rsp),%rdi + call __ecp_nistz256_subx + + movq 32(%rsp),%rdx + leaq 32(%rsp),%rbx + movq %r12,%r14 + xorl %ecx,%ecx + movq %r12,0+0(%rsp) + movq %r13,%r10 + movq %r13,0+8(%rsp) + cmovzq %r8,%r11 + movq %r8,0+16(%rsp) + leaq 0-128(%rsp),%rsi + cmovzq %r9,%r12 + movq %r9,0+24(%rsp) + movq %r14,%r9 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq %xmm1,%rbx + movq %xmm1,%rdi + call __ecp_nistz256_sub_fromx + + leaq 160+56(%rsp),%rsi + + movq -48(%rsi),%r15 + + movq -40(%rsi),%r14 + + movq -32(%rsi),%r13 + + movq -24(%rsi),%r12 + + movq -16(%rsi),%rbx + + movq -8(%rsi),%rbp + + leaq (%rsi),%rsp + +L$point_doublex_epilogue: + ret + + +.globl _ecp_nistz256_point_add_adx +.private_extern _ecp_nistz256_point_add_adx + +.p2align 5 +.alt_entry _ecp_nistz256_point_add_adx +_ecp_nistz256_point_add_adx: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $576+8,%rsp + +L$point_addx_body: + + movdqu 0(%rsi),%xmm0 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + movdqu 64(%rsi),%xmm4 + movdqu 80(%rsi),%xmm5 + movq %rsi,%rbx + movq %rdx,%rsi + movdqa %xmm0,384(%rsp) + movdqa %xmm1,384+16(%rsp) + movdqa %xmm2,416(%rsp) + movdqa %xmm3,416+16(%rsp) + movdqa %xmm4,448(%rsp) + movdqa %xmm5,448+16(%rsp) + por %xmm4,%xmm5 + + movdqu 0(%rsi),%xmm0 + pshufd $0xb1,%xmm5,%xmm3 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + por %xmm3,%xmm5 + movdqu 48(%rsi),%xmm3 + movq 64+0(%rsi),%rdx + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + movdqa %xmm0,480(%rsp) + pshufd $0x1e,%xmm5,%xmm4 + movdqa %xmm1,480+16(%rsp) + movdqu 64(%rsi),%xmm0 + movdqu 80(%rsi),%xmm1 + movdqa %xmm2,512(%rsp) + movdqa %xmm3,512+16(%rsp) + por %xmm4,%xmm5 + pxor %xmm4,%xmm4 + por %xmm0,%xmm1 + movq %rdi,%xmm0 + + leaq 64-128(%rsi),%rsi + movq %rdx,544+0(%rsp) + movq %r14,544+8(%rsp) + movq %r15,544+16(%rsp) + movq %r8,544+24(%rsp) + leaq 96(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + pcmpeqd %xmm4,%xmm5 + pshufd $0xb1,%xmm1,%xmm4 + por %xmm1,%xmm4 + pshufd $0,%xmm5,%xmm5 + pshufd $0x1e,%xmm4,%xmm3 + por %xmm3,%xmm4 + pxor %xmm3,%xmm3 + pcmpeqd %xmm3,%xmm4 + pshufd $0,%xmm4,%xmm4 + movq 64+0(%rbx),%rdx + movq 64+8(%rbx),%r14 + movq 64+16(%rbx),%r15 + movq 64+24(%rbx),%r8 + movq %rbx,%xmm1 + + leaq 64-128(%rbx),%rsi + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 544(%rsp),%rdx + leaq 544(%rsp),%rbx + movq 0+96(%rsp),%r9 + movq 8+96(%rsp),%r10 + leaq -128+96(%rsp),%rsi + movq 16+96(%rsp),%r11 + movq 24+96(%rsp),%r12 + leaq 224(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 448(%rsp),%rdx + leaq 448(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 416(%rsp),%rdx + leaq 416(%rsp),%rbx + movq 0+224(%rsp),%r9 + movq 8+224(%rsp),%r10 + leaq -128+224(%rsp),%rsi + movq 16+224(%rsp),%r11 + movq 24+224(%rsp),%r12 + leaq 224(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 512(%rsp),%rdx + leaq 512(%rsp),%rbx + movq 0+256(%rsp),%r9 + movq 8+256(%rsp),%r10 + leaq -128+256(%rsp),%rsi + movq 16+256(%rsp),%r11 + movq 24+256(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 224(%rsp),%rbx + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + orq %r13,%r12 + movdqa %xmm4,%xmm2 + orq %r8,%r12 + orq %r9,%r12 + por %xmm5,%xmm2 + movq %r12,%xmm3 + + movq 384(%rsp),%rdx + leaq 384(%rsp),%rbx + movq 0+96(%rsp),%r9 + movq 8+96(%rsp),%r10 + leaq -128+96(%rsp),%rsi + movq 16+96(%rsp),%r11 + movq 24+96(%rsp),%r12 + leaq 160(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 480(%rsp),%rdx + leaq 480(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 192(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 160(%rsp),%rbx + leaq 0(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + orq %r13,%r12 + orq %r8,%r12 + orq %r9,%r12 + + movq %xmm2,%r8 + movq %xmm3,%r9 + orq %r8,%r12 +.byte 0x3e + jnz L$add_proceedx + + + + testq %r9,%r9 + jz L$add_doublex + + + + + + + movq %xmm0,%rdi + pxor %xmm0,%xmm0 + movdqu %xmm0,0(%rdi) + movdqu %xmm0,16(%rdi) + movdqu %xmm0,32(%rdi) + movdqu %xmm0,48(%rdi) + movdqu %xmm0,64(%rdi) + movdqu %xmm0,80(%rdi) + jmp L$add_donex + +.p2align 5 +L$add_doublex: + movq %xmm1,%rsi + movq %xmm0,%rdi + addq $416,%rsp + + jmp L$point_double_shortcutx + + +.p2align 5 +L$add_proceedx: + movq 0+64(%rsp),%rdx + movq 8+64(%rsp),%r14 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r15 + movq 24+64(%rsp),%r8 + leaq 96(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 448(%rsp),%rdx + leaq 448(%rsp),%rbx + movq 0+0(%rsp),%r9 + movq 8+0(%rsp),%r10 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r11 + movq 24+0(%rsp),%r12 + leaq 352(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 0+0(%rsp),%rdx + movq 8+0(%rsp),%r14 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 544(%rsp),%rdx + leaq 544(%rsp),%rbx + movq 0+352(%rsp),%r9 + movq 8+352(%rsp),%r10 + leaq -128+352(%rsp),%rsi + movq 16+352(%rsp),%r11 + movq 24+352(%rsp),%r12 + leaq 352(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 0(%rsp),%rdx + leaq 0(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 160(%rsp),%rdx + leaq 160(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 192(%rsp),%rdi + call __ecp_nistz256_mul_montx + + + + + xorq %r11,%r11 + addq %r12,%r12 + leaq 96(%rsp),%rsi + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + movq 0(%rsi),%rax + cmovcq %rbp,%r13 + movq 8(%rsi),%rbp + cmovcq %rcx,%r8 + movq 16(%rsi),%rcx + cmovcq %r10,%r9 + movq 24(%rsi),%r10 + + call __ecp_nistz256_subx + + leaq 128(%rsp),%rbx + leaq 288(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 192+0(%rsp),%rax + movq 192+8(%rsp),%rbp + movq 192+16(%rsp),%rcx + movq 192+24(%rsp),%r10 + leaq 320(%rsp),%rdi + + call __ecp_nistz256_subx + + movq %r12,0(%rdi) + movq %r13,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq 128(%rsp),%rdx + leaq 128(%rsp),%rbx + movq 0+224(%rsp),%r9 + movq 8+224(%rsp),%r10 + leaq -128+224(%rsp),%rsi + movq 16+224(%rsp),%r11 + movq 24+224(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 320(%rsp),%rdx + leaq 320(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 320(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 256(%rsp),%rbx + leaq 320(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq %xmm0,%rdi + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 352(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 352+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 544(%rsp),%xmm2 + pand 544+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 448(%rsp),%xmm2 + pand 448+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,64(%rdi) + movdqu %xmm3,80(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 288(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 288+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 480(%rsp),%xmm2 + pand 480+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 384(%rsp),%xmm2 + pand 384+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 320(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 320+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 512(%rsp),%xmm2 + pand 512+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 416(%rsp),%xmm2 + pand 416+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + +L$add_donex: + leaq 576+56(%rsp),%rsi + + movq -48(%rsi),%r15 + + movq -40(%rsi),%r14 + + movq -32(%rsi),%r13 + + movq -24(%rsi),%r12 + + movq -16(%rsi),%rbx + + movq -8(%rsi),%rbp + + leaq (%rsi),%rsp + +L$point_addx_epilogue: + ret + + +.globl _ecp_nistz256_point_add_affine_adx +.private_extern _ecp_nistz256_point_add_affine_adx + +.p2align 5 +.alt_entry _ecp_nistz256_point_add_affine_adx +_ecp_nistz256_point_add_affine_adx: + +_CET_ENDBR + pushq %rbp + + pushq %rbx + + pushq %r12 + + pushq %r13 + + pushq %r14 + + pushq %r15 + + subq $480+8,%rsp + +L$add_affinex_body: + + movdqu 0(%rsi),%xmm0 + movq %rdx,%rbx + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + movdqu 64(%rsi),%xmm4 + movdqu 80(%rsi),%xmm5 + movq 64+0(%rsi),%rdx + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + movdqa %xmm0,320(%rsp) + movdqa %xmm1,320+16(%rsp) + movdqa %xmm2,352(%rsp) + movdqa %xmm3,352+16(%rsp) + movdqa %xmm4,384(%rsp) + movdqa %xmm5,384+16(%rsp) + por %xmm4,%xmm5 + + movdqu 0(%rbx),%xmm0 + pshufd $0xb1,%xmm5,%xmm3 + movdqu 16(%rbx),%xmm1 + movdqu 32(%rbx),%xmm2 + por %xmm3,%xmm5 + movdqu 48(%rbx),%xmm3 + movdqa %xmm0,416(%rsp) + pshufd $0x1e,%xmm5,%xmm4 + movdqa %xmm1,416+16(%rsp) + por %xmm0,%xmm1 + movq %rdi,%xmm0 + movdqa %xmm2,448(%rsp) + movdqa %xmm3,448+16(%rsp) + por %xmm2,%xmm3 + por %xmm4,%xmm5 + pxor %xmm4,%xmm4 + por %xmm1,%xmm3 + + leaq 64-128(%rsi),%rsi + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + pcmpeqd %xmm4,%xmm5 + pshufd $0xb1,%xmm3,%xmm4 + movq 0(%rbx),%rdx + + movq %r12,%r9 + por %xmm3,%xmm4 + pshufd $0,%xmm5,%xmm5 + pshufd $0x1e,%xmm4,%xmm3 + movq %r13,%r10 + por %xmm3,%xmm4 + pxor %xmm3,%xmm3 + movq %r14,%r11 + pcmpeqd %xmm3,%xmm4 + pshufd $0,%xmm4,%xmm4 + + leaq 32-128(%rsp),%rsi + movq %r15,%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 320(%rsp),%rbx + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 384(%rsp),%rdx + leaq 384(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 384(%rsp),%rdx + leaq 384(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 288(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 448(%rsp),%rdx + leaq 448(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 352(%rsp),%rbx + leaq 96(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 0+64(%rsp),%rdx + movq 8+64(%rsp),%r14 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r15 + movq 24+64(%rsp),%r8 + leaq 128(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 0+96(%rsp),%rdx + movq 8+96(%rsp),%r14 + leaq -128+96(%rsp),%rsi + movq 16+96(%rsp),%r15 + movq 24+96(%rsp),%r8 + leaq 192(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 128(%rsp),%rdx + leaq 128(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 160(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 320(%rsp),%rdx + leaq 320(%rsp),%rbx + movq 0+128(%rsp),%r9 + movq 8+128(%rsp),%r10 + leaq -128+128(%rsp),%rsi + movq 16+128(%rsp),%r11 + movq 24+128(%rsp),%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montx + + + + + xorq %r11,%r11 + addq %r12,%r12 + leaq 192(%rsp),%rsi + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + movq 0(%rsi),%rax + cmovcq %rbp,%r13 + movq 8(%rsi),%rbp + cmovcq %rcx,%r8 + movq 16(%rsi),%rcx + cmovcq %r10,%r9 + movq 24(%rsi),%r10 + + call __ecp_nistz256_subx + + leaq 160(%rsp),%rbx + leaq 224(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 0+0(%rsp),%rax + movq 0+8(%rsp),%rbp + movq 0+16(%rsp),%rcx + movq 0+24(%rsp),%r10 + leaq 64(%rsp),%rdi + + call __ecp_nistz256_subx + + movq %r12,0(%rdi) + movq %r13,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq 352(%rsp),%rdx + leaq 352(%rsp),%rbx + movq 0+160(%rsp),%r9 + movq 8+160(%rsp),%r10 + leaq -128+160(%rsp),%rsi + movq 16+160(%rsp),%r11 + movq 24+160(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 96(%rsp),%rdx + leaq 96(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 64(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 32(%rsp),%rbx + leaq 256(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq %xmm0,%rdi + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 288(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 288+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand L$ONE_mont(%rip),%xmm2 + pand L$ONE_mont+16(%rip),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 384(%rsp),%xmm2 + pand 384+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,64(%rdi) + movdqu %xmm3,80(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 224(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 224+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 416(%rsp),%xmm2 + pand 416+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 320(%rsp),%xmm2 + pand 320+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 256(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 256+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 448(%rsp),%xmm2 + pand 448+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 352(%rsp),%xmm2 + pand 352+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + + leaq 480+56(%rsp),%rsi + + movq -48(%rsi),%r15 + + movq -40(%rsi),%r14 + + movq -32(%rsi),%r13 + + movq -24(%rsi),%r12 + + movq -16(%rsi),%rbx + + movq -8(%rsi),%rbp + + leaq (%rsi),%rsp + +L$add_affinex_epilogue: + ret + + #endif
diff --git a/gen/bcm/p256-x86_64-asm-linux.S b/gen/bcm/p256-x86_64-asm-linux.S index 474b13b..059e4cf 100644 --- a/gen/bcm/p256-x86_64-asm-linux.S +++ b/gen/bcm/p256-x86_64-asm-linux.S
@@ -7,7 +7,22 @@ .text +.section .rodata +.align 64 p256_constants: +.Lpoly: +.quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 + +.LOne: +.long 1,1,1,1,1,1,1,1 +.LTwo: +.long 2,2,2,2,2,2,2,2 +.LThree: +.long 3,3,3,3,3,3,3,3 +.LONE_mont: +.quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe + + .Lord: .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 .LordK: @@ -16,6 +31,66 @@ +.globl ecp_nistz256_neg +.hidden ecp_nistz256_neg +.type ecp_nistz256_neg,@function +.align 32 +ecp_nistz256_neg: +.cfi_startproc +_CET_ENDBR + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-16 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-24 +.Lneg_body: + + xorq %r8,%r8 + xorq %r9,%r9 + xorq %r10,%r10 + xorq %r11,%r11 + xorq %r13,%r13 + + subq 0(%rsi),%r8 + sbbq 8(%rsi),%r9 + sbbq 16(%rsi),%r10 + movq %r8,%rax + sbbq 24(%rsi),%r11 + leaq .Lpoly(%rip),%rsi + movq %r9,%rdx + sbbq $0,%r13 + + addq 0(%rsi),%r8 + movq %r10,%rcx + adcq 8(%rsi),%r9 + adcq 16(%rsi),%r10 + movq %r11,%r12 + adcq 24(%rsi),%r11 + testq %r13,%r13 + + cmovzq %rax,%r8 + cmovzq %rdx,%r9 + movq %r8,0(%rdi) + cmovzq %rcx,%r10 + movq %r9,8(%rdi) + cmovzq %r12,%r11 + movq %r10,16(%rdi) + movq %r11,24(%rdi) + + movq 0(%rsp),%r13 +.cfi_restore %r13 + movq 8(%rsp),%r12 +.cfi_restore %r12 + leaq 16(%rsp),%rsp +.cfi_adjust_cfa_offset -16 +.Lneg_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_neg,.-ecp_nistz256_neg + + + @@ -1104,4 +1179,3422 @@ ret .cfi_endproc .size ecp_nistz256_ord_sqr_mont_adx,.-ecp_nistz256_ord_sqr_mont_adx + + + + + + +.globl ecp_nistz256_mul_mont_nohw +.hidden ecp_nistz256_mul_mont_nohw +.type ecp_nistz256_mul_mont_nohw,@function +.align 32 +ecp_nistz256_mul_mont_nohw: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 +.Lmul_body: + movq %rdx,%rbx + movq 0(%rdx),%rax + movq 0(%rsi),%r9 + movq 8(%rsi),%r10 + movq 16(%rsi),%r11 + movq 24(%rsi),%r12 + + call __ecp_nistz256_mul_montq + + movq 0(%rsp),%r15 +.cfi_restore %r15 + movq 8(%rsp),%r14 +.cfi_restore %r14 + movq 16(%rsp),%r13 +.cfi_restore %r13 + movq 24(%rsp),%r12 +.cfi_restore %r12 + movq 32(%rsp),%rbx +.cfi_restore %rbx + movq 40(%rsp),%rbp +.cfi_restore %rbp + leaq 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lmul_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_mul_mont_nohw,.-ecp_nistz256_mul_mont_nohw + +.type __ecp_nistz256_mul_montq,@function +.align 32 +__ecp_nistz256_mul_montq: +.cfi_startproc + + + movq %rax,%rbp + mulq %r9 + movq .Lpoly+8(%rip),%r14 + movq %rax,%r8 + movq %rbp,%rax + movq %rdx,%r9 + + mulq %r10 + movq .Lpoly+24(%rip),%r15 + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r10 + + mulq %r11 + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq %r12 + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + xorq %r13,%r13 + movq %rdx,%r12 + + + + + + + + + + + movq %r8,%rbp + shlq $32,%r8 + mulq %r15 + shrq $32,%rbp + addq %r8,%r9 + adcq %rbp,%r10 + adcq %rax,%r11 + movq 8(%rbx),%rax + adcq %rdx,%r12 + adcq $0,%r13 + xorq %r8,%r8 + + + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r9 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rcx,%r10 + adcq $0,%rdx + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 16(%rsi) + addq %rcx,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 24(%rsi) + addq %rcx,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %r9,%rax + adcq %rdx,%r13 + adcq $0,%r8 + + + + movq %r9,%rbp + shlq $32,%r9 + mulq %r15 + shrq $32,%rbp + addq %r9,%r10 + adcq %rbp,%r11 + adcq %rax,%r12 + movq 16(%rbx),%rax + adcq %rdx,%r13 + adcq $0,%r8 + xorq %r9,%r9 + + + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r10 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rcx,%r11 + adcq $0,%rdx + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 16(%rsi) + addq %rcx,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 24(%rsi) + addq %rcx,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %r10,%rax + adcq %rdx,%r8 + adcq $0,%r9 + + + + movq %r10,%rbp + shlq $32,%r10 + mulq %r15 + shrq $32,%rbp + addq %r10,%r11 + adcq %rbp,%r12 + adcq %rax,%r13 + movq 24(%rbx),%rax + adcq %rdx,%r8 + adcq $0,%r9 + xorq %r10,%r10 + + + + movq %rax,%rbp + mulq 0(%rsi) + addq %rax,%r11 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 8(%rsi) + addq %rcx,%r12 + adcq $0,%rdx + addq %rax,%r12 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 16(%rsi) + addq %rcx,%r13 + adcq $0,%rdx + addq %rax,%r13 + movq %rbp,%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq 24(%rsi) + addq %rcx,%r8 + adcq $0,%rdx + addq %rax,%r8 + movq %r11,%rax + adcq %rdx,%r9 + adcq $0,%r10 + + + + movq %r11,%rbp + shlq $32,%r11 + mulq %r15 + shrq $32,%rbp + addq %r11,%r12 + adcq %rbp,%r13 + movq %r12,%rcx + adcq %rax,%r8 + adcq %rdx,%r9 + movq %r13,%rbp + adcq $0,%r10 + + + + subq $-1,%r12 + movq %r8,%rbx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%rdx + sbbq %r15,%r9 + sbbq $0,%r10 + + cmovcq %rcx,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rbx,%r8 + movq %r13,8(%rdi) + cmovcq %rdx,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq + + + + + + + + +.globl ecp_nistz256_sqr_mont_nohw +.hidden ecp_nistz256_sqr_mont_nohw +.type ecp_nistz256_sqr_mont_nohw,@function +.align 32 +ecp_nistz256_sqr_mont_nohw: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 +.Lsqr_body: + movq 0(%rsi),%rax + movq 8(%rsi),%r14 + movq 16(%rsi),%r15 + movq 24(%rsi),%r8 + + call __ecp_nistz256_sqr_montq + + movq 0(%rsp),%r15 +.cfi_restore %r15 + movq 8(%rsp),%r14 +.cfi_restore %r14 + movq 16(%rsp),%r13 +.cfi_restore %r13 + movq 24(%rsp),%r12 +.cfi_restore %r12 + movq 32(%rsp),%rbx +.cfi_restore %rbx + movq 40(%rsp),%rbp +.cfi_restore %rbp + leaq 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lsqr_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_sqr_mont_nohw,.-ecp_nistz256_sqr_mont_nohw + +.type __ecp_nistz256_sqr_montq,@function +.align 32 +__ecp_nistz256_sqr_montq: +.cfi_startproc + movq %rax,%r13 + mulq %r14 + movq %rax,%r9 + movq %r15,%rax + movq %rdx,%r10 + + mulq %r13 + addq %rax,%r10 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%r11 + + mulq %r13 + addq %rax,%r11 + movq %r15,%rax + adcq $0,%rdx + movq %rdx,%r12 + + + mulq %r14 + addq %rax,%r11 + movq %r8,%rax + adcq $0,%rdx + movq %rdx,%rbp + + mulq %r14 + addq %rax,%r12 + movq %r8,%rax + adcq $0,%rdx + addq %rbp,%r12 + movq %rdx,%r13 + adcq $0,%r13 + + + mulq %r15 + xorq %r15,%r15 + addq %rax,%r13 + movq 0(%rsi),%rax + movq %rdx,%r14 + adcq $0,%r14 + + addq %r9,%r9 + adcq %r10,%r10 + adcq %r11,%r11 + adcq %r12,%r12 + adcq %r13,%r13 + adcq %r14,%r14 + adcq $0,%r15 + + mulq %rax + movq %rax,%r8 + movq 8(%rsi),%rax + movq %rdx,%rcx + + mulq %rax + addq %rcx,%r9 + adcq %rax,%r10 + movq 16(%rsi),%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq %rax + addq %rcx,%r11 + adcq %rax,%r12 + movq 24(%rsi),%rax + adcq $0,%rdx + movq %rdx,%rcx + + mulq %rax + addq %rcx,%r13 + adcq %rax,%r14 + movq %r8,%rax + adcq %rdx,%r15 + + movq .Lpoly+8(%rip),%rsi + movq .Lpoly+24(%rip),%rbp + + + + + movq %r8,%rcx + shlq $32,%r8 + mulq %rbp + shrq $32,%rcx + addq %r8,%r9 + adcq %rcx,%r10 + adcq %rax,%r11 + movq %r9,%rax + adcq $0,%rdx + + + + movq %r9,%rcx + shlq $32,%r9 + movq %rdx,%r8 + mulq %rbp + shrq $32,%rcx + addq %r9,%r10 + adcq %rcx,%r11 + adcq %rax,%r8 + movq %r10,%rax + adcq $0,%rdx + + + + movq %r10,%rcx + shlq $32,%r10 + movq %rdx,%r9 + mulq %rbp + shrq $32,%rcx + addq %r10,%r11 + adcq %rcx,%r8 + adcq %rax,%r9 + movq %r11,%rax + adcq $0,%rdx + + + + movq %r11,%rcx + shlq $32,%r11 + movq %rdx,%r10 + mulq %rbp + shrq $32,%rcx + addq %r11,%r8 + adcq %rcx,%r9 + adcq %rax,%r10 + adcq $0,%rdx + xorq %r11,%r11 + + + + addq %r8,%r12 + adcq %r9,%r13 + movq %r12,%r8 + adcq %r10,%r14 + adcq %rdx,%r15 + movq %r13,%r9 + adcq $0,%r11 + + subq $-1,%r12 + movq %r14,%r10 + sbbq %rsi,%r13 + sbbq $0,%r14 + movq %r15,%rcx + sbbq %rbp,%r15 + sbbq $0,%r11 + + cmovcq %r8,%r12 + cmovcq %r9,%r13 + movq %r12,0(%rdi) + cmovcq %r10,%r14 + movq %r13,8(%rdi) + cmovcq %rcx,%r15 + movq %r14,16(%rdi) + movq %r15,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq +.globl ecp_nistz256_mul_mont_adx +.hidden ecp_nistz256_mul_mont_adx +.type ecp_nistz256_mul_mont_adx,@function +.align 32 +ecp_nistz256_mul_mont_adx: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 +.Lmulx_body: + movq %rdx,%rbx + movq 0(%rdx),%rdx + movq 0(%rsi),%r9 + movq 8(%rsi),%r10 + movq 16(%rsi),%r11 + movq 24(%rsi),%r12 + leaq -128(%rsi),%rsi + + call __ecp_nistz256_mul_montx + + movq 0(%rsp),%r15 +.cfi_restore %r15 + movq 8(%rsp),%r14 +.cfi_restore %r14 + movq 16(%rsp),%r13 +.cfi_restore %r13 + movq 24(%rsp),%r12 +.cfi_restore %r12 + movq 32(%rsp),%rbx +.cfi_restore %rbx + movq 40(%rsp),%rbp +.cfi_restore %rbp + leaq 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lmulx_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_mul_mont_adx,.-ecp_nistz256_mul_mont_adx + +.type __ecp_nistz256_mul_montx,@function +.align 32 +__ecp_nistz256_mul_montx: +.cfi_startproc + + + mulxq %r9,%r8,%r9 + mulxq %r10,%rcx,%r10 + movq $32,%r14 + xorq %r13,%r13 + mulxq %r11,%rbp,%r11 + movq .Lpoly+24(%rip),%r15 + adcq %rcx,%r9 + mulxq %r12,%rcx,%r12 + movq %r8,%rdx + adcq %rbp,%r10 + shlxq %r14,%r8,%rbp + adcq %rcx,%r11 + shrxq %r14,%r8,%rcx + adcq $0,%r12 + + + + addq %rbp,%r9 + adcq %rcx,%r10 + + mulxq %r15,%rcx,%rbp + movq 8(%rbx),%rdx + adcq %rcx,%r11 + adcq %rbp,%r12 + adcq $0,%r13 + xorq %r8,%r8 + + + + mulxq 0+128(%rsi),%rcx,%rbp + adcxq %rcx,%r9 + adoxq %rbp,%r10 + + mulxq 8+128(%rsi),%rcx,%rbp + adcxq %rcx,%r10 + adoxq %rbp,%r11 + + mulxq 16+128(%rsi),%rcx,%rbp + adcxq %rcx,%r11 + adoxq %rbp,%r12 + + mulxq 24+128(%rsi),%rcx,%rbp + movq %r9,%rdx + adcxq %rcx,%r12 + shlxq %r14,%r9,%rcx + adoxq %rbp,%r13 + shrxq %r14,%r9,%rbp + + adcxq %r8,%r13 + adoxq %r8,%r8 + adcq $0,%r8 + + + + addq %rcx,%r10 + adcq %rbp,%r11 + + mulxq %r15,%rcx,%rbp + movq 16(%rbx),%rdx + adcq %rcx,%r12 + adcq %rbp,%r13 + adcq $0,%r8 + xorq %r9,%r9 + + + + mulxq 0+128(%rsi),%rcx,%rbp + adcxq %rcx,%r10 + adoxq %rbp,%r11 + + mulxq 8+128(%rsi),%rcx,%rbp + adcxq %rcx,%r11 + adoxq %rbp,%r12 + + mulxq 16+128(%rsi),%rcx,%rbp + adcxq %rcx,%r12 + adoxq %rbp,%r13 + + mulxq 24+128(%rsi),%rcx,%rbp + movq %r10,%rdx + adcxq %rcx,%r13 + shlxq %r14,%r10,%rcx + adoxq %rbp,%r8 + shrxq %r14,%r10,%rbp + + adcxq %r9,%r8 + adoxq %r9,%r9 + adcq $0,%r9 + + + + addq %rcx,%r11 + adcq %rbp,%r12 + + mulxq %r15,%rcx,%rbp + movq 24(%rbx),%rdx + adcq %rcx,%r13 + adcq %rbp,%r8 + adcq $0,%r9 + xorq %r10,%r10 + + + + mulxq 0+128(%rsi),%rcx,%rbp + adcxq %rcx,%r11 + adoxq %rbp,%r12 + + mulxq 8+128(%rsi),%rcx,%rbp + adcxq %rcx,%r12 + adoxq %rbp,%r13 + + mulxq 16+128(%rsi),%rcx,%rbp + adcxq %rcx,%r13 + adoxq %rbp,%r8 + + mulxq 24+128(%rsi),%rcx,%rbp + movq %r11,%rdx + adcxq %rcx,%r8 + shlxq %r14,%r11,%rcx + adoxq %rbp,%r9 + shrxq %r14,%r11,%rbp + + adcxq %r10,%r9 + adoxq %r10,%r10 + adcq $0,%r10 + + + + addq %rcx,%r12 + adcq %rbp,%r13 + + mulxq %r15,%rcx,%rbp + movq %r12,%rbx + movq .Lpoly+8(%rip),%r14 + adcq %rcx,%r8 + movq %r13,%rdx + adcq %rbp,%r9 + adcq $0,%r10 + + + + xorl %eax,%eax + movq %r8,%rcx + sbbq $-1,%r12 + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%rbp + sbbq %r15,%r9 + sbbq $0,%r10 + + cmovcq %rbx,%r12 + cmovcq %rdx,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %rbp,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx + +.globl ecp_nistz256_sqr_mont_adx +.hidden ecp_nistz256_sqr_mont_adx +.type ecp_nistz256_sqr_mont_adx,@function +.align 32 +ecp_nistz256_sqr_mont_adx: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 +.Lsqrx_body: + movq 0(%rsi),%rdx + movq 8(%rsi),%r14 + movq 16(%rsi),%r15 + movq 24(%rsi),%r8 + leaq -128(%rsi),%rsi + + call __ecp_nistz256_sqr_montx + + movq 0(%rsp),%r15 +.cfi_restore %r15 + movq 8(%rsp),%r14 +.cfi_restore %r14 + movq 16(%rsp),%r13 +.cfi_restore %r13 + movq 24(%rsp),%r12 +.cfi_restore %r12 + movq 32(%rsp),%rbx +.cfi_restore %rbx + movq 40(%rsp),%rbp +.cfi_restore %rbp + leaq 48(%rsp),%rsp +.cfi_adjust_cfa_offset -48 +.Lsqrx_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_sqr_mont_adx,.-ecp_nistz256_sqr_mont_adx + +.type __ecp_nistz256_sqr_montx,@function +.align 32 +__ecp_nistz256_sqr_montx: +.cfi_startproc + mulxq %r14,%r9,%r10 + mulxq %r15,%rcx,%r11 + xorl %eax,%eax + adcq %rcx,%r10 + mulxq %r8,%rbp,%r12 + movq %r14,%rdx + adcq %rbp,%r11 + adcq $0,%r12 + xorq %r13,%r13 + + + mulxq %r15,%rcx,%rbp + adcxq %rcx,%r11 + adoxq %rbp,%r12 + + mulxq %r8,%rcx,%rbp + movq %r15,%rdx + adcxq %rcx,%r12 + adoxq %rbp,%r13 + adcq $0,%r13 + + + mulxq %r8,%rcx,%r14 + movq 0+128(%rsi),%rdx + xorq %r15,%r15 + adcxq %r9,%r9 + adoxq %rcx,%r13 + adcxq %r10,%r10 + adoxq %r15,%r14 + + mulxq %rdx,%r8,%rbp + movq 8+128(%rsi),%rdx + adcxq %r11,%r11 + adoxq %rbp,%r9 + adcxq %r12,%r12 + mulxq %rdx,%rcx,%rax + movq 16+128(%rsi),%rdx + adcxq %r13,%r13 + adoxq %rcx,%r10 + adcxq %r14,%r14 +.byte 0x67 + mulxq %rdx,%rcx,%rbp + movq 24+128(%rsi),%rdx + adoxq %rax,%r11 + adcxq %r15,%r15 + adoxq %rcx,%r12 + movq $32,%rsi + adoxq %rbp,%r13 +.byte 0x67,0x67 + mulxq %rdx,%rcx,%rax + movq .Lpoly+24(%rip),%rdx + adoxq %rcx,%r14 + shlxq %rsi,%r8,%rcx + adoxq %rax,%r15 + shrxq %rsi,%r8,%rax + movq %rdx,%rbp + + + addq %rcx,%r9 + adcq %rax,%r10 + + mulxq %r8,%rcx,%r8 + adcq %rcx,%r11 + shlxq %rsi,%r9,%rcx + adcq $0,%r8 + shrxq %rsi,%r9,%rax + + + addq %rcx,%r10 + adcq %rax,%r11 + + mulxq %r9,%rcx,%r9 + adcq %rcx,%r8 + shlxq %rsi,%r10,%rcx + adcq $0,%r9 + shrxq %rsi,%r10,%rax + + + addq %rcx,%r11 + adcq %rax,%r8 + + mulxq %r10,%rcx,%r10 + adcq %rcx,%r9 + shlxq %rsi,%r11,%rcx + adcq $0,%r10 + shrxq %rsi,%r11,%rax + + + addq %rcx,%r8 + adcq %rax,%r9 + + mulxq %r11,%rcx,%r11 + adcq %rcx,%r10 + adcq $0,%r11 + + xorq %rdx,%rdx + addq %r8,%r12 + movq .Lpoly+8(%rip),%rsi + adcq %r9,%r13 + movq %r12,%r8 + adcq %r10,%r14 + adcq %r11,%r15 + movq %r13,%r9 + adcq $0,%rdx + + subq $-1,%r12 + movq %r14,%r10 + sbbq %rsi,%r13 + sbbq $0,%r14 + movq %r15,%r11 + sbbq %rbp,%r15 + sbbq $0,%rdx + + cmovcq %r8,%r12 + cmovcq %r9,%r13 + movq %r12,0(%rdi) + cmovcq %r10,%r14 + movq %r13,8(%rdi) + cmovcq %r11,%r15 + movq %r14,16(%rdi) + movq %r15,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx + + +.globl ecp_nistz256_select_w5_nohw +.hidden ecp_nistz256_select_w5_nohw +.type ecp_nistz256_select_w5_nohw,@function +.align 32 +ecp_nistz256_select_w5_nohw: +.cfi_startproc +_CET_ENDBR + movdqa .LOne(%rip),%xmm0 + movd %edx,%xmm1 + + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + pxor %xmm6,%xmm6 + pxor %xmm7,%xmm7 + + movdqa %xmm0,%xmm8 + pshufd $0,%xmm1,%xmm1 + + movq $16,%rax +.Lselect_loop_sse_w5: + + movdqa %xmm8,%xmm15 + paddd %xmm0,%xmm8 + pcmpeqd %xmm1,%xmm15 + + movdqa 0(%rsi),%xmm9 + movdqa 16(%rsi),%xmm10 + movdqa 32(%rsi),%xmm11 + movdqa 48(%rsi),%xmm12 + movdqa 64(%rsi),%xmm13 + movdqa 80(%rsi),%xmm14 + leaq 96(%rsi),%rsi + + pand %xmm15,%xmm9 + pand %xmm15,%xmm10 + por %xmm9,%xmm2 + pand %xmm15,%xmm11 + por %xmm10,%xmm3 + pand %xmm15,%xmm12 + por %xmm11,%xmm4 + pand %xmm15,%xmm13 + por %xmm12,%xmm5 + pand %xmm15,%xmm14 + por %xmm13,%xmm6 + por %xmm14,%xmm7 + + decq %rax + jnz .Lselect_loop_sse_w5 + + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + movdqu %xmm4,32(%rdi) + movdqu %xmm5,48(%rdi) + movdqu %xmm6,64(%rdi) + movdqu %xmm7,80(%rdi) + ret +.cfi_endproc +.LSEH_end_ecp_nistz256_select_w5_nohw: +.size ecp_nistz256_select_w5_nohw,.-ecp_nistz256_select_w5_nohw + + + +.globl ecp_nistz256_select_w7_nohw +.hidden ecp_nistz256_select_w7_nohw +.type ecp_nistz256_select_w7_nohw,@function +.align 32 +ecp_nistz256_select_w7_nohw: +.cfi_startproc +_CET_ENDBR + movdqa .LOne(%rip),%xmm8 + movd %edx,%xmm1 + + pxor %xmm2,%xmm2 + pxor %xmm3,%xmm3 + pxor %xmm4,%xmm4 + pxor %xmm5,%xmm5 + + movdqa %xmm8,%xmm0 + pshufd $0,%xmm1,%xmm1 + movq $64,%rax + +.Lselect_loop_sse_w7: + movdqa %xmm8,%xmm15 + paddd %xmm0,%xmm8 + movdqa 0(%rsi),%xmm9 + movdqa 16(%rsi),%xmm10 + pcmpeqd %xmm1,%xmm15 + movdqa 32(%rsi),%xmm11 + movdqa 48(%rsi),%xmm12 + leaq 64(%rsi),%rsi + + pand %xmm15,%xmm9 + pand %xmm15,%xmm10 + por %xmm9,%xmm2 + pand %xmm15,%xmm11 + por %xmm10,%xmm3 + pand %xmm15,%xmm12 + por %xmm11,%xmm4 + prefetcht0 255(%rsi) + por %xmm12,%xmm5 + + decq %rax + jnz .Lselect_loop_sse_w7 + + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + movdqu %xmm4,32(%rdi) + movdqu %xmm5,48(%rdi) + ret +.cfi_endproc +.LSEH_end_ecp_nistz256_select_w7_nohw: +.size ecp_nistz256_select_w7_nohw,.-ecp_nistz256_select_w7_nohw + + +.globl ecp_nistz256_select_w5_avx2 +.hidden ecp_nistz256_select_w5_avx2 +.type ecp_nistz256_select_w5_avx2,@function +.align 32 +ecp_nistz256_select_w5_avx2: +.cfi_startproc +_CET_ENDBR + vzeroupper + vmovdqa .LTwo(%rip),%ymm0 + + vpxor %ymm2,%ymm2,%ymm2 + vpxor %ymm3,%ymm3,%ymm3 + vpxor %ymm4,%ymm4,%ymm4 + + vmovdqa .LOne(%rip),%ymm5 + vmovdqa .LTwo(%rip),%ymm10 + + vmovd %edx,%xmm1 + vpermd %ymm1,%ymm2,%ymm1 + + movq $8,%rax +.Lselect_loop_avx2_w5: + + vmovdqa 0(%rsi),%ymm6 + vmovdqa 32(%rsi),%ymm7 + vmovdqa 64(%rsi),%ymm8 + + vmovdqa 96(%rsi),%ymm11 + vmovdqa 128(%rsi),%ymm12 + vmovdqa 160(%rsi),%ymm13 + + vpcmpeqd %ymm1,%ymm5,%ymm9 + vpcmpeqd %ymm1,%ymm10,%ymm14 + + vpaddd %ymm0,%ymm5,%ymm5 + vpaddd %ymm0,%ymm10,%ymm10 + leaq 192(%rsi),%rsi + + vpand %ymm9,%ymm6,%ymm6 + vpand %ymm9,%ymm7,%ymm7 + vpand %ymm9,%ymm8,%ymm8 + vpand %ymm14,%ymm11,%ymm11 + vpand %ymm14,%ymm12,%ymm12 + vpand %ymm14,%ymm13,%ymm13 + + vpxor %ymm6,%ymm2,%ymm2 + vpxor %ymm7,%ymm3,%ymm3 + vpxor %ymm8,%ymm4,%ymm4 + vpxor %ymm11,%ymm2,%ymm2 + vpxor %ymm12,%ymm3,%ymm3 + vpxor %ymm13,%ymm4,%ymm4 + + decq %rax + jnz .Lselect_loop_avx2_w5 + + vmovdqu %ymm2,0(%rdi) + vmovdqu %ymm3,32(%rdi) + vmovdqu %ymm4,64(%rdi) + vzeroupper + ret +.cfi_endproc +.LSEH_end_ecp_nistz256_select_w5_avx2: +.size ecp_nistz256_select_w5_avx2,.-ecp_nistz256_select_w5_avx2 + + + +.globl ecp_nistz256_select_w7_avx2 +.hidden ecp_nistz256_select_w7_avx2 +.type ecp_nistz256_select_w7_avx2,@function +.align 32 +ecp_nistz256_select_w7_avx2: +.cfi_startproc +_CET_ENDBR + vzeroupper + vmovdqa .LThree(%rip),%ymm0 + + vpxor %ymm2,%ymm2,%ymm2 + vpxor %ymm3,%ymm3,%ymm3 + + vmovdqa .LOne(%rip),%ymm4 + vmovdqa .LTwo(%rip),%ymm8 + vmovdqa .LThree(%rip),%ymm12 + + vmovd %edx,%xmm1 + vpermd %ymm1,%ymm2,%ymm1 + + + movq $21,%rax +.Lselect_loop_avx2_w7: + + vmovdqa 0(%rsi),%ymm5 + vmovdqa 32(%rsi),%ymm6 + + vmovdqa 64(%rsi),%ymm9 + vmovdqa 96(%rsi),%ymm10 + + vmovdqa 128(%rsi),%ymm13 + vmovdqa 160(%rsi),%ymm14 + + vpcmpeqd %ymm1,%ymm4,%ymm7 + vpcmpeqd %ymm1,%ymm8,%ymm11 + vpcmpeqd %ymm1,%ymm12,%ymm15 + + vpaddd %ymm0,%ymm4,%ymm4 + vpaddd %ymm0,%ymm8,%ymm8 + vpaddd %ymm0,%ymm12,%ymm12 + leaq 192(%rsi),%rsi + + vpand %ymm7,%ymm5,%ymm5 + vpand %ymm7,%ymm6,%ymm6 + vpand %ymm11,%ymm9,%ymm9 + vpand %ymm11,%ymm10,%ymm10 + vpand %ymm15,%ymm13,%ymm13 + vpand %ymm15,%ymm14,%ymm14 + + vpxor %ymm5,%ymm2,%ymm2 + vpxor %ymm6,%ymm3,%ymm3 + vpxor %ymm9,%ymm2,%ymm2 + vpxor %ymm10,%ymm3,%ymm3 + vpxor %ymm13,%ymm2,%ymm2 + vpxor %ymm14,%ymm3,%ymm3 + + decq %rax + jnz .Lselect_loop_avx2_w7 + + + vmovdqa 0(%rsi),%ymm5 + vmovdqa 32(%rsi),%ymm6 + + vpcmpeqd %ymm1,%ymm4,%ymm7 + + vpand %ymm7,%ymm5,%ymm5 + vpand %ymm7,%ymm6,%ymm6 + + vpxor %ymm5,%ymm2,%ymm2 + vpxor %ymm6,%ymm3,%ymm3 + + vmovdqu %ymm2,0(%rdi) + vmovdqu %ymm3,32(%rdi) + vzeroupper + ret +.cfi_endproc +.LSEH_end_ecp_nistz256_select_w7_avx2: +.size ecp_nistz256_select_w7_avx2,.-ecp_nistz256_select_w7_avx2 +.type __ecp_nistz256_add_toq,@function +.align 32 +__ecp_nistz256_add_toq: +.cfi_startproc + xorq %r11,%r11 + addq 0(%rbx),%r12 + adcq 8(%rbx),%r13 + movq %r12,%rax + adcq 16(%rbx),%r8 + adcq 24(%rbx),%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq + +.type __ecp_nistz256_sub_fromq,@function +.align 32 +__ecp_nistz256_sub_fromq: +.cfi_startproc + subq 0(%rbx),%r12 + sbbq 8(%rbx),%r13 + movq %r12,%rax + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + movq %r13,%rbp + sbbq %r11,%r11 + + addq $-1,%r12 + movq %r8,%rcx + adcq %r14,%r13 + adcq $0,%r8 + movq %r9,%r10 + adcq %r15,%r9 + testq %r11,%r11 + + cmovzq %rax,%r12 + cmovzq %rbp,%r13 + movq %r12,0(%rdi) + cmovzq %rcx,%r8 + movq %r13,8(%rdi) + cmovzq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq + +.type __ecp_nistz256_subq,@function +.align 32 +__ecp_nistz256_subq: +.cfi_startproc + subq %r12,%rax + sbbq %r13,%rbp + movq %rax,%r12 + sbbq %r8,%rcx + sbbq %r9,%r10 + movq %rbp,%r13 + sbbq %r11,%r11 + + addq $-1,%rax + movq %rcx,%r8 + adcq %r14,%rbp + adcq $0,%rcx + movq %r10,%r9 + adcq %r15,%r10 + testq %r11,%r11 + + cmovnzq %rax,%r12 + cmovnzq %rbp,%r13 + cmovnzq %rcx,%r8 + cmovnzq %r10,%r9 + + ret +.cfi_endproc +.size __ecp_nistz256_subq,.-__ecp_nistz256_subq + +.type __ecp_nistz256_mul_by_2q,@function +.align 32 +__ecp_nistz256_mul_by_2q: +.cfi_startproc + xorq %r11,%r11 + addq %r12,%r12 + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q +.globl ecp_nistz256_point_double_nohw +.hidden ecp_nistz256_point_double_nohw +.type ecp_nistz256_point_double_nohw,@function +.align 32 +ecp_nistz256_point_double_nohw: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $160+8,%rsp +.cfi_adjust_cfa_offset 32*5+8 +.Lpoint_doubleq_body: + +.Lpoint_double_shortcutq: + movdqu 0(%rsi),%xmm0 + movq %rsi,%rbx + movdqu 16(%rsi),%xmm1 + movq 32+0(%rsi),%r12 + movq 32+8(%rsi),%r13 + movq 32+16(%rsi),%r8 + movq 32+24(%rsi),%r9 + movq .Lpoly+8(%rip),%r14 + movq .Lpoly+24(%rip),%r15 + movdqa %xmm0,96(%rsp) + movdqa %xmm1,96+16(%rsp) + leaq 32(%rdi),%r10 + leaq 64(%rdi),%r11 + movq %rdi,%xmm0 + movq %r10,%xmm1 + movq %r11,%xmm2 + + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_by_2q + + movq 64+0(%rsi),%rax + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + leaq 64-0(%rsi),%rsi + leaq 64(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 0+0(%rsp),%rax + movq 8+0(%rsp),%r14 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + leaq 0(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 32(%rbx),%rax + movq 64+0(%rbx),%r9 + movq 64+8(%rbx),%r10 + movq 64+16(%rbx),%r11 + movq 64+24(%rbx),%r12 + leaq 64-0(%rbx),%rsi + leaq 32(%rbx),%rbx + movq %xmm2,%rdi + call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_by_2q + + movq 96+0(%rsp),%r12 + movq 96+8(%rsp),%r13 + leaq 64(%rsp),%rbx + movq 96+16(%rsp),%r8 + movq 96+24(%rsp),%r9 + leaq 32(%rsp),%rdi + call __ecp_nistz256_add_toq + + movq 96+0(%rsp),%r12 + movq 96+8(%rsp),%r13 + leaq 64(%rsp),%rbx + movq 96+16(%rsp),%r8 + movq 96+24(%rsp),%r9 + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 0+0(%rsp),%rax + movq 8+0(%rsp),%r14 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + movq %xmm1,%rdi + call __ecp_nistz256_sqr_montq + xorq %r9,%r9 + movq %r12,%rax + addq $-1,%r12 + movq %r13,%r10 + adcq %rsi,%r13 + movq %r14,%rcx + adcq $0,%r14 + movq %r15,%r8 + adcq %rbp,%r15 + adcq $0,%r9 + xorq %rsi,%rsi + testq $1,%rax + + cmovzq %rax,%r12 + cmovzq %r10,%r13 + cmovzq %rcx,%r14 + cmovzq %r8,%r15 + cmovzq %rsi,%r9 + + movq %r13,%rax + shrq $1,%r12 + shlq $63,%rax + movq %r14,%r10 + shrq $1,%r13 + orq %rax,%r12 + shlq $63,%r10 + movq %r15,%rcx + shrq $1,%r14 + orq %r10,%r13 + shlq $63,%rcx + movq %r12,0(%rdi) + shrq $1,%r15 + movq %r13,8(%rdi) + shlq $63,%r9 + orq %rcx,%r14 + orq %r9,%r15 + movq %r14,16(%rdi) + movq %r15,24(%rdi) + movq 64(%rsp),%rax + leaq 64(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_by_2q + + leaq 32(%rsp),%rbx + leaq 32(%rsp),%rdi + call __ecp_nistz256_add_toq + + movq 96(%rsp),%rax + leaq 96(%rsp),%rbx + movq 0+0(%rsp),%r9 + movq 8+0(%rsp),%r10 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r11 + movq 24+0(%rsp),%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_by_2q + + movq 0+32(%rsp),%rax + movq 8+32(%rsp),%r14 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r15 + movq 24+32(%rsp),%r8 + movq %xmm0,%rdi + call __ecp_nistz256_sqr_montq + + leaq 128(%rsp),%rbx + movq %r14,%r8 + movq %r15,%r9 + movq %rsi,%r14 + movq %rbp,%r15 + call __ecp_nistz256_sub_fromq + + movq 0+0(%rsp),%rax + movq 0+8(%rsp),%rbp + movq 0+16(%rsp),%rcx + movq 0+24(%rsp),%r10 + leaq 0(%rsp),%rdi + call __ecp_nistz256_subq + + movq 32(%rsp),%rax + leaq 32(%rsp),%rbx + movq %r12,%r14 + xorl %ecx,%ecx + movq %r12,0+0(%rsp) + movq %r13,%r10 + movq %r13,0+8(%rsp) + cmovzq %r8,%r11 + movq %r8,0+16(%rsp) + leaq 0-0(%rsp),%rsi + cmovzq %r9,%r12 + movq %r9,0+24(%rsp) + movq %r14,%r9 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq %xmm1,%rbx + movq %xmm1,%rdi + call __ecp_nistz256_sub_fromq + + leaq 160+56(%rsp),%rsi +.cfi_def_cfa %rsi,8 + movq -48(%rsi),%r15 +.cfi_restore %r15 + movq -40(%rsi),%r14 +.cfi_restore %r14 + movq -32(%rsi),%r13 +.cfi_restore %r13 + movq -24(%rsi),%r12 +.cfi_restore %r12 + movq -16(%rsi),%rbx +.cfi_restore %rbx + movq -8(%rsi),%rbp +.cfi_restore %rbp + leaq (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Lpoint_doubleq_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_double_nohw,.-ecp_nistz256_point_double_nohw +.globl ecp_nistz256_point_add_nohw +.hidden ecp_nistz256_point_add_nohw +.type ecp_nistz256_point_add_nohw,@function +.align 32 +ecp_nistz256_point_add_nohw: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $576+8,%rsp +.cfi_adjust_cfa_offset 32*18+8 +.Lpoint_addq_body: + + movdqu 0(%rsi),%xmm0 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + movdqu 64(%rsi),%xmm4 + movdqu 80(%rsi),%xmm5 + movq %rsi,%rbx + movq %rdx,%rsi + movdqa %xmm0,384(%rsp) + movdqa %xmm1,384+16(%rsp) + movdqa %xmm2,416(%rsp) + movdqa %xmm3,416+16(%rsp) + movdqa %xmm4,448(%rsp) + movdqa %xmm5,448+16(%rsp) + por %xmm4,%xmm5 + + movdqu 0(%rsi),%xmm0 + pshufd $0xb1,%xmm5,%xmm3 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + por %xmm3,%xmm5 + movdqu 48(%rsi),%xmm3 + movq 64+0(%rsi),%rax + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + movdqa %xmm0,480(%rsp) + pshufd $0x1e,%xmm5,%xmm4 + movdqa %xmm1,480+16(%rsp) + movdqu 64(%rsi),%xmm0 + movdqu 80(%rsi),%xmm1 + movdqa %xmm2,512(%rsp) + movdqa %xmm3,512+16(%rsp) + por %xmm4,%xmm5 + pxor %xmm4,%xmm4 + por %xmm0,%xmm1 + movq %rdi,%xmm0 + + leaq 64-0(%rsi),%rsi + movq %rax,544+0(%rsp) + movq %r14,544+8(%rsp) + movq %r15,544+16(%rsp) + movq %r8,544+24(%rsp) + leaq 96(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + pcmpeqd %xmm4,%xmm5 + pshufd $0xb1,%xmm1,%xmm4 + por %xmm1,%xmm4 + pshufd $0,%xmm5,%xmm5 + pshufd $0x1e,%xmm4,%xmm3 + por %xmm3,%xmm4 + pxor %xmm3,%xmm3 + pcmpeqd %xmm3,%xmm4 + pshufd $0,%xmm4,%xmm4 + movq 64+0(%rbx),%rax + movq 64+8(%rbx),%r14 + movq 64+16(%rbx),%r15 + movq 64+24(%rbx),%r8 + movq %rbx,%xmm1 + + leaq 64-0(%rbx),%rsi + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 544(%rsp),%rax + leaq 544(%rsp),%rbx + movq 0+96(%rsp),%r9 + movq 8+96(%rsp),%r10 + leaq 0+96(%rsp),%rsi + movq 16+96(%rsp),%r11 + movq 24+96(%rsp),%r12 + leaq 224(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 448(%rsp),%rax + leaq 448(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 416(%rsp),%rax + leaq 416(%rsp),%rbx + movq 0+224(%rsp),%r9 + movq 8+224(%rsp),%r10 + leaq 0+224(%rsp),%rsi + movq 16+224(%rsp),%r11 + movq 24+224(%rsp),%r12 + leaq 224(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 512(%rsp),%rax + leaq 512(%rsp),%rbx + movq 0+256(%rsp),%r9 + movq 8+256(%rsp),%r10 + leaq 0+256(%rsp),%rsi + movq 16+256(%rsp),%r11 + movq 24+256(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 224(%rsp),%rbx + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + orq %r13,%r12 + movdqa %xmm4,%xmm2 + orq %r8,%r12 + orq %r9,%r12 + por %xmm5,%xmm2 + movq %r12,%xmm3 + + movq 384(%rsp),%rax + leaq 384(%rsp),%rbx + movq 0+96(%rsp),%r9 + movq 8+96(%rsp),%r10 + leaq 0+96(%rsp),%rsi + movq 16+96(%rsp),%r11 + movq 24+96(%rsp),%r12 + leaq 160(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 480(%rsp),%rax + leaq 480(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 192(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 160(%rsp),%rbx + leaq 0(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + orq %r13,%r12 + orq %r8,%r12 + orq %r9,%r12 + + movq %xmm2,%r8 + movq %xmm3,%r9 + orq %r8,%r12 +.byte 0x3e + jnz .Ladd_proceedq + + + + testq %r9,%r9 + jz .Ladd_doubleq + + + + + + + movq %xmm0,%rdi + pxor %xmm0,%xmm0 + movdqu %xmm0,0(%rdi) + movdqu %xmm0,16(%rdi) + movdqu %xmm0,32(%rdi) + movdqu %xmm0,48(%rdi) + movdqu %xmm0,64(%rdi) + movdqu %xmm0,80(%rdi) + jmp .Ladd_doneq + +.align 32 +.Ladd_doubleq: + movq %xmm1,%rsi + movq %xmm0,%rdi + addq $416,%rsp +.cfi_adjust_cfa_offset -416 + jmp .Lpoint_double_shortcutq +.cfi_adjust_cfa_offset 416 + +.align 32 +.Ladd_proceedq: + movq 0+64(%rsp),%rax + movq 8+64(%rsp),%r14 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r15 + movq 24+64(%rsp),%r8 + leaq 96(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 448(%rsp),%rax + leaq 448(%rsp),%rbx + movq 0+0(%rsp),%r9 + movq 8+0(%rsp),%r10 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r11 + movq 24+0(%rsp),%r12 + leaq 352(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 0+0(%rsp),%rax + movq 8+0(%rsp),%r14 + leaq 0+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 544(%rsp),%rax + leaq 544(%rsp),%rbx + movq 0+352(%rsp),%r9 + movq 8+352(%rsp),%r10 + leaq 0+352(%rsp),%rsi + movq 16+352(%rsp),%r11 + movq 24+352(%rsp),%r12 + leaq 352(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 0(%rsp),%rax + leaq 0(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 160(%rsp),%rax + leaq 160(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 192(%rsp),%rdi + call __ecp_nistz256_mul_montq + + + + + xorq %r11,%r11 + addq %r12,%r12 + leaq 96(%rsp),%rsi + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + movq 0(%rsi),%rax + cmovcq %rbp,%r13 + movq 8(%rsi),%rbp + cmovcq %rcx,%r8 + movq 16(%rsi),%rcx + cmovcq %r10,%r9 + movq 24(%rsi),%r10 + + call __ecp_nistz256_subq + + leaq 128(%rsp),%rbx + leaq 288(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 192+0(%rsp),%rax + movq 192+8(%rsp),%rbp + movq 192+16(%rsp),%rcx + movq 192+24(%rsp),%r10 + leaq 320(%rsp),%rdi + + call __ecp_nistz256_subq + + movq %r12,0(%rdi) + movq %r13,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq 128(%rsp),%rax + leaq 128(%rsp),%rbx + movq 0+224(%rsp),%r9 + movq 8+224(%rsp),%r10 + leaq 0+224(%rsp),%rsi + movq 16+224(%rsp),%r11 + movq 24+224(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 320(%rsp),%rax + leaq 320(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 320(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 256(%rsp),%rbx + leaq 320(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq %xmm0,%rdi + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 352(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 352+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 544(%rsp),%xmm2 + pand 544+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 448(%rsp),%xmm2 + pand 448+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,64(%rdi) + movdqu %xmm3,80(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 288(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 288+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 480(%rsp),%xmm2 + pand 480+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 384(%rsp),%xmm2 + pand 384+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 320(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 320+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 512(%rsp),%xmm2 + pand 512+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 416(%rsp),%xmm2 + pand 416+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + +.Ladd_doneq: + leaq 576+56(%rsp),%rsi +.cfi_def_cfa %rsi,8 + movq -48(%rsi),%r15 +.cfi_restore %r15 + movq -40(%rsi),%r14 +.cfi_restore %r14 + movq -32(%rsi),%r13 +.cfi_restore %r13 + movq -24(%rsi),%r12 +.cfi_restore %r12 + movq -16(%rsi),%rbx +.cfi_restore %rbx + movq -8(%rsi),%rbp +.cfi_restore %rbp + leaq (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Lpoint_addq_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_add_nohw,.-ecp_nistz256_point_add_nohw +.globl ecp_nistz256_point_add_affine_nohw +.hidden ecp_nistz256_point_add_affine_nohw +.type ecp_nistz256_point_add_affine_nohw,@function +.align 32 +ecp_nistz256_point_add_affine_nohw: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $480+8,%rsp +.cfi_adjust_cfa_offset 32*15+8 +.Ladd_affineq_body: + + movdqu 0(%rsi),%xmm0 + movq %rdx,%rbx + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + movdqu 64(%rsi),%xmm4 + movdqu 80(%rsi),%xmm5 + movq 64+0(%rsi),%rax + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + movdqa %xmm0,320(%rsp) + movdqa %xmm1,320+16(%rsp) + movdqa %xmm2,352(%rsp) + movdqa %xmm3,352+16(%rsp) + movdqa %xmm4,384(%rsp) + movdqa %xmm5,384+16(%rsp) + por %xmm4,%xmm5 + + movdqu 0(%rbx),%xmm0 + pshufd $0xb1,%xmm5,%xmm3 + movdqu 16(%rbx),%xmm1 + movdqu 32(%rbx),%xmm2 + por %xmm3,%xmm5 + movdqu 48(%rbx),%xmm3 + movdqa %xmm0,416(%rsp) + pshufd $0x1e,%xmm5,%xmm4 + movdqa %xmm1,416+16(%rsp) + por %xmm0,%xmm1 + movq %rdi,%xmm0 + movdqa %xmm2,448(%rsp) + movdqa %xmm3,448+16(%rsp) + por %xmm2,%xmm3 + por %xmm4,%xmm5 + pxor %xmm4,%xmm4 + por %xmm1,%xmm3 + + leaq 64-0(%rsi),%rsi + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + pcmpeqd %xmm4,%xmm5 + pshufd $0xb1,%xmm3,%xmm4 + movq 0(%rbx),%rax + + movq %r12,%r9 + por %xmm3,%xmm4 + pshufd $0,%xmm5,%xmm5 + pshufd $0x1e,%xmm4,%xmm3 + movq %r13,%r10 + por %xmm3,%xmm4 + pxor %xmm3,%xmm3 + movq %r14,%r11 + pcmpeqd %xmm3,%xmm4 + pshufd $0,%xmm4,%xmm4 + + leaq 32-0(%rsp),%rsi + movq %r15,%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 320(%rsp),%rbx + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 384(%rsp),%rax + leaq 384(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 384(%rsp),%rax + leaq 384(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 288(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 448(%rsp),%rax + leaq 448(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq 0+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 352(%rsp),%rbx + leaq 96(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 0+64(%rsp),%rax + movq 8+64(%rsp),%r14 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r15 + movq 24+64(%rsp),%r8 + leaq 128(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 0+96(%rsp),%rax + movq 8+96(%rsp),%r14 + leaq 0+96(%rsp),%rsi + movq 16+96(%rsp),%r15 + movq 24+96(%rsp),%r8 + leaq 192(%rsp),%rdi + call __ecp_nistz256_sqr_montq + + movq 128(%rsp),%rax + leaq 128(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 160(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 320(%rsp),%rax + leaq 320(%rsp),%rbx + movq 0+128(%rsp),%r9 + movq 8+128(%rsp),%r10 + leaq 0+128(%rsp),%rsi + movq 16+128(%rsp),%r11 + movq 24+128(%rsp),%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montq + + + + + xorq %r11,%r11 + addq %r12,%r12 + leaq 192(%rsp),%rsi + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + movq 0(%rsi),%rax + cmovcq %rbp,%r13 + movq 8(%rsi),%rbp + cmovcq %rcx,%r8 + movq 16(%rsi),%rcx + cmovcq %r10,%r9 + movq 24(%rsi),%r10 + + call __ecp_nistz256_subq + + leaq 160(%rsp),%rbx + leaq 224(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq 0+0(%rsp),%rax + movq 0+8(%rsp),%rbp + movq 0+16(%rsp),%rcx + movq 0+24(%rsp),%r10 + leaq 64(%rsp),%rdi + + call __ecp_nistz256_subq + + movq %r12,0(%rdi) + movq %r13,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq 352(%rsp),%rax + leaq 352(%rsp),%rbx + movq 0+160(%rsp),%r9 + movq 8+160(%rsp),%r10 + leaq 0+160(%rsp),%rsi + movq 16+160(%rsp),%r11 + movq 24+160(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montq + + movq 96(%rsp),%rax + leaq 96(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq 0+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 64(%rsp),%rdi + call __ecp_nistz256_mul_montq + + leaq 32(%rsp),%rbx + leaq 256(%rsp),%rdi + call __ecp_nistz256_sub_fromq + + movq %xmm0,%rdi + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 288(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 288+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand .LONE_mont(%rip),%xmm2 + pand .LONE_mont+16(%rip),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 384(%rsp),%xmm2 + pand 384+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,64(%rdi) + movdqu %xmm3,80(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 224(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 224+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 416(%rsp),%xmm2 + pand 416+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 320(%rsp),%xmm2 + pand 320+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 256(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 256+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 448(%rsp),%xmm2 + pand 448+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 352(%rsp),%xmm2 + pand 352+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + + leaq 480+56(%rsp),%rsi +.cfi_def_cfa %rsi,8 + movq -48(%rsi),%r15 +.cfi_restore %r15 + movq -40(%rsi),%r14 +.cfi_restore %r14 + movq -32(%rsi),%r13 +.cfi_restore %r13 + movq -24(%rsi),%r12 +.cfi_restore %r12 + movq -16(%rsi),%rbx +.cfi_restore %rbx + movq -8(%rsi),%rbp +.cfi_restore %rbp + leaq (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Ladd_affineq_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_add_affine_nohw,.-ecp_nistz256_point_add_affine_nohw +.type __ecp_nistz256_add_tox,@function +.align 32 +__ecp_nistz256_add_tox: +.cfi_startproc + xorq %r11,%r11 + adcq 0(%rbx),%r12 + adcq 8(%rbx),%r13 + movq %r12,%rax + adcq 16(%rbx),%r8 + adcq 24(%rbx),%r9 + movq %r13,%rbp + adcq $0,%r11 + + xorq %r10,%r10 + sbbq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox + +.type __ecp_nistz256_sub_fromx,@function +.align 32 +__ecp_nistz256_sub_fromx: +.cfi_startproc + xorq %r11,%r11 + sbbq 0(%rbx),%r12 + sbbq 8(%rbx),%r13 + movq %r12,%rax + sbbq 16(%rbx),%r8 + sbbq 24(%rbx),%r9 + movq %r13,%rbp + sbbq $0,%r11 + + xorq %r10,%r10 + adcq $-1,%r12 + movq %r8,%rcx + adcq %r14,%r13 + adcq $0,%r8 + movq %r9,%r10 + adcq %r15,%r9 + + btq $0,%r11 + cmovncq %rax,%r12 + cmovncq %rbp,%r13 + movq %r12,0(%rdi) + cmovncq %rcx,%r8 + movq %r13,8(%rdi) + cmovncq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx + +.type __ecp_nistz256_subx,@function +.align 32 +__ecp_nistz256_subx: +.cfi_startproc + xorq %r11,%r11 + sbbq %r12,%rax + sbbq %r13,%rbp + movq %rax,%r12 + sbbq %r8,%rcx + sbbq %r9,%r10 + movq %rbp,%r13 + sbbq $0,%r11 + + xorq %r9,%r9 + adcq $-1,%rax + movq %rcx,%r8 + adcq %r14,%rbp + adcq $0,%rcx + movq %r10,%r9 + adcq %r15,%r10 + + btq $0,%r11 + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + cmovcq %rcx,%r8 + cmovcq %r10,%r9 + + ret +.cfi_endproc +.size __ecp_nistz256_subx,.-__ecp_nistz256_subx + +.type __ecp_nistz256_mul_by_2x,@function +.align 32 +__ecp_nistz256_mul_by_2x: +.cfi_startproc + xorq %r11,%r11 + adcq %r12,%r12 + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + xorq %r10,%r10 + sbbq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + cmovcq %rbp,%r13 + movq %r12,0(%rdi) + cmovcq %rcx,%r8 + movq %r13,8(%rdi) + cmovcq %r10,%r9 + movq %r8,16(%rdi) + movq %r9,24(%rdi) + + ret +.cfi_endproc +.size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x +.globl ecp_nistz256_point_double_adx +.hidden ecp_nistz256_point_double_adx +.type ecp_nistz256_point_double_adx,@function +.align 32 +ecp_nistz256_point_double_adx: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $160+8,%rsp +.cfi_adjust_cfa_offset 32*5+8 +.Lpoint_doublex_body: + +.Lpoint_double_shortcutx: + movdqu 0(%rsi),%xmm0 + movq %rsi,%rbx + movdqu 16(%rsi),%xmm1 + movq 32+0(%rsi),%r12 + movq 32+8(%rsi),%r13 + movq 32+16(%rsi),%r8 + movq 32+24(%rsi),%r9 + movq .Lpoly+8(%rip),%r14 + movq .Lpoly+24(%rip),%r15 + movdqa %xmm0,96(%rsp) + movdqa %xmm1,96+16(%rsp) + leaq 32(%rdi),%r10 + leaq 64(%rdi),%r11 + movq %rdi,%xmm0 + movq %r10,%xmm1 + movq %r11,%xmm2 + + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_by_2x + + movq 64+0(%rsi),%rdx + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + leaq 64-128(%rsi),%rsi + leaq 64(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 0+0(%rsp),%rdx + movq 8+0(%rsp),%r14 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + leaq 0(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 32(%rbx),%rdx + movq 64+0(%rbx),%r9 + movq 64+8(%rbx),%r10 + movq 64+16(%rbx),%r11 + movq 64+24(%rbx),%r12 + leaq 64-128(%rbx),%rsi + leaq 32(%rbx),%rbx + movq %xmm2,%rdi + call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_by_2x + + movq 96+0(%rsp),%r12 + movq 96+8(%rsp),%r13 + leaq 64(%rsp),%rbx + movq 96+16(%rsp),%r8 + movq 96+24(%rsp),%r9 + leaq 32(%rsp),%rdi + call __ecp_nistz256_add_tox + + movq 96+0(%rsp),%r12 + movq 96+8(%rsp),%r13 + leaq 64(%rsp),%rbx + movq 96+16(%rsp),%r8 + movq 96+24(%rsp),%r9 + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 0+0(%rsp),%rdx + movq 8+0(%rsp),%r14 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + movq %xmm1,%rdi + call __ecp_nistz256_sqr_montx + xorq %r9,%r9 + movq %r12,%rax + addq $-1,%r12 + movq %r13,%r10 + adcq %rsi,%r13 + movq %r14,%rcx + adcq $0,%r14 + movq %r15,%r8 + adcq %rbp,%r15 + adcq $0,%r9 + xorq %rsi,%rsi + testq $1,%rax + + cmovzq %rax,%r12 + cmovzq %r10,%r13 + cmovzq %rcx,%r14 + cmovzq %r8,%r15 + cmovzq %rsi,%r9 + + movq %r13,%rax + shrq $1,%r12 + shlq $63,%rax + movq %r14,%r10 + shrq $1,%r13 + orq %rax,%r12 + shlq $63,%r10 + movq %r15,%rcx + shrq $1,%r14 + orq %r10,%r13 + shlq $63,%rcx + movq %r12,0(%rdi) + shrq $1,%r15 + movq %r13,8(%rdi) + shlq $63,%r9 + orq %rcx,%r14 + orq %r9,%r15 + movq %r14,16(%rdi) + movq %r15,24(%rdi) + movq 64(%rsp),%rdx + leaq 64(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_by_2x + + leaq 32(%rsp),%rbx + leaq 32(%rsp),%rdi + call __ecp_nistz256_add_tox + + movq 96(%rsp),%rdx + leaq 96(%rsp),%rbx + movq 0+0(%rsp),%r9 + movq 8+0(%rsp),%r10 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r11 + movq 24+0(%rsp),%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_by_2x + + movq 0+32(%rsp),%rdx + movq 8+32(%rsp),%r14 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r15 + movq 24+32(%rsp),%r8 + movq %xmm0,%rdi + call __ecp_nistz256_sqr_montx + + leaq 128(%rsp),%rbx + movq %r14,%r8 + movq %r15,%r9 + movq %rsi,%r14 + movq %rbp,%r15 + call __ecp_nistz256_sub_fromx + + movq 0+0(%rsp),%rax + movq 0+8(%rsp),%rbp + movq 0+16(%rsp),%rcx + movq 0+24(%rsp),%r10 + leaq 0(%rsp),%rdi + call __ecp_nistz256_subx + + movq 32(%rsp),%rdx + leaq 32(%rsp),%rbx + movq %r12,%r14 + xorl %ecx,%ecx + movq %r12,0+0(%rsp) + movq %r13,%r10 + movq %r13,0+8(%rsp) + cmovzq %r8,%r11 + movq %r8,0+16(%rsp) + leaq 0-128(%rsp),%rsi + cmovzq %r9,%r12 + movq %r9,0+24(%rsp) + movq %r14,%r9 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq %xmm1,%rbx + movq %xmm1,%rdi + call __ecp_nistz256_sub_fromx + + leaq 160+56(%rsp),%rsi +.cfi_def_cfa %rsi,8 + movq -48(%rsi),%r15 +.cfi_restore %r15 + movq -40(%rsi),%r14 +.cfi_restore %r14 + movq -32(%rsi),%r13 +.cfi_restore %r13 + movq -24(%rsi),%r12 +.cfi_restore %r12 + movq -16(%rsi),%rbx +.cfi_restore %rbx + movq -8(%rsi),%rbp +.cfi_restore %rbp + leaq (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Lpoint_doublex_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_double_adx,.-ecp_nistz256_point_double_adx +.globl ecp_nistz256_point_add_adx +.hidden ecp_nistz256_point_add_adx +.type ecp_nistz256_point_add_adx,@function +.align 32 +ecp_nistz256_point_add_adx: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $576+8,%rsp +.cfi_adjust_cfa_offset 32*18+8 +.Lpoint_addx_body: + + movdqu 0(%rsi),%xmm0 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + movdqu 64(%rsi),%xmm4 + movdqu 80(%rsi),%xmm5 + movq %rsi,%rbx + movq %rdx,%rsi + movdqa %xmm0,384(%rsp) + movdqa %xmm1,384+16(%rsp) + movdqa %xmm2,416(%rsp) + movdqa %xmm3,416+16(%rsp) + movdqa %xmm4,448(%rsp) + movdqa %xmm5,448+16(%rsp) + por %xmm4,%xmm5 + + movdqu 0(%rsi),%xmm0 + pshufd $0xb1,%xmm5,%xmm3 + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + por %xmm3,%xmm5 + movdqu 48(%rsi),%xmm3 + movq 64+0(%rsi),%rdx + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + movdqa %xmm0,480(%rsp) + pshufd $0x1e,%xmm5,%xmm4 + movdqa %xmm1,480+16(%rsp) + movdqu 64(%rsi),%xmm0 + movdqu 80(%rsi),%xmm1 + movdqa %xmm2,512(%rsp) + movdqa %xmm3,512+16(%rsp) + por %xmm4,%xmm5 + pxor %xmm4,%xmm4 + por %xmm0,%xmm1 + movq %rdi,%xmm0 + + leaq 64-128(%rsi),%rsi + movq %rdx,544+0(%rsp) + movq %r14,544+8(%rsp) + movq %r15,544+16(%rsp) + movq %r8,544+24(%rsp) + leaq 96(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + pcmpeqd %xmm4,%xmm5 + pshufd $0xb1,%xmm1,%xmm4 + por %xmm1,%xmm4 + pshufd $0,%xmm5,%xmm5 + pshufd $0x1e,%xmm4,%xmm3 + por %xmm3,%xmm4 + pxor %xmm3,%xmm3 + pcmpeqd %xmm3,%xmm4 + pshufd $0,%xmm4,%xmm4 + movq 64+0(%rbx),%rdx + movq 64+8(%rbx),%r14 + movq 64+16(%rbx),%r15 + movq 64+24(%rbx),%r8 + movq %rbx,%xmm1 + + leaq 64-128(%rbx),%rsi + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 544(%rsp),%rdx + leaq 544(%rsp),%rbx + movq 0+96(%rsp),%r9 + movq 8+96(%rsp),%r10 + leaq -128+96(%rsp),%rsi + movq 16+96(%rsp),%r11 + movq 24+96(%rsp),%r12 + leaq 224(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 448(%rsp),%rdx + leaq 448(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 416(%rsp),%rdx + leaq 416(%rsp),%rbx + movq 0+224(%rsp),%r9 + movq 8+224(%rsp),%r10 + leaq -128+224(%rsp),%rsi + movq 16+224(%rsp),%r11 + movq 24+224(%rsp),%r12 + leaq 224(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 512(%rsp),%rdx + leaq 512(%rsp),%rbx + movq 0+256(%rsp),%r9 + movq 8+256(%rsp),%r10 + leaq -128+256(%rsp),%rsi + movq 16+256(%rsp),%r11 + movq 24+256(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 224(%rsp),%rbx + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + orq %r13,%r12 + movdqa %xmm4,%xmm2 + orq %r8,%r12 + orq %r9,%r12 + por %xmm5,%xmm2 + movq %r12,%xmm3 + + movq 384(%rsp),%rdx + leaq 384(%rsp),%rbx + movq 0+96(%rsp),%r9 + movq 8+96(%rsp),%r10 + leaq -128+96(%rsp),%rsi + movq 16+96(%rsp),%r11 + movq 24+96(%rsp),%r12 + leaq 160(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 480(%rsp),%rdx + leaq 480(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 192(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 160(%rsp),%rbx + leaq 0(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + orq %r13,%r12 + orq %r8,%r12 + orq %r9,%r12 + + movq %xmm2,%r8 + movq %xmm3,%r9 + orq %r8,%r12 +.byte 0x3e + jnz .Ladd_proceedx + + + + testq %r9,%r9 + jz .Ladd_doublex + + + + + + + movq %xmm0,%rdi + pxor %xmm0,%xmm0 + movdqu %xmm0,0(%rdi) + movdqu %xmm0,16(%rdi) + movdqu %xmm0,32(%rdi) + movdqu %xmm0,48(%rdi) + movdqu %xmm0,64(%rdi) + movdqu %xmm0,80(%rdi) + jmp .Ladd_donex + +.align 32 +.Ladd_doublex: + movq %xmm1,%rsi + movq %xmm0,%rdi + addq $416,%rsp +.cfi_adjust_cfa_offset -416 + jmp .Lpoint_double_shortcutx +.cfi_adjust_cfa_offset 416 + +.align 32 +.Ladd_proceedx: + movq 0+64(%rsp),%rdx + movq 8+64(%rsp),%r14 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r15 + movq 24+64(%rsp),%r8 + leaq 96(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 448(%rsp),%rdx + leaq 448(%rsp),%rbx + movq 0+0(%rsp),%r9 + movq 8+0(%rsp),%r10 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r11 + movq 24+0(%rsp),%r12 + leaq 352(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 0+0(%rsp),%rdx + movq 8+0(%rsp),%r14 + leaq -128+0(%rsp),%rsi + movq 16+0(%rsp),%r15 + movq 24+0(%rsp),%r8 + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 544(%rsp),%rdx + leaq 544(%rsp),%rbx + movq 0+352(%rsp),%r9 + movq 8+352(%rsp),%r10 + leaq -128+352(%rsp),%rsi + movq 16+352(%rsp),%r11 + movq 24+352(%rsp),%r12 + leaq 352(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 0(%rsp),%rdx + leaq 0(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 128(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 160(%rsp),%rdx + leaq 160(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 192(%rsp),%rdi + call __ecp_nistz256_mul_montx + + + + + xorq %r11,%r11 + addq %r12,%r12 + leaq 96(%rsp),%rsi + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + movq 0(%rsi),%rax + cmovcq %rbp,%r13 + movq 8(%rsi),%rbp + cmovcq %rcx,%r8 + movq 16(%rsi),%rcx + cmovcq %r10,%r9 + movq 24(%rsi),%r10 + + call __ecp_nistz256_subx + + leaq 128(%rsp),%rbx + leaq 288(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 192+0(%rsp),%rax + movq 192+8(%rsp),%rbp + movq 192+16(%rsp),%rcx + movq 192+24(%rsp),%r10 + leaq 320(%rsp),%rdi + + call __ecp_nistz256_subx + + movq %r12,0(%rdi) + movq %r13,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq 128(%rsp),%rdx + leaq 128(%rsp),%rbx + movq 0+224(%rsp),%r9 + movq 8+224(%rsp),%r10 + leaq -128+224(%rsp),%rsi + movq 16+224(%rsp),%r11 + movq 24+224(%rsp),%r12 + leaq 256(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 320(%rsp),%rdx + leaq 320(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 320(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 256(%rsp),%rbx + leaq 320(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq %xmm0,%rdi + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 352(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 352+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 544(%rsp),%xmm2 + pand 544+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 448(%rsp),%xmm2 + pand 448+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,64(%rdi) + movdqu %xmm3,80(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 288(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 288+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 480(%rsp),%xmm2 + pand 480+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 384(%rsp),%xmm2 + pand 384+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 320(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 320+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 512(%rsp),%xmm2 + pand 512+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 416(%rsp),%xmm2 + pand 416+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + +.Ladd_donex: + leaq 576+56(%rsp),%rsi +.cfi_def_cfa %rsi,8 + movq -48(%rsi),%r15 +.cfi_restore %r15 + movq -40(%rsi),%r14 +.cfi_restore %r14 + movq -32(%rsi),%r13 +.cfi_restore %r13 + movq -24(%rsi),%r12 +.cfi_restore %r12 + movq -16(%rsi),%rbx +.cfi_restore %rbx + movq -8(%rsi),%rbp +.cfi_restore %rbp + leaq (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Lpoint_addx_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_add_adx,.-ecp_nistz256_point_add_adx +.globl ecp_nistz256_point_add_affine_adx +.hidden ecp_nistz256_point_add_affine_adx +.type ecp_nistz256_point_add_affine_adx,@function +.align 32 +ecp_nistz256_point_add_affine_adx: +.cfi_startproc +_CET_ENDBR + pushq %rbp +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbp,-16 + pushq %rbx +.cfi_adjust_cfa_offset 8 +.cfi_offset %rbx,-24 + pushq %r12 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r12,-32 + pushq %r13 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r13,-40 + pushq %r14 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r14,-48 + pushq %r15 +.cfi_adjust_cfa_offset 8 +.cfi_offset %r15,-56 + subq $480+8,%rsp +.cfi_adjust_cfa_offset 32*15+8 +.Ladd_affinex_body: + + movdqu 0(%rsi),%xmm0 + movq %rdx,%rbx + movdqu 16(%rsi),%xmm1 + movdqu 32(%rsi),%xmm2 + movdqu 48(%rsi),%xmm3 + movdqu 64(%rsi),%xmm4 + movdqu 80(%rsi),%xmm5 + movq 64+0(%rsi),%rdx + movq 64+8(%rsi),%r14 + movq 64+16(%rsi),%r15 + movq 64+24(%rsi),%r8 + movdqa %xmm0,320(%rsp) + movdqa %xmm1,320+16(%rsp) + movdqa %xmm2,352(%rsp) + movdqa %xmm3,352+16(%rsp) + movdqa %xmm4,384(%rsp) + movdqa %xmm5,384+16(%rsp) + por %xmm4,%xmm5 + + movdqu 0(%rbx),%xmm0 + pshufd $0xb1,%xmm5,%xmm3 + movdqu 16(%rbx),%xmm1 + movdqu 32(%rbx),%xmm2 + por %xmm3,%xmm5 + movdqu 48(%rbx),%xmm3 + movdqa %xmm0,416(%rsp) + pshufd $0x1e,%xmm5,%xmm4 + movdqa %xmm1,416+16(%rsp) + por %xmm0,%xmm1 + movq %rdi,%xmm0 + movdqa %xmm2,448(%rsp) + movdqa %xmm3,448+16(%rsp) + por %xmm2,%xmm3 + por %xmm4,%xmm5 + pxor %xmm4,%xmm4 + por %xmm1,%xmm3 + + leaq 64-128(%rsi),%rsi + leaq 32(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + pcmpeqd %xmm4,%xmm5 + pshufd $0xb1,%xmm3,%xmm4 + movq 0(%rbx),%rdx + + movq %r12,%r9 + por %xmm3,%xmm4 + pshufd $0,%xmm5,%xmm5 + pshufd $0x1e,%xmm4,%xmm3 + movq %r13,%r10 + por %xmm3,%xmm4 + pxor %xmm3,%xmm3 + movq %r14,%r11 + pcmpeqd %xmm3,%xmm4 + pshufd $0,%xmm4,%xmm4 + + leaq 32-128(%rsp),%rsi + movq %r15,%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 320(%rsp),%rbx + leaq 64(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 384(%rsp),%rdx + leaq 384(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 384(%rsp),%rdx + leaq 384(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 288(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 448(%rsp),%rdx + leaq 448(%rsp),%rbx + movq 0+32(%rsp),%r9 + movq 8+32(%rsp),%r10 + leaq -128+32(%rsp),%rsi + movq 16+32(%rsp),%r11 + movq 24+32(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 352(%rsp),%rbx + leaq 96(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 0+64(%rsp),%rdx + movq 8+64(%rsp),%r14 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r15 + movq 24+64(%rsp),%r8 + leaq 128(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 0+96(%rsp),%rdx + movq 8+96(%rsp),%r14 + leaq -128+96(%rsp),%rsi + movq 16+96(%rsp),%r15 + movq 24+96(%rsp),%r8 + leaq 192(%rsp),%rdi + call __ecp_nistz256_sqr_montx + + movq 128(%rsp),%rdx + leaq 128(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 160(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 320(%rsp),%rdx + leaq 320(%rsp),%rbx + movq 0+128(%rsp),%r9 + movq 8+128(%rsp),%r10 + leaq -128+128(%rsp),%rsi + movq 16+128(%rsp),%r11 + movq 24+128(%rsp),%r12 + leaq 0(%rsp),%rdi + call __ecp_nistz256_mul_montx + + + + + xorq %r11,%r11 + addq %r12,%r12 + leaq 192(%rsp),%rsi + adcq %r13,%r13 + movq %r12,%rax + adcq %r8,%r8 + adcq %r9,%r9 + movq %r13,%rbp + adcq $0,%r11 + + subq $-1,%r12 + movq %r8,%rcx + sbbq %r14,%r13 + sbbq $0,%r8 + movq %r9,%r10 + sbbq %r15,%r9 + sbbq $0,%r11 + + cmovcq %rax,%r12 + movq 0(%rsi),%rax + cmovcq %rbp,%r13 + movq 8(%rsi),%rbp + cmovcq %rcx,%r8 + movq 16(%rsi),%rcx + cmovcq %r10,%r9 + movq 24(%rsi),%r10 + + call __ecp_nistz256_subx + + leaq 160(%rsp),%rbx + leaq 224(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq 0+0(%rsp),%rax + movq 0+8(%rsp),%rbp + movq 0+16(%rsp),%rcx + movq 0+24(%rsp),%r10 + leaq 64(%rsp),%rdi + + call __ecp_nistz256_subx + + movq %r12,0(%rdi) + movq %r13,8(%rdi) + movq %r8,16(%rdi) + movq %r9,24(%rdi) + movq 352(%rsp),%rdx + leaq 352(%rsp),%rbx + movq 0+160(%rsp),%r9 + movq 8+160(%rsp),%r10 + leaq -128+160(%rsp),%rsi + movq 16+160(%rsp),%r11 + movq 24+160(%rsp),%r12 + leaq 32(%rsp),%rdi + call __ecp_nistz256_mul_montx + + movq 96(%rsp),%rdx + leaq 96(%rsp),%rbx + movq 0+64(%rsp),%r9 + movq 8+64(%rsp),%r10 + leaq -128+64(%rsp),%rsi + movq 16+64(%rsp),%r11 + movq 24+64(%rsp),%r12 + leaq 64(%rsp),%rdi + call __ecp_nistz256_mul_montx + + leaq 32(%rsp),%rbx + leaq 256(%rsp),%rdi + call __ecp_nistz256_sub_fromx + + movq %xmm0,%rdi + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 288(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 288+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand .LONE_mont(%rip),%xmm2 + pand .LONE_mont+16(%rip),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 384(%rsp),%xmm2 + pand 384+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,64(%rdi) + movdqu %xmm3,80(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 224(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 224+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 416(%rsp),%xmm2 + pand 416+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 320(%rsp),%xmm2 + pand 320+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,0(%rdi) + movdqu %xmm3,16(%rdi) + + movdqa %xmm5,%xmm0 + movdqa %xmm5,%xmm1 + pandn 256(%rsp),%xmm0 + movdqa %xmm5,%xmm2 + pandn 256+16(%rsp),%xmm1 + movdqa %xmm5,%xmm3 + pand 448(%rsp),%xmm2 + pand 448+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + + movdqa %xmm4,%xmm0 + movdqa %xmm4,%xmm1 + pandn %xmm2,%xmm0 + movdqa %xmm4,%xmm2 + pandn %xmm3,%xmm1 + movdqa %xmm4,%xmm3 + pand 352(%rsp),%xmm2 + pand 352+16(%rsp),%xmm3 + por %xmm0,%xmm2 + por %xmm1,%xmm3 + movdqu %xmm2,32(%rdi) + movdqu %xmm3,48(%rdi) + + leaq 480+56(%rsp),%rsi +.cfi_def_cfa %rsi,8 + movq -48(%rsi),%r15 +.cfi_restore %r15 + movq -40(%rsi),%r14 +.cfi_restore %r14 + movq -32(%rsi),%r13 +.cfi_restore %r13 + movq -24(%rsi),%r12 +.cfi_restore %r12 + movq -16(%rsi),%rbx +.cfi_restore %rbx + movq -8(%rsi),%rbp +.cfi_restore %rbp + leaq (%rsi),%rsp +.cfi_def_cfa_register %rsp +.Ladd_affinex_epilogue: + ret +.cfi_endproc +.size ecp_nistz256_point_add_affine_adx,.-ecp_nistz256_point_add_affine_adx #endif
diff --git a/gen/bcm/p256-x86_64-asm-win.asm b/gen/bcm/p256-x86_64-asm-win.asm index b8d0680..232800b 100644 --- a/gen/bcm/p256-x86_64-asm-win.asm +++ b/gen/bcm/p256-x86_64-asm-win.asm
@@ -15,7 +15,22 @@ +section .rdata rdata align=8 +ALIGN 64 p256_constants: +$L$poly: + DQ 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 + +$L$One: + DD 1,1,1,1,1,1,1,1 +$L$Two: + DD 2,2,2,2,2,2,2,2 +$L$Three: + DD 3,3,3,3,3,3,3,3 +$L$ONE_mont: + DQ 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe + + $L$ord: DQ 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 $L$ordK: @@ -25,6 +40,73 @@ +global ecp_nistz256_neg + +ALIGN 32 +ecp_nistz256_neg: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_neg: + mov rdi,rcx + mov rsi,rdx + + + +_CET_ENDBR + push r12 + + push r13 + +$L$neg_body: + + xor r8,r8 + xor r9,r9 + xor r10,r10 + xor r11,r11 + xor r13,r13 + + sub r8,QWORD[rsi] + sbb r9,QWORD[8+rsi] + sbb r10,QWORD[16+rsi] + mov rax,r8 + sbb r11,QWORD[24+rsi] + lea rsi,[$L$poly] + mov rdx,r9 + sbb r13,0 + + add r8,QWORD[rsi] + mov rcx,r10 + adc r9,QWORD[8+rsi] + adc r10,QWORD[16+rsi] + mov r12,r11 + adc r11,QWORD[24+rsi] + test r13,r13 + + cmovz r8,rax + cmovz r9,rdx + mov QWORD[rdi],r8 + cmovz r10,rcx + mov QWORD[8+rdi],r9 + cmovz r11,r12 + mov QWORD[16+rdi],r10 + mov QWORD[24+rdi],r11 + + mov r13,QWORD[rsp] + + mov r12,QWORD[8+rsp] + + lea rsp,[16+rsp] + +$L$neg_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_neg: + + + @@ -1129,6 +1211,3554 @@ ret $L$SEH_end_ecp_nistz256_ord_sqr_mont_adx: + + + + + + +global ecp_nistz256_mul_mont_nohw + +ALIGN 32 +ecp_nistz256_mul_mont_nohw: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_mul_mont_nohw: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + +$L$mul_body: + mov rbx,rdx + mov rax,QWORD[rdx] + mov r9,QWORD[rsi] + mov r10,QWORD[8+rsi] + mov r11,QWORD[16+rsi] + mov r12,QWORD[24+rsi] + + call __ecp_nistz256_mul_montq + + mov r15,QWORD[rsp] + + mov r14,QWORD[8+rsp] + + mov r13,QWORD[16+rsp] + + mov r12,QWORD[24+rsp] + + mov rbx,QWORD[32+rsp] + + mov rbp,QWORD[40+rsp] + + lea rsp,[48+rsp] + +$L$mul_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_mul_mont_nohw: + + +ALIGN 32 +__ecp_nistz256_mul_montq: + + + + mov rbp,rax + mul r9 + mov r14,QWORD[(($L$poly+8))] + mov r8,rax + mov rax,rbp + mov r9,rdx + + mul r10 + mov r15,QWORD[(($L$poly+24))] + add r9,rax + mov rax,rbp + adc rdx,0 + mov r10,rdx + + mul r11 + add r10,rax + mov rax,rbp + adc rdx,0 + mov r11,rdx + + mul r12 + add r11,rax + mov rax,r8 + adc rdx,0 + xor r13,r13 + mov r12,rdx + + + + + + + + + + + mov rbp,r8 + shl r8,32 + mul r15 + shr rbp,32 + add r9,r8 + adc r10,rbp + adc r11,rax + mov rax,QWORD[8+rbx] + adc r12,rdx + adc r13,0 + xor r8,r8 + + + + mov rbp,rax + mul QWORD[rsi] + add r9,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[8+rsi] + add r10,rcx + adc rdx,0 + add r10,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[16+rsi] + add r11,rcx + adc rdx,0 + add r11,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[24+rsi] + add r12,rcx + adc rdx,0 + add r12,rax + mov rax,r9 + adc r13,rdx + adc r8,0 + + + + mov rbp,r9 + shl r9,32 + mul r15 + shr rbp,32 + add r10,r9 + adc r11,rbp + adc r12,rax + mov rax,QWORD[16+rbx] + adc r13,rdx + adc r8,0 + xor r9,r9 + + + + mov rbp,rax + mul QWORD[rsi] + add r10,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[8+rsi] + add r11,rcx + adc rdx,0 + add r11,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[16+rsi] + add r12,rcx + adc rdx,0 + add r12,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[24+rsi] + add r13,rcx + adc rdx,0 + add r13,rax + mov rax,r10 + adc r8,rdx + adc r9,0 + + + + mov rbp,r10 + shl r10,32 + mul r15 + shr rbp,32 + add r11,r10 + adc r12,rbp + adc r13,rax + mov rax,QWORD[24+rbx] + adc r8,rdx + adc r9,0 + xor r10,r10 + + + + mov rbp,rax + mul QWORD[rsi] + add r11,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[8+rsi] + add r12,rcx + adc rdx,0 + add r12,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[16+rsi] + add r13,rcx + adc rdx,0 + add r13,rax + mov rax,rbp + adc rdx,0 + mov rcx,rdx + + mul QWORD[24+rsi] + add r8,rcx + adc rdx,0 + add r8,rax + mov rax,r11 + adc r9,rdx + adc r10,0 + + + + mov rbp,r11 + shl r11,32 + mul r15 + shr rbp,32 + add r12,r11 + adc r13,rbp + mov rcx,r12 + adc r8,rax + adc r9,rdx + mov rbp,r13 + adc r10,0 + + + + sub r12,-1 + mov rbx,r8 + sbb r13,r14 + sbb r8,0 + mov rdx,r9 + sbb r9,r15 + sbb r10,0 + + cmovc r12,rcx + cmovc r13,rbp + mov QWORD[rdi],r12 + cmovc r8,rbx + mov QWORD[8+rdi],r13 + cmovc r9,rdx + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + + ret + + + + + + + + + + +global ecp_nistz256_sqr_mont_nohw + +ALIGN 32 +ecp_nistz256_sqr_mont_nohw: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_sqr_mont_nohw: + mov rdi,rcx + mov rsi,rdx + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + +$L$sqr_body: + mov rax,QWORD[rsi] + mov r14,QWORD[8+rsi] + mov r15,QWORD[16+rsi] + mov r8,QWORD[24+rsi] + + call __ecp_nistz256_sqr_montq + + mov r15,QWORD[rsp] + + mov r14,QWORD[8+rsp] + + mov r13,QWORD[16+rsp] + + mov r12,QWORD[24+rsp] + + mov rbx,QWORD[32+rsp] + + mov rbp,QWORD[40+rsp] + + lea rsp,[48+rsp] + +$L$sqr_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_sqr_mont_nohw: + + +ALIGN 32 +__ecp_nistz256_sqr_montq: + + mov r13,rax + mul r14 + mov r9,rax + mov rax,r15 + mov r10,rdx + + mul r13 + add r10,rax + mov rax,r8 + adc rdx,0 + mov r11,rdx + + mul r13 + add r11,rax + mov rax,r15 + adc rdx,0 + mov r12,rdx + + + mul r14 + add r11,rax + mov rax,r8 + adc rdx,0 + mov rbp,rdx + + mul r14 + add r12,rax + mov rax,r8 + adc rdx,0 + add r12,rbp + mov r13,rdx + adc r13,0 + + + mul r15 + xor r15,r15 + add r13,rax + mov rax,QWORD[rsi] + mov r14,rdx + adc r14,0 + + add r9,r9 + adc r10,r10 + adc r11,r11 + adc r12,r12 + adc r13,r13 + adc r14,r14 + adc r15,0 + + mul rax + mov r8,rax + mov rax,QWORD[8+rsi] + mov rcx,rdx + + mul rax + add r9,rcx + adc r10,rax + mov rax,QWORD[16+rsi] + adc rdx,0 + mov rcx,rdx + + mul rax + add r11,rcx + adc r12,rax + mov rax,QWORD[24+rsi] + adc rdx,0 + mov rcx,rdx + + mul rax + add r13,rcx + adc r14,rax + mov rax,r8 + adc r15,rdx + + mov rsi,QWORD[(($L$poly+8))] + mov rbp,QWORD[(($L$poly+24))] + + + + + mov rcx,r8 + shl r8,32 + mul rbp + shr rcx,32 + add r9,r8 + adc r10,rcx + adc r11,rax + mov rax,r9 + adc rdx,0 + + + + mov rcx,r9 + shl r9,32 + mov r8,rdx + mul rbp + shr rcx,32 + add r10,r9 + adc r11,rcx + adc r8,rax + mov rax,r10 + adc rdx,0 + + + + mov rcx,r10 + shl r10,32 + mov r9,rdx + mul rbp + shr rcx,32 + add r11,r10 + adc r8,rcx + adc r9,rax + mov rax,r11 + adc rdx,0 + + + + mov rcx,r11 + shl r11,32 + mov r10,rdx + mul rbp + shr rcx,32 + add r8,r11 + adc r9,rcx + adc r10,rax + adc rdx,0 + xor r11,r11 + + + + add r12,r8 + adc r13,r9 + mov r8,r12 + adc r14,r10 + adc r15,rdx + mov r9,r13 + adc r11,0 + + sub r12,-1 + mov r10,r14 + sbb r13,rsi + sbb r14,0 + mov rcx,r15 + sbb r15,rbp + sbb r11,0 + + cmovc r12,r8 + cmovc r13,r9 + mov QWORD[rdi],r12 + cmovc r14,r10 + mov QWORD[8+rdi],r13 + cmovc r15,rcx + mov QWORD[16+rdi],r14 + mov QWORD[24+rdi],r15 + + ret + + +global ecp_nistz256_mul_mont_adx + +ALIGN 32 +ecp_nistz256_mul_mont_adx: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_mul_mont_adx: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + +$L$mulx_body: + mov rbx,rdx + mov rdx,QWORD[rdx] + mov r9,QWORD[rsi] + mov r10,QWORD[8+rsi] + mov r11,QWORD[16+rsi] + mov r12,QWORD[24+rsi] + lea rsi,[((-128))+rsi] + + call __ecp_nistz256_mul_montx + + mov r15,QWORD[rsp] + + mov r14,QWORD[8+rsp] + + mov r13,QWORD[16+rsp] + + mov r12,QWORD[24+rsp] + + mov rbx,QWORD[32+rsp] + + mov rbp,QWORD[40+rsp] + + lea rsp,[48+rsp] + +$L$mulx_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_mul_mont_adx: + + +ALIGN 32 +__ecp_nistz256_mul_montx: + + + + mulx r9,r8,r9 + mulx r10,rcx,r10 + mov r14,32 + xor r13,r13 + mulx r11,rbp,r11 + mov r15,QWORD[(($L$poly+24))] + adc r9,rcx + mulx r12,rcx,r12 + mov rdx,r8 + adc r10,rbp + shlx rbp,r8,r14 + adc r11,rcx + shrx rcx,r8,r14 + adc r12,0 + + + + add r9,rbp + adc r10,rcx + + mulx rbp,rcx,r15 + mov rdx,QWORD[8+rbx] + adc r11,rcx + adc r12,rbp + adc r13,0 + xor r8,r8 + + + + mulx rbp,rcx,QWORD[((0+128))+rsi] + adcx r9,rcx + adox r10,rbp + + mulx rbp,rcx,QWORD[((8+128))+rsi] + adcx r10,rcx + adox r11,rbp + + mulx rbp,rcx,QWORD[((16+128))+rsi] + adcx r11,rcx + adox r12,rbp + + mulx rbp,rcx,QWORD[((24+128))+rsi] + mov rdx,r9 + adcx r12,rcx + shlx rcx,r9,r14 + adox r13,rbp + shrx rbp,r9,r14 + + adcx r13,r8 + adox r8,r8 + adc r8,0 + + + + add r10,rcx + adc r11,rbp + + mulx rbp,rcx,r15 + mov rdx,QWORD[16+rbx] + adc r12,rcx + adc r13,rbp + adc r8,0 + xor r9,r9 + + + + mulx rbp,rcx,QWORD[((0+128))+rsi] + adcx r10,rcx + adox r11,rbp + + mulx rbp,rcx,QWORD[((8+128))+rsi] + adcx r11,rcx + adox r12,rbp + + mulx rbp,rcx,QWORD[((16+128))+rsi] + adcx r12,rcx + adox r13,rbp + + mulx rbp,rcx,QWORD[((24+128))+rsi] + mov rdx,r10 + adcx r13,rcx + shlx rcx,r10,r14 + adox r8,rbp + shrx rbp,r10,r14 + + adcx r8,r9 + adox r9,r9 + adc r9,0 + + + + add r11,rcx + adc r12,rbp + + mulx rbp,rcx,r15 + mov rdx,QWORD[24+rbx] + adc r13,rcx + adc r8,rbp + adc r9,0 + xor r10,r10 + + + + mulx rbp,rcx,QWORD[((0+128))+rsi] + adcx r11,rcx + adox r12,rbp + + mulx rbp,rcx,QWORD[((8+128))+rsi] + adcx r12,rcx + adox r13,rbp + + mulx rbp,rcx,QWORD[((16+128))+rsi] + adcx r13,rcx + adox r8,rbp + + mulx rbp,rcx,QWORD[((24+128))+rsi] + mov rdx,r11 + adcx r8,rcx + shlx rcx,r11,r14 + adox r9,rbp + shrx rbp,r11,r14 + + adcx r9,r10 + adox r10,r10 + adc r10,0 + + + + add r12,rcx + adc r13,rbp + + mulx rbp,rcx,r15 + mov rbx,r12 + mov r14,QWORD[(($L$poly+8))] + adc r8,rcx + mov rdx,r13 + adc r9,rbp + adc r10,0 + + + + xor eax,eax + mov rcx,r8 + sbb r12,-1 + sbb r13,r14 + sbb r8,0 + mov rbp,r9 + sbb r9,r15 + sbb r10,0 + + cmovc r12,rbx + cmovc r13,rdx + mov QWORD[rdi],r12 + cmovc r8,rcx + mov QWORD[8+rdi],r13 + cmovc r9,rbp + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + + ret + + + +global ecp_nistz256_sqr_mont_adx + +ALIGN 32 +ecp_nistz256_sqr_mont_adx: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_sqr_mont_adx: + mov rdi,rcx + mov rsi,rdx + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + +$L$sqrx_body: + mov rdx,QWORD[rsi] + mov r14,QWORD[8+rsi] + mov r15,QWORD[16+rsi] + mov r8,QWORD[24+rsi] + lea rsi,[((-128))+rsi] + + call __ecp_nistz256_sqr_montx + + mov r15,QWORD[rsp] + + mov r14,QWORD[8+rsp] + + mov r13,QWORD[16+rsp] + + mov r12,QWORD[24+rsp] + + mov rbx,QWORD[32+rsp] + + mov rbp,QWORD[40+rsp] + + lea rsp,[48+rsp] + +$L$sqrx_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_sqr_mont_adx: + + +ALIGN 32 +__ecp_nistz256_sqr_montx: + + mulx r10,r9,r14 + mulx r11,rcx,r15 + xor eax,eax + adc r10,rcx + mulx r12,rbp,r8 + mov rdx,r14 + adc r11,rbp + adc r12,0 + xor r13,r13 + + + mulx rbp,rcx,r15 + adcx r11,rcx + adox r12,rbp + + mulx rbp,rcx,r8 + mov rdx,r15 + adcx r12,rcx + adox r13,rbp + adc r13,0 + + + mulx r14,rcx,r8 + mov rdx,QWORD[((0+128))+rsi] + xor r15,r15 + adcx r9,r9 + adox r13,rcx + adcx r10,r10 + adox r14,r15 + + mulx rbp,r8,rdx + mov rdx,QWORD[((8+128))+rsi] + adcx r11,r11 + adox r9,rbp + adcx r12,r12 + mulx rax,rcx,rdx + mov rdx,QWORD[((16+128))+rsi] + adcx r13,r13 + adox r10,rcx + adcx r14,r14 + DB 0x67 + mulx rbp,rcx,rdx + mov rdx,QWORD[((24+128))+rsi] + adox r11,rax + adcx r15,r15 + adox r12,rcx + mov rsi,32 + adox r13,rbp + DB 0x67,0x67 + mulx rax,rcx,rdx + mov rdx,QWORD[(($L$poly+24))] + adox r14,rcx + shlx rcx,r8,rsi + adox r15,rax + shrx rax,r8,rsi + mov rbp,rdx + + + add r9,rcx + adc r10,rax + + mulx r8,rcx,r8 + adc r11,rcx + shlx rcx,r9,rsi + adc r8,0 + shrx rax,r9,rsi + + + add r10,rcx + adc r11,rax + + mulx r9,rcx,r9 + adc r8,rcx + shlx rcx,r10,rsi + adc r9,0 + shrx rax,r10,rsi + + + add r11,rcx + adc r8,rax + + mulx r10,rcx,r10 + adc r9,rcx + shlx rcx,r11,rsi + adc r10,0 + shrx rax,r11,rsi + + + add r8,rcx + adc r9,rax + + mulx r11,rcx,r11 + adc r10,rcx + adc r11,0 + + xor rdx,rdx + add r12,r8 + mov rsi,QWORD[(($L$poly+8))] + adc r13,r9 + mov r8,r12 + adc r14,r10 + adc r15,r11 + mov r9,r13 + adc rdx,0 + + sub r12,-1 + mov r10,r14 + sbb r13,rsi + sbb r14,0 + mov r11,r15 + sbb r15,rbp + sbb rdx,0 + + cmovc r12,r8 + cmovc r13,r9 + mov QWORD[rdi],r12 + cmovc r14,r10 + mov QWORD[8+rdi],r13 + cmovc r15,r11 + mov QWORD[16+rdi],r14 + mov QWORD[24+rdi],r15 + + ret + + + + +global ecp_nistz256_select_w5_nohw + +ALIGN 32 +ecp_nistz256_select_w5_nohw: + +_CET_ENDBR + lea rax,[((-136))+rsp] +$L$SEH_begin_ecp_nistz256_select_w5_nohw: + DB 0x48,0x8d,0x60,0xe0 + DB 0x0f,0x29,0x70,0xe0 + DB 0x0f,0x29,0x78,0xf0 + DB 0x44,0x0f,0x29,0x00 + DB 0x44,0x0f,0x29,0x48,0x10 + DB 0x44,0x0f,0x29,0x50,0x20 + DB 0x44,0x0f,0x29,0x58,0x30 + DB 0x44,0x0f,0x29,0x60,0x40 + DB 0x44,0x0f,0x29,0x68,0x50 + DB 0x44,0x0f,0x29,0x70,0x60 + DB 0x44,0x0f,0x29,0x78,0x70 + movdqa xmm0,XMMWORD[$L$One] + movd xmm1,r8d + + pxor xmm2,xmm2 + pxor xmm3,xmm3 + pxor xmm4,xmm4 + pxor xmm5,xmm5 + pxor xmm6,xmm6 + pxor xmm7,xmm7 + + movdqa xmm8,xmm0 + pshufd xmm1,xmm1,0 + + mov rax,16 +$L$select_loop_sse_w5: + + movdqa xmm15,xmm8 + paddd xmm8,xmm0 + pcmpeqd xmm15,xmm1 + + movdqa xmm9,XMMWORD[rdx] + movdqa xmm10,XMMWORD[16+rdx] + movdqa xmm11,XMMWORD[32+rdx] + movdqa xmm12,XMMWORD[48+rdx] + movdqa xmm13,XMMWORD[64+rdx] + movdqa xmm14,XMMWORD[80+rdx] + lea rdx,[96+rdx] + + pand xmm9,xmm15 + pand xmm10,xmm15 + por xmm2,xmm9 + pand xmm11,xmm15 + por xmm3,xmm10 + pand xmm12,xmm15 + por xmm4,xmm11 + pand xmm13,xmm15 + por xmm5,xmm12 + pand xmm14,xmm15 + por xmm6,xmm13 + por xmm7,xmm14 + + dec rax + jnz NEAR $L$select_loop_sse_w5 + + movdqu XMMWORD[rcx],xmm2 + movdqu XMMWORD[16+rcx],xmm3 + movdqu XMMWORD[32+rcx],xmm4 + movdqu XMMWORD[48+rcx],xmm5 + movdqu XMMWORD[64+rcx],xmm6 + movdqu XMMWORD[80+rcx],xmm7 + movaps xmm6,XMMWORD[rsp] + movaps xmm7,XMMWORD[16+rsp] + movaps xmm8,XMMWORD[32+rsp] + movaps xmm9,XMMWORD[48+rsp] + movaps xmm10,XMMWORD[64+rsp] + movaps xmm11,XMMWORD[80+rsp] + movaps xmm12,XMMWORD[96+rsp] + movaps xmm13,XMMWORD[112+rsp] + movaps xmm14,XMMWORD[128+rsp] + movaps xmm15,XMMWORD[144+rsp] + lea rsp,[168+rsp] + ret + +$L$SEH_end_ecp_nistz256_select_w5_nohw: + + + + +global ecp_nistz256_select_w7_nohw + +ALIGN 32 +ecp_nistz256_select_w7_nohw: + +_CET_ENDBR + lea rax,[((-136))+rsp] +$L$SEH_begin_ecp_nistz256_select_w7_nohw: + DB 0x48,0x8d,0x60,0xe0 + DB 0x0f,0x29,0x70,0xe0 + DB 0x0f,0x29,0x78,0xf0 + DB 0x44,0x0f,0x29,0x00 + DB 0x44,0x0f,0x29,0x48,0x10 + DB 0x44,0x0f,0x29,0x50,0x20 + DB 0x44,0x0f,0x29,0x58,0x30 + DB 0x44,0x0f,0x29,0x60,0x40 + DB 0x44,0x0f,0x29,0x68,0x50 + DB 0x44,0x0f,0x29,0x70,0x60 + DB 0x44,0x0f,0x29,0x78,0x70 + movdqa xmm8,XMMWORD[$L$One] + movd xmm1,r8d + + pxor xmm2,xmm2 + pxor xmm3,xmm3 + pxor xmm4,xmm4 + pxor xmm5,xmm5 + + movdqa xmm0,xmm8 + pshufd xmm1,xmm1,0 + mov rax,64 + +$L$select_loop_sse_w7: + movdqa xmm15,xmm8 + paddd xmm8,xmm0 + movdqa xmm9,XMMWORD[rdx] + movdqa xmm10,XMMWORD[16+rdx] + pcmpeqd xmm15,xmm1 + movdqa xmm11,XMMWORD[32+rdx] + movdqa xmm12,XMMWORD[48+rdx] + lea rdx,[64+rdx] + + pand xmm9,xmm15 + pand xmm10,xmm15 + por xmm2,xmm9 + pand xmm11,xmm15 + por xmm3,xmm10 + pand xmm12,xmm15 + por xmm4,xmm11 + prefetcht0 [255+rdx] + por xmm5,xmm12 + + dec rax + jnz NEAR $L$select_loop_sse_w7 + + movdqu XMMWORD[rcx],xmm2 + movdqu XMMWORD[16+rcx],xmm3 + movdqu XMMWORD[32+rcx],xmm4 + movdqu XMMWORD[48+rcx],xmm5 + movaps xmm6,XMMWORD[rsp] + movaps xmm7,XMMWORD[16+rsp] + movaps xmm8,XMMWORD[32+rsp] + movaps xmm9,XMMWORD[48+rsp] + movaps xmm10,XMMWORD[64+rsp] + movaps xmm11,XMMWORD[80+rsp] + movaps xmm12,XMMWORD[96+rsp] + movaps xmm13,XMMWORD[112+rsp] + movaps xmm14,XMMWORD[128+rsp] + movaps xmm15,XMMWORD[144+rsp] + lea rsp,[168+rsp] + ret + +$L$SEH_end_ecp_nistz256_select_w7_nohw: + + + +global ecp_nistz256_select_w5_avx2 + +ALIGN 32 +ecp_nistz256_select_w5_avx2: + +_CET_ENDBR + vzeroupper + lea rax,[((-136))+rsp] + mov r11,rsp +$L$SEH_begin_ecp_nistz256_select_w5_avx2: + DB 0x48,0x8d,0x60,0xe0 + DB 0xc5,0xf8,0x29,0x70,0xe0 + DB 0xc5,0xf8,0x29,0x78,0xf0 + DB 0xc5,0x78,0x29,0x40,0x00 + DB 0xc5,0x78,0x29,0x48,0x10 + DB 0xc5,0x78,0x29,0x50,0x20 + DB 0xc5,0x78,0x29,0x58,0x30 + DB 0xc5,0x78,0x29,0x60,0x40 + DB 0xc5,0x78,0x29,0x68,0x50 + DB 0xc5,0x78,0x29,0x70,0x60 + DB 0xc5,0x78,0x29,0x78,0x70 + vmovdqa ymm0,YMMWORD[$L$Two] + + vpxor ymm2,ymm2,ymm2 + vpxor ymm3,ymm3,ymm3 + vpxor ymm4,ymm4,ymm4 + + vmovdqa ymm5,YMMWORD[$L$One] + vmovdqa ymm10,YMMWORD[$L$Two] + + vmovd xmm1,r8d + vpermd ymm1,ymm2,ymm1 + + mov rax,8 +$L$select_loop_avx2_w5: + + vmovdqa ymm6,YMMWORD[rdx] + vmovdqa ymm7,YMMWORD[32+rdx] + vmovdqa ymm8,YMMWORD[64+rdx] + + vmovdqa ymm11,YMMWORD[96+rdx] + vmovdqa ymm12,YMMWORD[128+rdx] + vmovdqa ymm13,YMMWORD[160+rdx] + + vpcmpeqd ymm9,ymm5,ymm1 + vpcmpeqd ymm14,ymm10,ymm1 + + vpaddd ymm5,ymm5,ymm0 + vpaddd ymm10,ymm10,ymm0 + lea rdx,[192+rdx] + + vpand ymm6,ymm6,ymm9 + vpand ymm7,ymm7,ymm9 + vpand ymm8,ymm8,ymm9 + vpand ymm11,ymm11,ymm14 + vpand ymm12,ymm12,ymm14 + vpand ymm13,ymm13,ymm14 + + vpxor ymm2,ymm2,ymm6 + vpxor ymm3,ymm3,ymm7 + vpxor ymm4,ymm4,ymm8 + vpxor ymm2,ymm2,ymm11 + vpxor ymm3,ymm3,ymm12 + vpxor ymm4,ymm4,ymm13 + + dec rax + jnz NEAR $L$select_loop_avx2_w5 + + vmovdqu YMMWORD[rcx],ymm2 + vmovdqu YMMWORD[32+rcx],ymm3 + vmovdqu YMMWORD[64+rcx],ymm4 + vzeroupper + movaps xmm6,XMMWORD[rsp] + movaps xmm7,XMMWORD[16+rsp] + movaps xmm8,XMMWORD[32+rsp] + movaps xmm9,XMMWORD[48+rsp] + movaps xmm10,XMMWORD[64+rsp] + movaps xmm11,XMMWORD[80+rsp] + movaps xmm12,XMMWORD[96+rsp] + movaps xmm13,XMMWORD[112+rsp] + movaps xmm14,XMMWORD[128+rsp] + movaps xmm15,XMMWORD[144+rsp] + lea rsp,[r11] + ret + +$L$SEH_end_ecp_nistz256_select_w5_avx2: + + + + +global ecp_nistz256_select_w7_avx2 + +ALIGN 32 +ecp_nistz256_select_w7_avx2: + +_CET_ENDBR + vzeroupper + mov r11,rsp + lea rax,[((-136))+rsp] +$L$SEH_begin_ecp_nistz256_select_w7_avx2: + DB 0x48,0x8d,0x60,0xe0 + DB 0xc5,0xf8,0x29,0x70,0xe0 + DB 0xc5,0xf8,0x29,0x78,0xf0 + DB 0xc5,0x78,0x29,0x40,0x00 + DB 0xc5,0x78,0x29,0x48,0x10 + DB 0xc5,0x78,0x29,0x50,0x20 + DB 0xc5,0x78,0x29,0x58,0x30 + DB 0xc5,0x78,0x29,0x60,0x40 + DB 0xc5,0x78,0x29,0x68,0x50 + DB 0xc5,0x78,0x29,0x70,0x60 + DB 0xc5,0x78,0x29,0x78,0x70 + vmovdqa ymm0,YMMWORD[$L$Three] + + vpxor ymm2,ymm2,ymm2 + vpxor ymm3,ymm3,ymm3 + + vmovdqa ymm4,YMMWORD[$L$One] + vmovdqa ymm8,YMMWORD[$L$Two] + vmovdqa ymm12,YMMWORD[$L$Three] + + vmovd xmm1,r8d + vpermd ymm1,ymm2,ymm1 + + + mov rax,21 +$L$select_loop_avx2_w7: + + vmovdqa ymm5,YMMWORD[rdx] + vmovdqa ymm6,YMMWORD[32+rdx] + + vmovdqa ymm9,YMMWORD[64+rdx] + vmovdqa ymm10,YMMWORD[96+rdx] + + vmovdqa ymm13,YMMWORD[128+rdx] + vmovdqa ymm14,YMMWORD[160+rdx] + + vpcmpeqd ymm7,ymm4,ymm1 + vpcmpeqd ymm11,ymm8,ymm1 + vpcmpeqd ymm15,ymm12,ymm1 + + vpaddd ymm4,ymm4,ymm0 + vpaddd ymm8,ymm8,ymm0 + vpaddd ymm12,ymm12,ymm0 + lea rdx,[192+rdx] + + vpand ymm5,ymm5,ymm7 + vpand ymm6,ymm6,ymm7 + vpand ymm9,ymm9,ymm11 + vpand ymm10,ymm10,ymm11 + vpand ymm13,ymm13,ymm15 + vpand ymm14,ymm14,ymm15 + + vpxor ymm2,ymm2,ymm5 + vpxor ymm3,ymm3,ymm6 + vpxor ymm2,ymm2,ymm9 + vpxor ymm3,ymm3,ymm10 + vpxor ymm2,ymm2,ymm13 + vpxor ymm3,ymm3,ymm14 + + dec rax + jnz NEAR $L$select_loop_avx2_w7 + + + vmovdqa ymm5,YMMWORD[rdx] + vmovdqa ymm6,YMMWORD[32+rdx] + + vpcmpeqd ymm7,ymm4,ymm1 + + vpand ymm5,ymm5,ymm7 + vpand ymm6,ymm6,ymm7 + + vpxor ymm2,ymm2,ymm5 + vpxor ymm3,ymm3,ymm6 + + vmovdqu YMMWORD[rcx],ymm2 + vmovdqu YMMWORD[32+rcx],ymm3 + vzeroupper + movaps xmm6,XMMWORD[rsp] + movaps xmm7,XMMWORD[16+rsp] + movaps xmm8,XMMWORD[32+rsp] + movaps xmm9,XMMWORD[48+rsp] + movaps xmm10,XMMWORD[64+rsp] + movaps xmm11,XMMWORD[80+rsp] + movaps xmm12,XMMWORD[96+rsp] + movaps xmm13,XMMWORD[112+rsp] + movaps xmm14,XMMWORD[128+rsp] + movaps xmm15,XMMWORD[144+rsp] + lea rsp,[r11] + ret + +$L$SEH_end_ecp_nistz256_select_w7_avx2: + + +ALIGN 32 +__ecp_nistz256_add_toq: + + xor r11,r11 + add r12,QWORD[rbx] + adc r13,QWORD[8+rbx] + mov rax,r12 + adc r8,QWORD[16+rbx] + adc r9,QWORD[24+rbx] + mov rbp,r13 + adc r11,0 + + sub r12,-1 + mov rcx,r8 + sbb r13,r14 + sbb r8,0 + mov r10,r9 + sbb r9,r15 + sbb r11,0 + + cmovc r12,rax + cmovc r13,rbp + mov QWORD[rdi],r12 + cmovc r8,rcx + mov QWORD[8+rdi],r13 + cmovc r9,r10 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + + ret + + + + +ALIGN 32 +__ecp_nistz256_sub_fromq: + + sub r12,QWORD[rbx] + sbb r13,QWORD[8+rbx] + mov rax,r12 + sbb r8,QWORD[16+rbx] + sbb r9,QWORD[24+rbx] + mov rbp,r13 + sbb r11,r11 + + add r12,-1 + mov rcx,r8 + adc r13,r14 + adc r8,0 + mov r10,r9 + adc r9,r15 + test r11,r11 + + cmovz r12,rax + cmovz r13,rbp + mov QWORD[rdi],r12 + cmovz r8,rcx + mov QWORD[8+rdi],r13 + cmovz r9,r10 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + + ret + + + + +ALIGN 32 +__ecp_nistz256_subq: + + sub rax,r12 + sbb rbp,r13 + mov r12,rax + sbb rcx,r8 + sbb r10,r9 + mov r13,rbp + sbb r11,r11 + + add rax,-1 + mov r8,rcx + adc rbp,r14 + adc rcx,0 + mov r9,r10 + adc r10,r15 + test r11,r11 + + cmovnz r12,rax + cmovnz r13,rbp + cmovnz r8,rcx + cmovnz r9,r10 + + ret + + + + +ALIGN 32 +__ecp_nistz256_mul_by_2q: + + xor r11,r11 + add r12,r12 + adc r13,r13 + mov rax,r12 + adc r8,r8 + adc r9,r9 + mov rbp,r13 + adc r11,0 + + sub r12,-1 + mov rcx,r8 + sbb r13,r14 + sbb r8,0 + mov r10,r9 + sbb r9,r15 + sbb r11,0 + + cmovc r12,rax + cmovc r13,rbp + mov QWORD[rdi],r12 + cmovc r8,rcx + mov QWORD[8+rdi],r13 + cmovc r9,r10 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + + ret + + +global ecp_nistz256_point_double_nohw + +ALIGN 32 +ecp_nistz256_point_double_nohw: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_point_double_nohw: + mov rdi,rcx + mov rsi,rdx + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,32*5+8 + +$L$point_doubleq_body: + +$L$point_double_shortcutq: + movdqu xmm0,XMMWORD[rsi] + mov rbx,rsi + movdqu xmm1,XMMWORD[16+rsi] + mov r12,QWORD[((32+0))+rsi] + mov r13,QWORD[((32+8))+rsi] + mov r8,QWORD[((32+16))+rsi] + mov r9,QWORD[((32+24))+rsi] + mov r14,QWORD[(($L$poly+8))] + mov r15,QWORD[(($L$poly+24))] + movdqa XMMWORD[96+rsp],xmm0 + movdqa XMMWORD[(96+16)+rsp],xmm1 + lea r10,[32+rdi] + lea r11,[64+rdi] + movq xmm0,rdi + movq xmm1,r10 + movq xmm2,r11 + + lea rdi,[rsp] + call __ecp_nistz256_mul_by_2q + + mov rax,QWORD[((64+0))+rsi] + mov r14,QWORD[((64+8))+rsi] + mov r15,QWORD[((64+16))+rsi] + mov r8,QWORD[((64+24))+rsi] + lea rsi,[((64-0))+rsi] + lea rdi,[64+rsp] + call __ecp_nistz256_sqr_montq + + mov rax,QWORD[((0+0))+rsp] + mov r14,QWORD[((8+0))+rsp] + lea rsi,[((0+0))+rsp] + mov r15,QWORD[((16+0))+rsp] + mov r8,QWORD[((24+0))+rsp] + lea rdi,[rsp] + call __ecp_nistz256_sqr_montq + + mov rax,QWORD[32+rbx] + mov r9,QWORD[((64+0))+rbx] + mov r10,QWORD[((64+8))+rbx] + mov r11,QWORD[((64+16))+rbx] + mov r12,QWORD[((64+24))+rbx] + lea rsi,[((64-0))+rbx] + lea rbx,[32+rbx] + movq rdi,xmm2 + call __ecp_nistz256_mul_montq + call __ecp_nistz256_mul_by_2q + + mov r12,QWORD[((96+0))+rsp] + mov r13,QWORD[((96+8))+rsp] + lea rbx,[64+rsp] + mov r8,QWORD[((96+16))+rsp] + mov r9,QWORD[((96+24))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_add_toq + + mov r12,QWORD[((96+0))+rsp] + mov r13,QWORD[((96+8))+rsp] + lea rbx,[64+rsp] + mov r8,QWORD[((96+16))+rsp] + mov r9,QWORD[((96+24))+rsp] + lea rdi,[64+rsp] + call __ecp_nistz256_sub_fromq + + mov rax,QWORD[((0+0))+rsp] + mov r14,QWORD[((8+0))+rsp] + lea rsi,[((0+0))+rsp] + mov r15,QWORD[((16+0))+rsp] + mov r8,QWORD[((24+0))+rsp] + movq rdi,xmm1 + call __ecp_nistz256_sqr_montq + xor r9,r9 + mov rax,r12 + add r12,-1 + mov r10,r13 + adc r13,rsi + mov rcx,r14 + adc r14,0 + mov r8,r15 + adc r15,rbp + adc r9,0 + xor rsi,rsi + test rax,1 + + cmovz r12,rax + cmovz r13,r10 + cmovz r14,rcx + cmovz r15,r8 + cmovz r9,rsi + + mov rax,r13 + shr r12,1 + shl rax,63 + mov r10,r14 + shr r13,1 + or r12,rax + shl r10,63 + mov rcx,r15 + shr r14,1 + or r13,r10 + shl rcx,63 + mov QWORD[rdi],r12 + shr r15,1 + mov QWORD[8+rdi],r13 + shl r9,63 + or r14,rcx + or r15,r9 + mov QWORD[16+rdi],r14 + mov QWORD[24+rdi],r15 + mov rax,QWORD[64+rsp] + lea rbx,[64+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((0+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_mul_montq + + lea rdi,[128+rsp] + call __ecp_nistz256_mul_by_2q + + lea rbx,[32+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_add_toq + + mov rax,QWORD[96+rsp] + lea rbx,[96+rsp] + mov r9,QWORD[((0+0))+rsp] + mov r10,QWORD[((8+0))+rsp] + lea rsi,[((0+0))+rsp] + mov r11,QWORD[((16+0))+rsp] + mov r12,QWORD[((24+0))+rsp] + lea rdi,[rsp] + call __ecp_nistz256_mul_montq + + lea rdi,[128+rsp] + call __ecp_nistz256_mul_by_2q + + mov rax,QWORD[((0+32))+rsp] + mov r14,QWORD[((8+32))+rsp] + lea rsi,[((0+32))+rsp] + mov r15,QWORD[((16+32))+rsp] + mov r8,QWORD[((24+32))+rsp] + movq rdi,xmm0 + call __ecp_nistz256_sqr_montq + + lea rbx,[128+rsp] + mov r8,r14 + mov r9,r15 + mov r14,rsi + mov r15,rbp + call __ecp_nistz256_sub_fromq + + mov rax,QWORD[((0+0))+rsp] + mov rbp,QWORD[((0+8))+rsp] + mov rcx,QWORD[((0+16))+rsp] + mov r10,QWORD[((0+24))+rsp] + lea rdi,[rsp] + call __ecp_nistz256_subq + + mov rax,QWORD[32+rsp] + lea rbx,[32+rsp] + mov r14,r12 + xor ecx,ecx + mov QWORD[((0+0))+rsp],r12 + mov r10,r13 + mov QWORD[((0+8))+rsp],r13 + cmovz r11,r8 + mov QWORD[((0+16))+rsp],r8 + lea rsi,[((0-0))+rsp] + cmovz r12,r9 + mov QWORD[((0+24))+rsp],r9 + mov r9,r14 + lea rdi,[rsp] + call __ecp_nistz256_mul_montq + + movq rbx,xmm1 + movq rdi,xmm1 + call __ecp_nistz256_sub_fromq + + lea rsi,[((160+56))+rsp] + + mov r15,QWORD[((-48))+rsi] + + mov r14,QWORD[((-40))+rsi] + + mov r13,QWORD[((-32))+rsi] + + mov r12,QWORD[((-24))+rsi] + + mov rbx,QWORD[((-16))+rsi] + + mov rbp,QWORD[((-8))+rsi] + + lea rsp,[rsi] + +$L$point_doubleq_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_point_double_nohw: +global ecp_nistz256_point_add_nohw + +ALIGN 32 +ecp_nistz256_point_add_nohw: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_point_add_nohw: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,32*18+8 + +$L$point_addq_body: + + movdqu xmm0,XMMWORD[rsi] + movdqu xmm1,XMMWORD[16+rsi] + movdqu xmm2,XMMWORD[32+rsi] + movdqu xmm3,XMMWORD[48+rsi] + movdqu xmm4,XMMWORD[64+rsi] + movdqu xmm5,XMMWORD[80+rsi] + mov rbx,rsi + mov rsi,rdx + movdqa XMMWORD[384+rsp],xmm0 + movdqa XMMWORD[(384+16)+rsp],xmm1 + movdqa XMMWORD[416+rsp],xmm2 + movdqa XMMWORD[(416+16)+rsp],xmm3 + movdqa XMMWORD[448+rsp],xmm4 + movdqa XMMWORD[(448+16)+rsp],xmm5 + por xmm5,xmm4 + + movdqu xmm0,XMMWORD[rsi] + pshufd xmm3,xmm5,0xb1 + movdqu xmm1,XMMWORD[16+rsi] + movdqu xmm2,XMMWORD[32+rsi] + por xmm5,xmm3 + movdqu xmm3,XMMWORD[48+rsi] + mov rax,QWORD[((64+0))+rsi] + mov r14,QWORD[((64+8))+rsi] + mov r15,QWORD[((64+16))+rsi] + mov r8,QWORD[((64+24))+rsi] + movdqa XMMWORD[480+rsp],xmm0 + pshufd xmm4,xmm5,0x1e + movdqa XMMWORD[(480+16)+rsp],xmm1 + movdqu xmm0,XMMWORD[64+rsi] + movdqu xmm1,XMMWORD[80+rsi] + movdqa XMMWORD[512+rsp],xmm2 + movdqa XMMWORD[(512+16)+rsp],xmm3 + por xmm5,xmm4 + pxor xmm4,xmm4 + por xmm1,xmm0 + movq xmm0,rdi + + lea rsi,[((64-0))+rsi] + mov QWORD[((544+0))+rsp],rax + mov QWORD[((544+8))+rsp],r14 + mov QWORD[((544+16))+rsp],r15 + mov QWORD[((544+24))+rsp],r8 + lea rdi,[96+rsp] + call __ecp_nistz256_sqr_montq + + pcmpeqd xmm5,xmm4 + pshufd xmm4,xmm1,0xb1 + por xmm4,xmm1 + pshufd xmm5,xmm5,0 + pshufd xmm3,xmm4,0x1e + por xmm4,xmm3 + pxor xmm3,xmm3 + pcmpeqd xmm4,xmm3 + pshufd xmm4,xmm4,0 + mov rax,QWORD[((64+0))+rbx] + mov r14,QWORD[((64+8))+rbx] + mov r15,QWORD[((64+16))+rbx] + mov r8,QWORD[((64+24))+rbx] + movq xmm1,rbx + + lea rsi,[((64-0))+rbx] + lea rdi,[32+rsp] + call __ecp_nistz256_sqr_montq + + mov rax,QWORD[544+rsp] + lea rbx,[544+rsp] + mov r9,QWORD[((0+96))+rsp] + mov r10,QWORD[((8+96))+rsp] + lea rsi,[((0+96))+rsp] + mov r11,QWORD[((16+96))+rsp] + mov r12,QWORD[((24+96))+rsp] + lea rdi,[224+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[448+rsp] + lea rbx,[448+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((0+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[256+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[416+rsp] + lea rbx,[416+rsp] + mov r9,QWORD[((0+224))+rsp] + mov r10,QWORD[((8+224))+rsp] + lea rsi,[((0+224))+rsp] + mov r11,QWORD[((16+224))+rsp] + mov r12,QWORD[((24+224))+rsp] + lea rdi,[224+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[512+rsp] + lea rbx,[512+rsp] + mov r9,QWORD[((0+256))+rsp] + mov r10,QWORD[((8+256))+rsp] + lea rsi,[((0+256))+rsp] + mov r11,QWORD[((16+256))+rsp] + mov r12,QWORD[((24+256))+rsp] + lea rdi,[256+rsp] + call __ecp_nistz256_mul_montq + + lea rbx,[224+rsp] + lea rdi,[64+rsp] + call __ecp_nistz256_sub_fromq + + or r12,r13 + movdqa xmm2,xmm4 + or r12,r8 + or r12,r9 + por xmm2,xmm5 + movq xmm3,r12 + + mov rax,QWORD[384+rsp] + lea rbx,[384+rsp] + mov r9,QWORD[((0+96))+rsp] + mov r10,QWORD[((8+96))+rsp] + lea rsi,[((0+96))+rsp] + mov r11,QWORD[((16+96))+rsp] + mov r12,QWORD[((24+96))+rsp] + lea rdi,[160+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[480+rsp] + lea rbx,[480+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((0+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[192+rsp] + call __ecp_nistz256_mul_montq + + lea rbx,[160+rsp] + lea rdi,[rsp] + call __ecp_nistz256_sub_fromq + + or r12,r13 + or r12,r8 + or r12,r9 + + movq r8,xmm2 + movq r9,xmm3 + or r12,r8 + DB 0x3e + jnz NEAR $L$add_proceedq + + + + test r9,r9 + jz NEAR $L$add_doubleq + + + + + + + movq rdi,xmm0 + pxor xmm0,xmm0 + movdqu XMMWORD[rdi],xmm0 + movdqu XMMWORD[16+rdi],xmm0 + movdqu XMMWORD[32+rdi],xmm0 + movdqu XMMWORD[48+rdi],xmm0 + movdqu XMMWORD[64+rdi],xmm0 + movdqu XMMWORD[80+rdi],xmm0 + jmp NEAR $L$add_doneq + +ALIGN 32 +$L$add_doubleq: + movq rsi,xmm1 + movq rdi,xmm0 + add rsp,416 + + jmp NEAR $L$point_double_shortcutq + + +ALIGN 32 +$L$add_proceedq: + mov rax,QWORD[((0+64))+rsp] + mov r14,QWORD[((8+64))+rsp] + lea rsi,[((0+64))+rsp] + mov r15,QWORD[((16+64))+rsp] + mov r8,QWORD[((24+64))+rsp] + lea rdi,[96+rsp] + call __ecp_nistz256_sqr_montq + + mov rax,QWORD[448+rsp] + lea rbx,[448+rsp] + mov r9,QWORD[((0+0))+rsp] + mov r10,QWORD[((8+0))+rsp] + lea rsi,[((0+0))+rsp] + mov r11,QWORD[((16+0))+rsp] + mov r12,QWORD[((24+0))+rsp] + lea rdi,[352+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[((0+0))+rsp] + mov r14,QWORD[((8+0))+rsp] + lea rsi,[((0+0))+rsp] + mov r15,QWORD[((16+0))+rsp] + mov r8,QWORD[((24+0))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_sqr_montq + + mov rax,QWORD[544+rsp] + lea rbx,[544+rsp] + mov r9,QWORD[((0+352))+rsp] + mov r10,QWORD[((8+352))+rsp] + lea rsi,[((0+352))+rsp] + mov r11,QWORD[((16+352))+rsp] + mov r12,QWORD[((24+352))+rsp] + lea rdi,[352+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[rsp] + lea rbx,[rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((0+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[128+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[160+rsp] + lea rbx,[160+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((0+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[192+rsp] + call __ecp_nistz256_mul_montq + + + + + xor r11,r11 + add r12,r12 + lea rsi,[96+rsp] + adc r13,r13 + mov rax,r12 + adc r8,r8 + adc r9,r9 + mov rbp,r13 + adc r11,0 + + sub r12,-1 + mov rcx,r8 + sbb r13,r14 + sbb r8,0 + mov r10,r9 + sbb r9,r15 + sbb r11,0 + + cmovc r12,rax + mov rax,QWORD[rsi] + cmovc r13,rbp + mov rbp,QWORD[8+rsi] + cmovc r8,rcx + mov rcx,QWORD[16+rsi] + cmovc r9,r10 + mov r10,QWORD[24+rsi] + + call __ecp_nistz256_subq + + lea rbx,[128+rsp] + lea rdi,[288+rsp] + call __ecp_nistz256_sub_fromq + + mov rax,QWORD[((192+0))+rsp] + mov rbp,QWORD[((192+8))+rsp] + mov rcx,QWORD[((192+16))+rsp] + mov r10,QWORD[((192+24))+rsp] + lea rdi,[320+rsp] + + call __ecp_nistz256_subq + + mov QWORD[rdi],r12 + mov QWORD[8+rdi],r13 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + mov rax,QWORD[128+rsp] + lea rbx,[128+rsp] + mov r9,QWORD[((0+224))+rsp] + mov r10,QWORD[((8+224))+rsp] + lea rsi,[((0+224))+rsp] + mov r11,QWORD[((16+224))+rsp] + mov r12,QWORD[((24+224))+rsp] + lea rdi,[256+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[320+rsp] + lea rbx,[320+rsp] + mov r9,QWORD[((0+64))+rsp] + mov r10,QWORD[((8+64))+rsp] + lea rsi,[((0+64))+rsp] + mov r11,QWORD[((16+64))+rsp] + mov r12,QWORD[((24+64))+rsp] + lea rdi,[320+rsp] + call __ecp_nistz256_mul_montq + + lea rbx,[256+rsp] + lea rdi,[320+rsp] + call __ecp_nistz256_sub_fromq + + movq rdi,xmm0 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[352+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((352+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[544+rsp] + pand xmm3,XMMWORD[((544+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[448+rsp] + pand xmm3,XMMWORD[((448+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[64+rdi],xmm2 + movdqu XMMWORD[80+rdi],xmm3 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[288+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((288+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[480+rsp] + pand xmm3,XMMWORD[((480+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[384+rsp] + pand xmm3,XMMWORD[((384+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[rdi],xmm2 + movdqu XMMWORD[16+rdi],xmm3 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[320+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((320+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[512+rsp] + pand xmm3,XMMWORD[((512+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[416+rsp] + pand xmm3,XMMWORD[((416+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[32+rdi],xmm2 + movdqu XMMWORD[48+rdi],xmm3 + +$L$add_doneq: + lea rsi,[((576+56))+rsp] + + mov r15,QWORD[((-48))+rsi] + + mov r14,QWORD[((-40))+rsi] + + mov r13,QWORD[((-32))+rsi] + + mov r12,QWORD[((-24))+rsi] + + mov rbx,QWORD[((-16))+rsi] + + mov rbp,QWORD[((-8))+rsi] + + lea rsp,[rsi] + +$L$point_addq_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_point_add_nohw: +global ecp_nistz256_point_add_affine_nohw + +ALIGN 32 +ecp_nistz256_point_add_affine_nohw: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_point_add_affine_nohw: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,32*15+8 + +$L$add_affineq_body: + + movdqu xmm0,XMMWORD[rsi] + mov rbx,rdx + movdqu xmm1,XMMWORD[16+rsi] + movdqu xmm2,XMMWORD[32+rsi] + movdqu xmm3,XMMWORD[48+rsi] + movdqu xmm4,XMMWORD[64+rsi] + movdqu xmm5,XMMWORD[80+rsi] + mov rax,QWORD[((64+0))+rsi] + mov r14,QWORD[((64+8))+rsi] + mov r15,QWORD[((64+16))+rsi] + mov r8,QWORD[((64+24))+rsi] + movdqa XMMWORD[320+rsp],xmm0 + movdqa XMMWORD[(320+16)+rsp],xmm1 + movdqa XMMWORD[352+rsp],xmm2 + movdqa XMMWORD[(352+16)+rsp],xmm3 + movdqa XMMWORD[384+rsp],xmm4 + movdqa XMMWORD[(384+16)+rsp],xmm5 + por xmm5,xmm4 + + movdqu xmm0,XMMWORD[rbx] + pshufd xmm3,xmm5,0xb1 + movdqu xmm1,XMMWORD[16+rbx] + movdqu xmm2,XMMWORD[32+rbx] + por xmm5,xmm3 + movdqu xmm3,XMMWORD[48+rbx] + movdqa XMMWORD[416+rsp],xmm0 + pshufd xmm4,xmm5,0x1e + movdqa XMMWORD[(416+16)+rsp],xmm1 + por xmm1,xmm0 + movq xmm0,rdi + movdqa XMMWORD[448+rsp],xmm2 + movdqa XMMWORD[(448+16)+rsp],xmm3 + por xmm3,xmm2 + por xmm5,xmm4 + pxor xmm4,xmm4 + por xmm3,xmm1 + + lea rsi,[((64-0))+rsi] + lea rdi,[32+rsp] + call __ecp_nistz256_sqr_montq + + pcmpeqd xmm5,xmm4 + pshufd xmm4,xmm3,0xb1 + mov rax,QWORD[rbx] + + mov r9,r12 + por xmm4,xmm3 + pshufd xmm5,xmm5,0 + pshufd xmm3,xmm4,0x1e + mov r10,r13 + por xmm4,xmm3 + pxor xmm3,xmm3 + mov r11,r14 + pcmpeqd xmm4,xmm3 + pshufd xmm4,xmm4,0 + + lea rsi,[((32-0))+rsp] + mov r12,r15 + lea rdi,[rsp] + call __ecp_nistz256_mul_montq + + lea rbx,[320+rsp] + lea rdi,[64+rsp] + call __ecp_nistz256_sub_fromq + + mov rax,QWORD[384+rsp] + lea rbx,[384+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((0+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[384+rsp] + lea rbx,[384+rsp] + mov r9,QWORD[((0+64))+rsp] + mov r10,QWORD[((8+64))+rsp] + lea rsi,[((0+64))+rsp] + mov r11,QWORD[((16+64))+rsp] + mov r12,QWORD[((24+64))+rsp] + lea rdi,[288+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[448+rsp] + lea rbx,[448+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((0+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_mul_montq + + lea rbx,[352+rsp] + lea rdi,[96+rsp] + call __ecp_nistz256_sub_fromq + + mov rax,QWORD[((0+64))+rsp] + mov r14,QWORD[((8+64))+rsp] + lea rsi,[((0+64))+rsp] + mov r15,QWORD[((16+64))+rsp] + mov r8,QWORD[((24+64))+rsp] + lea rdi,[128+rsp] + call __ecp_nistz256_sqr_montq + + mov rax,QWORD[((0+96))+rsp] + mov r14,QWORD[((8+96))+rsp] + lea rsi,[((0+96))+rsp] + mov r15,QWORD[((16+96))+rsp] + mov r8,QWORD[((24+96))+rsp] + lea rdi,[192+rsp] + call __ecp_nistz256_sqr_montq + + mov rax,QWORD[128+rsp] + lea rbx,[128+rsp] + mov r9,QWORD[((0+64))+rsp] + mov r10,QWORD[((8+64))+rsp] + lea rsi,[((0+64))+rsp] + mov r11,QWORD[((16+64))+rsp] + mov r12,QWORD[((24+64))+rsp] + lea rdi,[160+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[320+rsp] + lea rbx,[320+rsp] + mov r9,QWORD[((0+128))+rsp] + mov r10,QWORD[((8+128))+rsp] + lea rsi,[((0+128))+rsp] + mov r11,QWORD[((16+128))+rsp] + mov r12,QWORD[((24+128))+rsp] + lea rdi,[rsp] + call __ecp_nistz256_mul_montq + + + + + xor r11,r11 + add r12,r12 + lea rsi,[192+rsp] + adc r13,r13 + mov rax,r12 + adc r8,r8 + adc r9,r9 + mov rbp,r13 + adc r11,0 + + sub r12,-1 + mov rcx,r8 + sbb r13,r14 + sbb r8,0 + mov r10,r9 + sbb r9,r15 + sbb r11,0 + + cmovc r12,rax + mov rax,QWORD[rsi] + cmovc r13,rbp + mov rbp,QWORD[8+rsi] + cmovc r8,rcx + mov rcx,QWORD[16+rsi] + cmovc r9,r10 + mov r10,QWORD[24+rsi] + + call __ecp_nistz256_subq + + lea rbx,[160+rsp] + lea rdi,[224+rsp] + call __ecp_nistz256_sub_fromq + + mov rax,QWORD[((0+0))+rsp] + mov rbp,QWORD[((0+8))+rsp] + mov rcx,QWORD[((0+16))+rsp] + mov r10,QWORD[((0+24))+rsp] + lea rdi,[64+rsp] + + call __ecp_nistz256_subq + + mov QWORD[rdi],r12 + mov QWORD[8+rdi],r13 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + mov rax,QWORD[352+rsp] + lea rbx,[352+rsp] + mov r9,QWORD[((0+160))+rsp] + mov r10,QWORD[((8+160))+rsp] + lea rsi,[((0+160))+rsp] + mov r11,QWORD[((16+160))+rsp] + mov r12,QWORD[((24+160))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_mul_montq + + mov rax,QWORD[96+rsp] + lea rbx,[96+rsp] + mov r9,QWORD[((0+64))+rsp] + mov r10,QWORD[((8+64))+rsp] + lea rsi,[((0+64))+rsp] + mov r11,QWORD[((16+64))+rsp] + mov r12,QWORD[((24+64))+rsp] + lea rdi,[64+rsp] + call __ecp_nistz256_mul_montq + + lea rbx,[32+rsp] + lea rdi,[256+rsp] + call __ecp_nistz256_sub_fromq + + movq rdi,xmm0 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[288+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((288+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[$L$ONE_mont] + pand xmm3,XMMWORD[(($L$ONE_mont+16))] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[384+rsp] + pand xmm3,XMMWORD[((384+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[64+rdi],xmm2 + movdqu XMMWORD[80+rdi],xmm3 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[224+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((224+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[416+rsp] + pand xmm3,XMMWORD[((416+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[320+rsp] + pand xmm3,XMMWORD[((320+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[rdi],xmm2 + movdqu XMMWORD[16+rdi],xmm3 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[256+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((256+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[448+rsp] + pand xmm3,XMMWORD[((448+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[352+rsp] + pand xmm3,XMMWORD[((352+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[32+rdi],xmm2 + movdqu XMMWORD[48+rdi],xmm3 + + lea rsi,[((480+56))+rsp] + + mov r15,QWORD[((-48))+rsi] + + mov r14,QWORD[((-40))+rsi] + + mov r13,QWORD[((-32))+rsi] + + mov r12,QWORD[((-24))+rsi] + + mov rbx,QWORD[((-16))+rsi] + + mov rbp,QWORD[((-8))+rsi] + + lea rsp,[rsi] + +$L$add_affineq_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_point_add_affine_nohw: + +ALIGN 32 +__ecp_nistz256_add_tox: + + xor r11,r11 + adc r12,QWORD[rbx] + adc r13,QWORD[8+rbx] + mov rax,r12 + adc r8,QWORD[16+rbx] + adc r9,QWORD[24+rbx] + mov rbp,r13 + adc r11,0 + + xor r10,r10 + sbb r12,-1 + mov rcx,r8 + sbb r13,r14 + sbb r8,0 + mov r10,r9 + sbb r9,r15 + sbb r11,0 + + cmovc r12,rax + cmovc r13,rbp + mov QWORD[rdi],r12 + cmovc r8,rcx + mov QWORD[8+rdi],r13 + cmovc r9,r10 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + + ret + + + + +ALIGN 32 +__ecp_nistz256_sub_fromx: + + xor r11,r11 + sbb r12,QWORD[rbx] + sbb r13,QWORD[8+rbx] + mov rax,r12 + sbb r8,QWORD[16+rbx] + sbb r9,QWORD[24+rbx] + mov rbp,r13 + sbb r11,0 + + xor r10,r10 + adc r12,-1 + mov rcx,r8 + adc r13,r14 + adc r8,0 + mov r10,r9 + adc r9,r15 + + bt r11,0 + cmovnc r12,rax + cmovnc r13,rbp + mov QWORD[rdi],r12 + cmovnc r8,rcx + mov QWORD[8+rdi],r13 + cmovnc r9,r10 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + + ret + + + + +ALIGN 32 +__ecp_nistz256_subx: + + xor r11,r11 + sbb rax,r12 + sbb rbp,r13 + mov r12,rax + sbb rcx,r8 + sbb r10,r9 + mov r13,rbp + sbb r11,0 + + xor r9,r9 + adc rax,-1 + mov r8,rcx + adc rbp,r14 + adc rcx,0 + mov r9,r10 + adc r10,r15 + + bt r11,0 + cmovc r12,rax + cmovc r13,rbp + cmovc r8,rcx + cmovc r9,r10 + + ret + + + + +ALIGN 32 +__ecp_nistz256_mul_by_2x: + + xor r11,r11 + adc r12,r12 + adc r13,r13 + mov rax,r12 + adc r8,r8 + adc r9,r9 + mov rbp,r13 + adc r11,0 + + xor r10,r10 + sbb r12,-1 + mov rcx,r8 + sbb r13,r14 + sbb r8,0 + mov r10,r9 + sbb r9,r15 + sbb r11,0 + + cmovc r12,rax + cmovc r13,rbp + mov QWORD[rdi],r12 + cmovc r8,rcx + mov QWORD[8+rdi],r13 + cmovc r9,r10 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + + ret + + +global ecp_nistz256_point_double_adx + +ALIGN 32 +ecp_nistz256_point_double_adx: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_point_double_adx: + mov rdi,rcx + mov rsi,rdx + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,32*5+8 + +$L$point_doublex_body: + +$L$point_double_shortcutx: + movdqu xmm0,XMMWORD[rsi] + mov rbx,rsi + movdqu xmm1,XMMWORD[16+rsi] + mov r12,QWORD[((32+0))+rsi] + mov r13,QWORD[((32+8))+rsi] + mov r8,QWORD[((32+16))+rsi] + mov r9,QWORD[((32+24))+rsi] + mov r14,QWORD[(($L$poly+8))] + mov r15,QWORD[(($L$poly+24))] + movdqa XMMWORD[96+rsp],xmm0 + movdqa XMMWORD[(96+16)+rsp],xmm1 + lea r10,[32+rdi] + lea r11,[64+rdi] + movq xmm0,rdi + movq xmm1,r10 + movq xmm2,r11 + + lea rdi,[rsp] + call __ecp_nistz256_mul_by_2x + + mov rdx,QWORD[((64+0))+rsi] + mov r14,QWORD[((64+8))+rsi] + mov r15,QWORD[((64+16))+rsi] + mov r8,QWORD[((64+24))+rsi] + lea rsi,[((64-128))+rsi] + lea rdi,[64+rsp] + call __ecp_nistz256_sqr_montx + + mov rdx,QWORD[((0+0))+rsp] + mov r14,QWORD[((8+0))+rsp] + lea rsi,[((-128+0))+rsp] + mov r15,QWORD[((16+0))+rsp] + mov r8,QWORD[((24+0))+rsp] + lea rdi,[rsp] + call __ecp_nistz256_sqr_montx + + mov rdx,QWORD[32+rbx] + mov r9,QWORD[((64+0))+rbx] + mov r10,QWORD[((64+8))+rbx] + mov r11,QWORD[((64+16))+rbx] + mov r12,QWORD[((64+24))+rbx] + lea rsi,[((64-128))+rbx] + lea rbx,[32+rbx] + movq rdi,xmm2 + call __ecp_nistz256_mul_montx + call __ecp_nistz256_mul_by_2x + + mov r12,QWORD[((96+0))+rsp] + mov r13,QWORD[((96+8))+rsp] + lea rbx,[64+rsp] + mov r8,QWORD[((96+16))+rsp] + mov r9,QWORD[((96+24))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_add_tox + + mov r12,QWORD[((96+0))+rsp] + mov r13,QWORD[((96+8))+rsp] + lea rbx,[64+rsp] + mov r8,QWORD[((96+16))+rsp] + mov r9,QWORD[((96+24))+rsp] + lea rdi,[64+rsp] + call __ecp_nistz256_sub_fromx + + mov rdx,QWORD[((0+0))+rsp] + mov r14,QWORD[((8+0))+rsp] + lea rsi,[((-128+0))+rsp] + mov r15,QWORD[((16+0))+rsp] + mov r8,QWORD[((24+0))+rsp] + movq rdi,xmm1 + call __ecp_nistz256_sqr_montx + xor r9,r9 + mov rax,r12 + add r12,-1 + mov r10,r13 + adc r13,rsi + mov rcx,r14 + adc r14,0 + mov r8,r15 + adc r15,rbp + adc r9,0 + xor rsi,rsi + test rax,1 + + cmovz r12,rax + cmovz r13,r10 + cmovz r14,rcx + cmovz r15,r8 + cmovz r9,rsi + + mov rax,r13 + shr r12,1 + shl rax,63 + mov r10,r14 + shr r13,1 + or r12,rax + shl r10,63 + mov rcx,r15 + shr r14,1 + or r13,r10 + shl rcx,63 + mov QWORD[rdi],r12 + shr r15,1 + mov QWORD[8+rdi],r13 + shl r9,63 + or r14,rcx + or r15,r9 + mov QWORD[16+rdi],r14 + mov QWORD[24+rdi],r15 + mov rdx,QWORD[64+rsp] + lea rbx,[64+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((-128+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_mul_montx + + lea rdi,[128+rsp] + call __ecp_nistz256_mul_by_2x + + lea rbx,[32+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_add_tox + + mov rdx,QWORD[96+rsp] + lea rbx,[96+rsp] + mov r9,QWORD[((0+0))+rsp] + mov r10,QWORD[((8+0))+rsp] + lea rsi,[((-128+0))+rsp] + mov r11,QWORD[((16+0))+rsp] + mov r12,QWORD[((24+0))+rsp] + lea rdi,[rsp] + call __ecp_nistz256_mul_montx + + lea rdi,[128+rsp] + call __ecp_nistz256_mul_by_2x + + mov rdx,QWORD[((0+32))+rsp] + mov r14,QWORD[((8+32))+rsp] + lea rsi,[((-128+32))+rsp] + mov r15,QWORD[((16+32))+rsp] + mov r8,QWORD[((24+32))+rsp] + movq rdi,xmm0 + call __ecp_nistz256_sqr_montx + + lea rbx,[128+rsp] + mov r8,r14 + mov r9,r15 + mov r14,rsi + mov r15,rbp + call __ecp_nistz256_sub_fromx + + mov rax,QWORD[((0+0))+rsp] + mov rbp,QWORD[((0+8))+rsp] + mov rcx,QWORD[((0+16))+rsp] + mov r10,QWORD[((0+24))+rsp] + lea rdi,[rsp] + call __ecp_nistz256_subx + + mov rdx,QWORD[32+rsp] + lea rbx,[32+rsp] + mov r14,r12 + xor ecx,ecx + mov QWORD[((0+0))+rsp],r12 + mov r10,r13 + mov QWORD[((0+8))+rsp],r13 + cmovz r11,r8 + mov QWORD[((0+16))+rsp],r8 + lea rsi,[((0-128))+rsp] + cmovz r12,r9 + mov QWORD[((0+24))+rsp],r9 + mov r9,r14 + lea rdi,[rsp] + call __ecp_nistz256_mul_montx + + movq rbx,xmm1 + movq rdi,xmm1 + call __ecp_nistz256_sub_fromx + + lea rsi,[((160+56))+rsp] + + mov r15,QWORD[((-48))+rsi] + + mov r14,QWORD[((-40))+rsi] + + mov r13,QWORD[((-32))+rsi] + + mov r12,QWORD[((-24))+rsi] + + mov rbx,QWORD[((-16))+rsi] + + mov rbp,QWORD[((-8))+rsi] + + lea rsp,[rsi] + +$L$point_doublex_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_point_double_adx: +global ecp_nistz256_point_add_adx + +ALIGN 32 +ecp_nistz256_point_add_adx: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_point_add_adx: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,32*18+8 + +$L$point_addx_body: + + movdqu xmm0,XMMWORD[rsi] + movdqu xmm1,XMMWORD[16+rsi] + movdqu xmm2,XMMWORD[32+rsi] + movdqu xmm3,XMMWORD[48+rsi] + movdqu xmm4,XMMWORD[64+rsi] + movdqu xmm5,XMMWORD[80+rsi] + mov rbx,rsi + mov rsi,rdx + movdqa XMMWORD[384+rsp],xmm0 + movdqa XMMWORD[(384+16)+rsp],xmm1 + movdqa XMMWORD[416+rsp],xmm2 + movdqa XMMWORD[(416+16)+rsp],xmm3 + movdqa XMMWORD[448+rsp],xmm4 + movdqa XMMWORD[(448+16)+rsp],xmm5 + por xmm5,xmm4 + + movdqu xmm0,XMMWORD[rsi] + pshufd xmm3,xmm5,0xb1 + movdqu xmm1,XMMWORD[16+rsi] + movdqu xmm2,XMMWORD[32+rsi] + por xmm5,xmm3 + movdqu xmm3,XMMWORD[48+rsi] + mov rdx,QWORD[((64+0))+rsi] + mov r14,QWORD[((64+8))+rsi] + mov r15,QWORD[((64+16))+rsi] + mov r8,QWORD[((64+24))+rsi] + movdqa XMMWORD[480+rsp],xmm0 + pshufd xmm4,xmm5,0x1e + movdqa XMMWORD[(480+16)+rsp],xmm1 + movdqu xmm0,XMMWORD[64+rsi] + movdqu xmm1,XMMWORD[80+rsi] + movdqa XMMWORD[512+rsp],xmm2 + movdqa XMMWORD[(512+16)+rsp],xmm3 + por xmm5,xmm4 + pxor xmm4,xmm4 + por xmm1,xmm0 + movq xmm0,rdi + + lea rsi,[((64-128))+rsi] + mov QWORD[((544+0))+rsp],rdx + mov QWORD[((544+8))+rsp],r14 + mov QWORD[((544+16))+rsp],r15 + mov QWORD[((544+24))+rsp],r8 + lea rdi,[96+rsp] + call __ecp_nistz256_sqr_montx + + pcmpeqd xmm5,xmm4 + pshufd xmm4,xmm1,0xb1 + por xmm4,xmm1 + pshufd xmm5,xmm5,0 + pshufd xmm3,xmm4,0x1e + por xmm4,xmm3 + pxor xmm3,xmm3 + pcmpeqd xmm4,xmm3 + pshufd xmm4,xmm4,0 + mov rdx,QWORD[((64+0))+rbx] + mov r14,QWORD[((64+8))+rbx] + mov r15,QWORD[((64+16))+rbx] + mov r8,QWORD[((64+24))+rbx] + movq xmm1,rbx + + lea rsi,[((64-128))+rbx] + lea rdi,[32+rsp] + call __ecp_nistz256_sqr_montx + + mov rdx,QWORD[544+rsp] + lea rbx,[544+rsp] + mov r9,QWORD[((0+96))+rsp] + mov r10,QWORD[((8+96))+rsp] + lea rsi,[((-128+96))+rsp] + mov r11,QWORD[((16+96))+rsp] + mov r12,QWORD[((24+96))+rsp] + lea rdi,[224+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[448+rsp] + lea rbx,[448+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((-128+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[256+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[416+rsp] + lea rbx,[416+rsp] + mov r9,QWORD[((0+224))+rsp] + mov r10,QWORD[((8+224))+rsp] + lea rsi,[((-128+224))+rsp] + mov r11,QWORD[((16+224))+rsp] + mov r12,QWORD[((24+224))+rsp] + lea rdi,[224+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[512+rsp] + lea rbx,[512+rsp] + mov r9,QWORD[((0+256))+rsp] + mov r10,QWORD[((8+256))+rsp] + lea rsi,[((-128+256))+rsp] + mov r11,QWORD[((16+256))+rsp] + mov r12,QWORD[((24+256))+rsp] + lea rdi,[256+rsp] + call __ecp_nistz256_mul_montx + + lea rbx,[224+rsp] + lea rdi,[64+rsp] + call __ecp_nistz256_sub_fromx + + or r12,r13 + movdqa xmm2,xmm4 + or r12,r8 + or r12,r9 + por xmm2,xmm5 + movq xmm3,r12 + + mov rdx,QWORD[384+rsp] + lea rbx,[384+rsp] + mov r9,QWORD[((0+96))+rsp] + mov r10,QWORD[((8+96))+rsp] + lea rsi,[((-128+96))+rsp] + mov r11,QWORD[((16+96))+rsp] + mov r12,QWORD[((24+96))+rsp] + lea rdi,[160+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[480+rsp] + lea rbx,[480+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((-128+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[192+rsp] + call __ecp_nistz256_mul_montx + + lea rbx,[160+rsp] + lea rdi,[rsp] + call __ecp_nistz256_sub_fromx + + or r12,r13 + or r12,r8 + or r12,r9 + + movq r8,xmm2 + movq r9,xmm3 + or r12,r8 + DB 0x3e + jnz NEAR $L$add_proceedx + + + + test r9,r9 + jz NEAR $L$add_doublex + + + + + + + movq rdi,xmm0 + pxor xmm0,xmm0 + movdqu XMMWORD[rdi],xmm0 + movdqu XMMWORD[16+rdi],xmm0 + movdqu XMMWORD[32+rdi],xmm0 + movdqu XMMWORD[48+rdi],xmm0 + movdqu XMMWORD[64+rdi],xmm0 + movdqu XMMWORD[80+rdi],xmm0 + jmp NEAR $L$add_donex + +ALIGN 32 +$L$add_doublex: + movq rsi,xmm1 + movq rdi,xmm0 + add rsp,416 + + jmp NEAR $L$point_double_shortcutx + + +ALIGN 32 +$L$add_proceedx: + mov rdx,QWORD[((0+64))+rsp] + mov r14,QWORD[((8+64))+rsp] + lea rsi,[((-128+64))+rsp] + mov r15,QWORD[((16+64))+rsp] + mov r8,QWORD[((24+64))+rsp] + lea rdi,[96+rsp] + call __ecp_nistz256_sqr_montx + + mov rdx,QWORD[448+rsp] + lea rbx,[448+rsp] + mov r9,QWORD[((0+0))+rsp] + mov r10,QWORD[((8+0))+rsp] + lea rsi,[((-128+0))+rsp] + mov r11,QWORD[((16+0))+rsp] + mov r12,QWORD[((24+0))+rsp] + lea rdi,[352+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[((0+0))+rsp] + mov r14,QWORD[((8+0))+rsp] + lea rsi,[((-128+0))+rsp] + mov r15,QWORD[((16+0))+rsp] + mov r8,QWORD[((24+0))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_sqr_montx + + mov rdx,QWORD[544+rsp] + lea rbx,[544+rsp] + mov r9,QWORD[((0+352))+rsp] + mov r10,QWORD[((8+352))+rsp] + lea rsi,[((-128+352))+rsp] + mov r11,QWORD[((16+352))+rsp] + mov r12,QWORD[((24+352))+rsp] + lea rdi,[352+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[rsp] + lea rbx,[rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((-128+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[128+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[160+rsp] + lea rbx,[160+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((-128+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[192+rsp] + call __ecp_nistz256_mul_montx + + + + + xor r11,r11 + add r12,r12 + lea rsi,[96+rsp] + adc r13,r13 + mov rax,r12 + adc r8,r8 + adc r9,r9 + mov rbp,r13 + adc r11,0 + + sub r12,-1 + mov rcx,r8 + sbb r13,r14 + sbb r8,0 + mov r10,r9 + sbb r9,r15 + sbb r11,0 + + cmovc r12,rax + mov rax,QWORD[rsi] + cmovc r13,rbp + mov rbp,QWORD[8+rsi] + cmovc r8,rcx + mov rcx,QWORD[16+rsi] + cmovc r9,r10 + mov r10,QWORD[24+rsi] + + call __ecp_nistz256_subx + + lea rbx,[128+rsp] + lea rdi,[288+rsp] + call __ecp_nistz256_sub_fromx + + mov rax,QWORD[((192+0))+rsp] + mov rbp,QWORD[((192+8))+rsp] + mov rcx,QWORD[((192+16))+rsp] + mov r10,QWORD[((192+24))+rsp] + lea rdi,[320+rsp] + + call __ecp_nistz256_subx + + mov QWORD[rdi],r12 + mov QWORD[8+rdi],r13 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + mov rdx,QWORD[128+rsp] + lea rbx,[128+rsp] + mov r9,QWORD[((0+224))+rsp] + mov r10,QWORD[((8+224))+rsp] + lea rsi,[((-128+224))+rsp] + mov r11,QWORD[((16+224))+rsp] + mov r12,QWORD[((24+224))+rsp] + lea rdi,[256+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[320+rsp] + lea rbx,[320+rsp] + mov r9,QWORD[((0+64))+rsp] + mov r10,QWORD[((8+64))+rsp] + lea rsi,[((-128+64))+rsp] + mov r11,QWORD[((16+64))+rsp] + mov r12,QWORD[((24+64))+rsp] + lea rdi,[320+rsp] + call __ecp_nistz256_mul_montx + + lea rbx,[256+rsp] + lea rdi,[320+rsp] + call __ecp_nistz256_sub_fromx + + movq rdi,xmm0 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[352+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((352+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[544+rsp] + pand xmm3,XMMWORD[((544+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[448+rsp] + pand xmm3,XMMWORD[((448+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[64+rdi],xmm2 + movdqu XMMWORD[80+rdi],xmm3 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[288+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((288+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[480+rsp] + pand xmm3,XMMWORD[((480+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[384+rsp] + pand xmm3,XMMWORD[((384+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[rdi],xmm2 + movdqu XMMWORD[16+rdi],xmm3 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[320+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((320+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[512+rsp] + pand xmm3,XMMWORD[((512+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[416+rsp] + pand xmm3,XMMWORD[((416+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[32+rdi],xmm2 + movdqu XMMWORD[48+rdi],xmm3 + +$L$add_donex: + lea rsi,[((576+56))+rsp] + + mov r15,QWORD[((-48))+rsi] + + mov r14,QWORD[((-40))+rsi] + + mov r13,QWORD[((-32))+rsi] + + mov r12,QWORD[((-24))+rsi] + + mov rbx,QWORD[((-16))+rsi] + + mov rbp,QWORD[((-8))+rsi] + + lea rsp,[rsi] + +$L$point_addx_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_point_add_adx: +global ecp_nistz256_point_add_affine_adx + +ALIGN 32 +ecp_nistz256_point_add_affine_adx: + mov QWORD[8+rsp],rdi ;WIN64 prologue + mov QWORD[16+rsp],rsi + mov rax,rsp +$L$SEH_begin_ecp_nistz256_point_add_affine_adx: + mov rdi,rcx + mov rsi,rdx + mov rdx,r8 + + + +_CET_ENDBR + push rbp + + push rbx + + push r12 + + push r13 + + push r14 + + push r15 + + sub rsp,32*15+8 + +$L$add_affinex_body: + + movdqu xmm0,XMMWORD[rsi] + mov rbx,rdx + movdqu xmm1,XMMWORD[16+rsi] + movdqu xmm2,XMMWORD[32+rsi] + movdqu xmm3,XMMWORD[48+rsi] + movdqu xmm4,XMMWORD[64+rsi] + movdqu xmm5,XMMWORD[80+rsi] + mov rdx,QWORD[((64+0))+rsi] + mov r14,QWORD[((64+8))+rsi] + mov r15,QWORD[((64+16))+rsi] + mov r8,QWORD[((64+24))+rsi] + movdqa XMMWORD[320+rsp],xmm0 + movdqa XMMWORD[(320+16)+rsp],xmm1 + movdqa XMMWORD[352+rsp],xmm2 + movdqa XMMWORD[(352+16)+rsp],xmm3 + movdqa XMMWORD[384+rsp],xmm4 + movdqa XMMWORD[(384+16)+rsp],xmm5 + por xmm5,xmm4 + + movdqu xmm0,XMMWORD[rbx] + pshufd xmm3,xmm5,0xb1 + movdqu xmm1,XMMWORD[16+rbx] + movdqu xmm2,XMMWORD[32+rbx] + por xmm5,xmm3 + movdqu xmm3,XMMWORD[48+rbx] + movdqa XMMWORD[416+rsp],xmm0 + pshufd xmm4,xmm5,0x1e + movdqa XMMWORD[(416+16)+rsp],xmm1 + por xmm1,xmm0 + movq xmm0,rdi + movdqa XMMWORD[448+rsp],xmm2 + movdqa XMMWORD[(448+16)+rsp],xmm3 + por xmm3,xmm2 + por xmm5,xmm4 + pxor xmm4,xmm4 + por xmm3,xmm1 + + lea rsi,[((64-128))+rsi] + lea rdi,[32+rsp] + call __ecp_nistz256_sqr_montx + + pcmpeqd xmm5,xmm4 + pshufd xmm4,xmm3,0xb1 + mov rdx,QWORD[rbx] + + mov r9,r12 + por xmm4,xmm3 + pshufd xmm5,xmm5,0 + pshufd xmm3,xmm4,0x1e + mov r10,r13 + por xmm4,xmm3 + pxor xmm3,xmm3 + mov r11,r14 + pcmpeqd xmm4,xmm3 + pshufd xmm4,xmm4,0 + + lea rsi,[((32-128))+rsp] + mov r12,r15 + lea rdi,[rsp] + call __ecp_nistz256_mul_montx + + lea rbx,[320+rsp] + lea rdi,[64+rsp] + call __ecp_nistz256_sub_fromx + + mov rdx,QWORD[384+rsp] + lea rbx,[384+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((-128+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[384+rsp] + lea rbx,[384+rsp] + mov r9,QWORD[((0+64))+rsp] + mov r10,QWORD[((8+64))+rsp] + lea rsi,[((-128+64))+rsp] + mov r11,QWORD[((16+64))+rsp] + mov r12,QWORD[((24+64))+rsp] + lea rdi,[288+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[448+rsp] + lea rbx,[448+rsp] + mov r9,QWORD[((0+32))+rsp] + mov r10,QWORD[((8+32))+rsp] + lea rsi,[((-128+32))+rsp] + mov r11,QWORD[((16+32))+rsp] + mov r12,QWORD[((24+32))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_mul_montx + + lea rbx,[352+rsp] + lea rdi,[96+rsp] + call __ecp_nistz256_sub_fromx + + mov rdx,QWORD[((0+64))+rsp] + mov r14,QWORD[((8+64))+rsp] + lea rsi,[((-128+64))+rsp] + mov r15,QWORD[((16+64))+rsp] + mov r8,QWORD[((24+64))+rsp] + lea rdi,[128+rsp] + call __ecp_nistz256_sqr_montx + + mov rdx,QWORD[((0+96))+rsp] + mov r14,QWORD[((8+96))+rsp] + lea rsi,[((-128+96))+rsp] + mov r15,QWORD[((16+96))+rsp] + mov r8,QWORD[((24+96))+rsp] + lea rdi,[192+rsp] + call __ecp_nistz256_sqr_montx + + mov rdx,QWORD[128+rsp] + lea rbx,[128+rsp] + mov r9,QWORD[((0+64))+rsp] + mov r10,QWORD[((8+64))+rsp] + lea rsi,[((-128+64))+rsp] + mov r11,QWORD[((16+64))+rsp] + mov r12,QWORD[((24+64))+rsp] + lea rdi,[160+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[320+rsp] + lea rbx,[320+rsp] + mov r9,QWORD[((0+128))+rsp] + mov r10,QWORD[((8+128))+rsp] + lea rsi,[((-128+128))+rsp] + mov r11,QWORD[((16+128))+rsp] + mov r12,QWORD[((24+128))+rsp] + lea rdi,[rsp] + call __ecp_nistz256_mul_montx + + + + + xor r11,r11 + add r12,r12 + lea rsi,[192+rsp] + adc r13,r13 + mov rax,r12 + adc r8,r8 + adc r9,r9 + mov rbp,r13 + adc r11,0 + + sub r12,-1 + mov rcx,r8 + sbb r13,r14 + sbb r8,0 + mov r10,r9 + sbb r9,r15 + sbb r11,0 + + cmovc r12,rax + mov rax,QWORD[rsi] + cmovc r13,rbp + mov rbp,QWORD[8+rsi] + cmovc r8,rcx + mov rcx,QWORD[16+rsi] + cmovc r9,r10 + mov r10,QWORD[24+rsi] + + call __ecp_nistz256_subx + + lea rbx,[160+rsp] + lea rdi,[224+rsp] + call __ecp_nistz256_sub_fromx + + mov rax,QWORD[((0+0))+rsp] + mov rbp,QWORD[((0+8))+rsp] + mov rcx,QWORD[((0+16))+rsp] + mov r10,QWORD[((0+24))+rsp] + lea rdi,[64+rsp] + + call __ecp_nistz256_subx + + mov QWORD[rdi],r12 + mov QWORD[8+rdi],r13 + mov QWORD[16+rdi],r8 + mov QWORD[24+rdi],r9 + mov rdx,QWORD[352+rsp] + lea rbx,[352+rsp] + mov r9,QWORD[((0+160))+rsp] + mov r10,QWORD[((8+160))+rsp] + lea rsi,[((-128+160))+rsp] + mov r11,QWORD[((16+160))+rsp] + mov r12,QWORD[((24+160))+rsp] + lea rdi,[32+rsp] + call __ecp_nistz256_mul_montx + + mov rdx,QWORD[96+rsp] + lea rbx,[96+rsp] + mov r9,QWORD[((0+64))+rsp] + mov r10,QWORD[((8+64))+rsp] + lea rsi,[((-128+64))+rsp] + mov r11,QWORD[((16+64))+rsp] + mov r12,QWORD[((24+64))+rsp] + lea rdi,[64+rsp] + call __ecp_nistz256_mul_montx + + lea rbx,[32+rsp] + lea rdi,[256+rsp] + call __ecp_nistz256_sub_fromx + + movq rdi,xmm0 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[288+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((288+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[$L$ONE_mont] + pand xmm3,XMMWORD[(($L$ONE_mont+16))] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[384+rsp] + pand xmm3,XMMWORD[((384+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[64+rdi],xmm2 + movdqu XMMWORD[80+rdi],xmm3 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[224+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((224+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[416+rsp] + pand xmm3,XMMWORD[((416+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[320+rsp] + pand xmm3,XMMWORD[((320+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[rdi],xmm2 + movdqu XMMWORD[16+rdi],xmm3 + + movdqa xmm0,xmm5 + movdqa xmm1,xmm5 + pandn xmm0,XMMWORD[256+rsp] + movdqa xmm2,xmm5 + pandn xmm1,XMMWORD[((256+16))+rsp] + movdqa xmm3,xmm5 + pand xmm2,XMMWORD[448+rsp] + pand xmm3,XMMWORD[((448+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + + movdqa xmm0,xmm4 + movdqa xmm1,xmm4 + pandn xmm0,xmm2 + movdqa xmm2,xmm4 + pandn xmm1,xmm3 + movdqa xmm3,xmm4 + pand xmm2,XMMWORD[352+rsp] + pand xmm3,XMMWORD[((352+16))+rsp] + por xmm2,xmm0 + por xmm3,xmm1 + movdqu XMMWORD[32+rdi],xmm2 + movdqu XMMWORD[48+rdi],xmm3 + + lea rsi,[((480+56))+rsp] + + mov r15,QWORD[((-48))+rsi] + + mov r14,QWORD[((-40))+rsi] + + mov r13,QWORD[((-32))+rsi] + + mov r12,QWORD[((-24))+rsi] + + mov rbx,QWORD[((-16))+rsi] + + mov rbp,QWORD[((-8))+rsi] + + lea rsp,[rsi] + +$L$add_affinex_epilogue: + mov rdi,QWORD[8+rsp] ;WIN64 epilogue + mov rsi,QWORD[16+rsp] + ret + +$L$SEH_end_ecp_nistz256_point_add_affine_adx: EXTERN __imp_RtlVirtualUnwind @@ -1263,6 +4893,10 @@ section .pdata rdata align=4 ALIGN 4 + DD $L$SEH_begin_ecp_nistz256_neg wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_neg wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_neg wrt ..imagebase + DD $L$SEH_begin_ecp_nistz256_ord_mul_mont_nohw wrt ..imagebase DD $L$SEH_end_ecp_nistz256_ord_mul_mont_nohw wrt ..imagebase DD $L$SEH_info_ecp_nistz256_ord_mul_mont_nohw wrt ..imagebase @@ -1277,9 +4911,63 @@ DD $L$SEH_begin_ecp_nistz256_ord_sqr_mont_adx wrt ..imagebase DD $L$SEH_end_ecp_nistz256_ord_sqr_mont_adx wrt ..imagebase DD $L$SEH_info_ecp_nistz256_ord_sqr_mont_adx wrt ..imagebase + DD $L$SEH_begin_ecp_nistz256_mul_mont_nohw wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_mul_mont_nohw wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_mul_mont_nohw wrt ..imagebase + + DD $L$SEH_begin_ecp_nistz256_sqr_mont_nohw wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_sqr_mont_nohw wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_sqr_mont_nohw wrt ..imagebase + DD $L$SEH_begin_ecp_nistz256_mul_mont_adx wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_mul_mont_adx wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_mul_mont_adx wrt ..imagebase + + DD $L$SEH_begin_ecp_nistz256_sqr_mont_adx wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_sqr_mont_adx wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_sqr_mont_adx wrt ..imagebase + DD $L$SEH_begin_ecp_nistz256_select_w5_nohw wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_select_w5_nohw wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_select_wX_nohw wrt ..imagebase + + DD $L$SEH_begin_ecp_nistz256_select_w7_nohw wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_select_w7_nohw wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_select_wX_nohw wrt ..imagebase + DD $L$SEH_begin_ecp_nistz256_select_w5_avx2 wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_select_w5_avx2 wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_select_wX_avx2 wrt ..imagebase + + DD $L$SEH_begin_ecp_nistz256_select_w7_avx2 wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_select_w7_avx2 wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_select_wX_avx2 wrt ..imagebase + DD $L$SEH_begin_ecp_nistz256_point_double_nohw wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_point_double_nohw wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_point_double_nohw wrt ..imagebase + + DD $L$SEH_begin_ecp_nistz256_point_add_nohw wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_point_add_nohw wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_point_add_nohw wrt ..imagebase + + DD $L$SEH_begin_ecp_nistz256_point_add_affine_nohw wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_point_add_affine_nohw wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_point_add_affine_nohw wrt ..imagebase + DD $L$SEH_begin_ecp_nistz256_point_double_adx wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_point_double_adx wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_point_double_adx wrt ..imagebase + + DD $L$SEH_begin_ecp_nistz256_point_add_adx wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_point_add_adx wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_point_add_adx wrt ..imagebase + + DD $L$SEH_begin_ecp_nistz256_point_add_affine_adx wrt ..imagebase + DD $L$SEH_end_ecp_nistz256_point_add_affine_adx wrt ..imagebase + DD $L$SEH_info_ecp_nistz256_point_add_affine_adx wrt ..imagebase section .xdata rdata align=8 ALIGN 8 +$L$SEH_info_ecp_nistz256_neg: + DB 9,0,0,0 + DD short_handler wrt ..imagebase + DD $L$neg_body wrt ..imagebase,$L$neg_epilogue wrt ..imagebase $L$SEH_info_ecp_nistz256_ord_mul_mont_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase @@ -1300,6 +4988,86 @@ DD full_handler wrt ..imagebase DD $L$ord_sqrx_body wrt ..imagebase,$L$ord_sqrx_epilogue wrt ..imagebase DD 48,0 +$L$SEH_info_ecp_nistz256_mul_mont_nohw: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase + DD 48,0 +$L$SEH_info_ecp_nistz256_sqr_mont_nohw: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$sqr_body wrt ..imagebase,$L$sqr_epilogue wrt ..imagebase + DD 48,0 +$L$SEH_info_ecp_nistz256_mul_mont_adx: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$mulx_body wrt ..imagebase,$L$mulx_epilogue wrt ..imagebase + DD 48,0 +$L$SEH_info_ecp_nistz256_sqr_mont_adx: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$sqrx_body wrt ..imagebase,$L$sqrx_epilogue wrt ..imagebase + DD 48,0 +$L$SEH_info_ecp_nistz256_select_wX_nohw: + DB 0x01,0x33,0x16,0x00 + DB 0x33,0xf8,0x09,0x00 + DB 0x2e,0xe8,0x08,0x00 + DB 0x29,0xd8,0x07,0x00 + DB 0x24,0xc8,0x06,0x00 + DB 0x1f,0xb8,0x05,0x00 + DB 0x1a,0xa8,0x04,0x00 + DB 0x15,0x98,0x03,0x00 + DB 0x10,0x88,0x02,0x00 + DB 0x0c,0x78,0x01,0x00 + DB 0x08,0x68,0x00,0x00 + DB 0x04,0x01,0x15,0x00 +ALIGN 8 +$L$SEH_info_ecp_nistz256_select_wX_avx2: + DB 0x01,0x36,0x17,0x0b + DB 0x36,0xf8,0x09,0x00 + DB 0x31,0xe8,0x08,0x00 + DB 0x2c,0xd8,0x07,0x00 + DB 0x27,0xc8,0x06,0x00 + DB 0x22,0xb8,0x05,0x00 + DB 0x1d,0xa8,0x04,0x00 + DB 0x18,0x98,0x03,0x00 + DB 0x13,0x88,0x02,0x00 + DB 0x0e,0x78,0x01,0x00 + DB 0x09,0x68,0x00,0x00 + DB 0x04,0x01,0x15,0x00 + DB 0x00,0xb3,0x00,0x00 +ALIGN 8 +$L$SEH_info_ecp_nistz256_point_double_nohw: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$point_doubleq_body wrt ..imagebase,$L$point_doubleq_epilogue wrt ..imagebase + DD 32*5+56,0 +$L$SEH_info_ecp_nistz256_point_add_nohw: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$point_addq_body wrt ..imagebase,$L$point_addq_epilogue wrt ..imagebase + DD 32*18+56,0 +$L$SEH_info_ecp_nistz256_point_add_affine_nohw: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$add_affineq_body wrt ..imagebase,$L$add_affineq_epilogue wrt ..imagebase + DD 32*15+56,0 +ALIGN 8 +$L$SEH_info_ecp_nistz256_point_double_adx: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$point_doublex_body wrt ..imagebase,$L$point_doublex_epilogue wrt ..imagebase + DD 32*5+56,0 +$L$SEH_info_ecp_nistz256_point_add_adx: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$point_addx_body wrt ..imagebase,$L$point_addx_epilogue wrt ..imagebase + DD 32*18+56,0 +$L$SEH_info_ecp_nistz256_point_add_affine_adx: + DB 9,0,0,0 + DD full_handler wrt ..imagebase + DD $L$add_affinex_body wrt ..imagebase,$L$add_affinex_epilogue wrt ..imagebase + DD 32*15+56,0 %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret
diff --git a/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc b/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc index 7ab85df..e29d4be 100644 --- a/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc +++ b/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc
@@ -227,14 +227,38 @@ %define chacha20_poly1305_seal BORINGSSL_PREFIX %+ _chacha20_poly1305_seal %define chacha20_poly1305_seal_avx2 BORINGSSL_PREFIX %+ _chacha20_poly1305_seal_avx2 %define chacha20_poly1305_seal_sse41 BORINGSSL_PREFIX %+ _chacha20_poly1305_seal_sse41 +%define ecp_nistz256_div_by_2 BORINGSSL_PREFIX %+ _ecp_nistz256_div_by_2 +%define ecp_nistz256_mul_by_2 BORINGSSL_PREFIX %+ _ecp_nistz256_mul_by_2 +%define ecp_nistz256_mul_by_3 BORINGSSL_PREFIX %+ _ecp_nistz256_mul_by_3 %define ecp_nistz256_mul_mont BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont +%define ecp_nistz256_mul_mont_adx BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont_adx +%define ecp_nistz256_mul_mont_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont_nohw +%define ecp_nistz256_neg BORINGSSL_PREFIX %+ _ecp_nistz256_neg %define ecp_nistz256_ord_mul_mont BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont %define ecp_nistz256_ord_mul_mont_adx BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont_adx %define ecp_nistz256_ord_mul_mont_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont_nohw %define ecp_nistz256_ord_sqr_mont BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont %define ecp_nistz256_ord_sqr_mont_adx BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont_adx %define ecp_nistz256_ord_sqr_mont_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont_nohw +%define ecp_nistz256_point_add BORINGSSL_PREFIX %+ _ecp_nistz256_point_add +%define ecp_nistz256_point_add_adx BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_adx +%define ecp_nistz256_point_add_affine BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine +%define ecp_nistz256_point_add_affine_adx BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine_adx +%define ecp_nistz256_point_add_affine_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine_nohw +%define ecp_nistz256_point_add_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_nohw +%define ecp_nistz256_point_double BORINGSSL_PREFIX %+ _ecp_nistz256_point_double +%define ecp_nistz256_point_double_adx BORINGSSL_PREFIX %+ _ecp_nistz256_point_double_adx +%define ecp_nistz256_point_double_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_point_double_nohw +%define ecp_nistz256_select_w5 BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5 +%define ecp_nistz256_select_w5_avx2 BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5_avx2 +%define ecp_nistz256_select_w5_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5_nohw +%define ecp_nistz256_select_w7 BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7 +%define ecp_nistz256_select_w7_avx2 BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7_avx2 +%define ecp_nistz256_select_w7_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7_nohw %define ecp_nistz256_sqr_mont BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont +%define ecp_nistz256_sqr_mont_adx BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont_adx +%define ecp_nistz256_sqr_mont_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont_nohw +%define ecp_nistz256_sub BORINGSSL_PREFIX %+ _ecp_nistz256_sub %define fiat_curve25519_adx_mul BORINGSSL_PREFIX %+ _fiat_curve25519_adx_mul %define fiat_curve25519_adx_square BORINGSSL_PREFIX %+ _fiat_curve25519_adx_square %define fiat_p256_adx_mul BORINGSSL_PREFIX %+ _fiat_p256_adx_mul
diff --git a/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc b/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc index 24eb900..060a50b 100644 --- a/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc +++ b/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc
@@ -227,14 +227,38 @@ %define _chacha20_poly1305_seal _ %+ BORINGSSL_PREFIX %+ _chacha20_poly1305_seal %define _chacha20_poly1305_seal_avx2 _ %+ BORINGSSL_PREFIX %+ _chacha20_poly1305_seal_avx2 %define _chacha20_poly1305_seal_sse41 _ %+ BORINGSSL_PREFIX %+ _chacha20_poly1305_seal_sse41 +%define _ecp_nistz256_div_by_2 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_div_by_2 +%define _ecp_nistz256_mul_by_2 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_by_2 +%define _ecp_nistz256_mul_by_3 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_by_3 %define _ecp_nistz256_mul_mont _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont +%define _ecp_nistz256_mul_mont_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont_adx +%define _ecp_nistz256_mul_mont_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont_nohw +%define _ecp_nistz256_neg _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_neg %define _ecp_nistz256_ord_mul_mont _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont %define _ecp_nistz256_ord_mul_mont_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont_adx %define _ecp_nistz256_ord_mul_mont_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont_nohw %define _ecp_nistz256_ord_sqr_mont _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont %define _ecp_nistz256_ord_sqr_mont_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont_adx %define _ecp_nistz256_ord_sqr_mont_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont_nohw +%define _ecp_nistz256_point_add _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add +%define _ecp_nistz256_point_add_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_adx +%define _ecp_nistz256_point_add_affine _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine +%define _ecp_nistz256_point_add_affine_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine_adx +%define _ecp_nistz256_point_add_affine_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine_nohw +%define _ecp_nistz256_point_add_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_nohw +%define _ecp_nistz256_point_double _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_double +%define _ecp_nistz256_point_double_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_double_adx +%define _ecp_nistz256_point_double_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_double_nohw +%define _ecp_nistz256_select_w5 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5 +%define _ecp_nistz256_select_w5_avx2 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5_avx2 +%define _ecp_nistz256_select_w5_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5_nohw +%define _ecp_nistz256_select_w7 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7 +%define _ecp_nistz256_select_w7_avx2 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7_avx2 +%define _ecp_nistz256_select_w7_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7_nohw %define _ecp_nistz256_sqr_mont _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont +%define _ecp_nistz256_sqr_mont_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont_adx +%define _ecp_nistz256_sqr_mont_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont_nohw +%define _ecp_nistz256_sub _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_sub %define _fiat_curve25519_adx_mul _ %+ BORINGSSL_PREFIX %+ _fiat_curve25519_adx_mul %define _fiat_curve25519_adx_square _ %+ BORINGSSL_PREFIX %+ _fiat_curve25519_adx_square %define _fiat_p256_adx_mul _ %+ BORINGSSL_PREFIX %+ _fiat_p256_adx_mul
diff --git a/gen/sources.bzl b/gen/sources.bzl index b932a75..dc1e31d 100644 --- a/gen/sources.bzl +++ b/gen/sources.bzl
@@ -65,6 +65,7 @@ "crypto/fipsmodule/ec/ec_montgomery.cc.inc", "crypto/fipsmodule/ec/felem.cc.inc", "crypto/fipsmodule/ec/oct.cc.inc", + "crypto/fipsmodule/ec/p256-nistz.cc.inc", "crypto/fipsmodule/ec/p256.cc.inc", "crypto/fipsmodule/ec/scalar.cc.inc", "crypto/fipsmodule/ec/simple.cc.inc", @@ -656,7 +657,7 @@ "crypto/fipsmodule/ec/builtin_curves.h", "crypto/fipsmodule/ec/internal.h", "crypto/fipsmodule/ec/p256-nistz-table.h", - "crypto/fipsmodule/ec/p256_internal.h", + "crypto/fipsmodule/ec/p256-nistz.h", "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/entropy/internal.h", @@ -703,6 +704,7 @@ "third_party/fiat/p256_32.h", "third_party/fiat/p256_64.h", "third_party/fiat/p256_64_msvc.h", + "third_party/fiat/p256_field.c.inc", "third_party/fiat/p256_field_32.br.c.inc", "third_party/fiat/p256_field_64.br.c.inc", "third_party/fiat/p256_point.br.c.inc", @@ -779,6 +781,7 @@ "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", + "crypto/fipsmodule/ec/p256-nistz_test.cc", "crypto/fipsmodule/ec/p256_test.cc", "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/entropy/jitter_test.cc", @@ -893,7 +896,7 @@ "crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt", "crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt", "crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt", - "crypto/fipsmodule/ec/p256-tests.txt", + "crypto/fipsmodule/ec/p256-nistz_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt", "crypto/fipsmodule/keccak/keccak_tests.txt",
diff --git a/gen/sources.cmake b/gen/sources.cmake index 35f3f32..634d641 100644 --- a/gen/sources.cmake +++ b/gen/sources.cmake
@@ -69,6 +69,7 @@ crypto/fipsmodule/ec/ec_montgomery.cc.inc crypto/fipsmodule/ec/felem.cc.inc crypto/fipsmodule/ec/oct.cc.inc + crypto/fipsmodule/ec/p256-nistz.cc.inc crypto/fipsmodule/ec/p256.cc.inc crypto/fipsmodule/ec/scalar.cc.inc crypto/fipsmodule/ec/simple.cc.inc @@ -678,7 +679,7 @@ crypto/fipsmodule/ec/builtin_curves.h crypto/fipsmodule/ec/internal.h crypto/fipsmodule/ec/p256-nistz-table.h - crypto/fipsmodule/ec/p256_internal.h + crypto/fipsmodule/ec/p256-nistz.h crypto/fipsmodule/ec/p256_table.h crypto/fipsmodule/ecdsa/internal.h crypto/fipsmodule/entropy/internal.h @@ -725,6 +726,7 @@ third_party/fiat/p256_32.h third_party/fiat/p256_64.h third_party/fiat/p256_64_msvc.h + third_party/fiat/p256_field.c.inc third_party/fiat/p256_field_32.br.c.inc third_party/fiat/p256_field_64.br.c.inc third_party/fiat/p256_point.br.c.inc @@ -807,6 +809,7 @@ crypto/fipsmodule/bn/bn_test.cc crypto/fipsmodule/cmac/cmac_test.cc crypto/fipsmodule/ec/ec_test.cc + crypto/fipsmodule/ec/p256-nistz_test.cc crypto/fipsmodule/ec/p256_test.cc crypto/fipsmodule/ecdsa/ecdsa_test.cc crypto/fipsmodule/entropy/jitter_test.cc @@ -925,7 +928,7 @@ crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt - crypto/fipsmodule/ec/p256-tests.txt + crypto/fipsmodule/ec/p256-nistz_tests.txt crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt crypto/fipsmodule/keccak/keccak_tests.txt
diff --git a/gen/sources.gni b/gen/sources.gni index 729fb9d..7500262 100644 --- a/gen/sources.gni +++ b/gen/sources.gni
@@ -65,6 +65,7 @@ "crypto/fipsmodule/ec/ec_montgomery.cc.inc", "crypto/fipsmodule/ec/felem.cc.inc", "crypto/fipsmodule/ec/oct.cc.inc", + "crypto/fipsmodule/ec/p256-nistz.cc.inc", "crypto/fipsmodule/ec/p256.cc.inc", "crypto/fipsmodule/ec/scalar.cc.inc", "crypto/fipsmodule/ec/simple.cc.inc", @@ -656,7 +657,7 @@ "crypto/fipsmodule/ec/builtin_curves.h", "crypto/fipsmodule/ec/internal.h", "crypto/fipsmodule/ec/p256-nistz-table.h", - "crypto/fipsmodule/ec/p256_internal.h", + "crypto/fipsmodule/ec/p256-nistz.h", "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/entropy/internal.h", @@ -703,6 +704,7 @@ "third_party/fiat/p256_32.h", "third_party/fiat/p256_64.h", "third_party/fiat/p256_64_msvc.h", + "third_party/fiat/p256_field.c.inc", "third_party/fiat/p256_field_32.br.c.inc", "third_party/fiat/p256_field_64.br.c.inc", "third_party/fiat/p256_point.br.c.inc", @@ -779,6 +781,7 @@ "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", + "crypto/fipsmodule/ec/p256-nistz_test.cc", "crypto/fipsmodule/ec/p256_test.cc", "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/entropy/jitter_test.cc", @@ -893,7 +896,7 @@ "crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt", "crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt", "crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt", - "crypto/fipsmodule/ec/p256-tests.txt", + "crypto/fipsmodule/ec/p256-nistz_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt", "crypto/fipsmodule/keccak/keccak_tests.txt",
diff --git a/gen/sources.json b/gen/sources.json index 67c64d8..ca85acb 100644 --- a/gen/sources.json +++ b/gen/sources.json
@@ -50,6 +50,7 @@ "crypto/fipsmodule/ec/ec_montgomery.cc.inc", "crypto/fipsmodule/ec/felem.cc.inc", "crypto/fipsmodule/ec/oct.cc.inc", + "crypto/fipsmodule/ec/p256-nistz.cc.inc", "crypto/fipsmodule/ec/p256.cc.inc", "crypto/fipsmodule/ec/scalar.cc.inc", "crypto/fipsmodule/ec/simple.cc.inc", @@ -638,7 +639,7 @@ "crypto/fipsmodule/ec/builtin_curves.h", "crypto/fipsmodule/ec/internal.h", "crypto/fipsmodule/ec/p256-nistz-table.h", - "crypto/fipsmodule/ec/p256_internal.h", + "crypto/fipsmodule/ec/p256-nistz.h", "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/entropy/internal.h", @@ -685,6 +686,7 @@ "third_party/fiat/p256_32.h", "third_party/fiat/p256_64.h", "third_party/fiat/p256_64_msvc.h", + "third_party/fiat/p256_field.c.inc", "third_party/fiat/p256_field_32.br.c.inc", "third_party/fiat/p256_field_64.br.c.inc", "third_party/fiat/p256_point.br.c.inc" @@ -760,6 +762,7 @@ "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", + "crypto/fipsmodule/ec/p256-nistz_test.cc", "crypto/fipsmodule/ec/p256_test.cc", "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/entropy/jitter_test.cc", @@ -872,7 +875,7 @@ "crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt", "crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt", "crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt", - "crypto/fipsmodule/ec/p256-tests.txt", + "crypto/fipsmodule/ec/p256-nistz_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt", "crypto/fipsmodule/keccak/keccak_tests.txt",
diff --git a/gen/sources.mk b/gen/sources.mk index d66c688..e150461 100644 --- a/gen/sources.mk +++ b/gen/sources.mk
@@ -64,6 +64,7 @@ crypto/fipsmodule/ec/ec_montgomery.cc.inc \ crypto/fipsmodule/ec/felem.cc.inc \ crypto/fipsmodule/ec/oct.cc.inc \ + crypto/fipsmodule/ec/p256-nistz.cc.inc \ crypto/fipsmodule/ec/p256.cc.inc \ crypto/fipsmodule/ec/scalar.cc.inc \ crypto/fipsmodule/ec/simple.cc.inc \ @@ -646,7 +647,7 @@ crypto/fipsmodule/ec/builtin_curves.h \ crypto/fipsmodule/ec/internal.h \ crypto/fipsmodule/ec/p256-nistz-table.h \ - crypto/fipsmodule/ec/p256_internal.h \ + crypto/fipsmodule/ec/p256-nistz.h \ crypto/fipsmodule/ec/p256_table.h \ crypto/fipsmodule/ecdsa/internal.h \ crypto/fipsmodule/entropy/internal.h \ @@ -693,6 +694,7 @@ third_party/fiat/p256_32.h \ third_party/fiat/p256_64.h \ third_party/fiat/p256_64_msvc.h \ + third_party/fiat/p256_field.c.inc \ third_party/fiat/p256_field_32.br.c.inc \ third_party/fiat/p256_field_64.br.c.inc \ third_party/fiat/p256_point.br.c.inc @@ -766,6 +768,7 @@ crypto/fipsmodule/bn/bn_test.cc \ crypto/fipsmodule/cmac/cmac_test.cc \ crypto/fipsmodule/ec/ec_test.cc \ + crypto/fipsmodule/ec/p256-nistz_test.cc \ crypto/fipsmodule/ec/p256_test.cc \ crypto/fipsmodule/ecdsa/ecdsa_test.cc \ crypto/fipsmodule/entropy/jitter_test.cc \ @@ -878,7 +881,7 @@ crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt \ crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt \ crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt \ - crypto/fipsmodule/ec/p256-tests.txt \ + crypto/fipsmodule/ec/p256-nistz_tests.txt \ crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt \ crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt \ crypto/fipsmodule/keccak/keccak_tests.txt \
diff --git a/include/openssl/prefix_symbols_internal_S.h b/include/openssl/prefix_symbols_internal_S.h index 170ac1c..4e5a51b 100644 --- a/include/openssl/prefix_symbols_internal_S.h +++ b/include/openssl/prefix_symbols_internal_S.h
@@ -231,14 +231,38 @@ #define _chacha20_poly1305_seal BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal) #define _chacha20_poly1305_seal_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal_avx2) #define _chacha20_poly1305_seal_sse41 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal_sse41) +#define _ecp_nistz256_div_by_2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_div_by_2) +#define _ecp_nistz256_mul_by_2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_by_2) +#define _ecp_nistz256_mul_by_3 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_by_3) #define _ecp_nistz256_mul_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont) +#define _ecp_nistz256_mul_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont_adx) +#define _ecp_nistz256_mul_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont_nohw) +#define _ecp_nistz256_neg BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_neg) #define _ecp_nistz256_ord_mul_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont) #define _ecp_nistz256_ord_mul_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont_adx) #define _ecp_nistz256_ord_mul_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont_nohw) #define _ecp_nistz256_ord_sqr_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont) #define _ecp_nistz256_ord_sqr_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont_adx) #define _ecp_nistz256_ord_sqr_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont_nohw) +#define _ecp_nistz256_point_add BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add) +#define _ecp_nistz256_point_add_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_adx) +#define _ecp_nistz256_point_add_affine BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine) +#define _ecp_nistz256_point_add_affine_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine_adx) +#define _ecp_nistz256_point_add_affine_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine_nohw) +#define _ecp_nistz256_point_add_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_nohw) +#define _ecp_nistz256_point_double BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double) +#define _ecp_nistz256_point_double_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double_adx) +#define _ecp_nistz256_point_double_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double_nohw) +#define _ecp_nistz256_select_w5 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5) +#define _ecp_nistz256_select_w5_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5_avx2) +#define _ecp_nistz256_select_w5_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5_nohw) +#define _ecp_nistz256_select_w7 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7) +#define _ecp_nistz256_select_w7_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7_avx2) +#define _ecp_nistz256_select_w7_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7_nohw) #define _ecp_nistz256_sqr_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont) +#define _ecp_nistz256_sqr_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont_adx) +#define _ecp_nistz256_sqr_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont_nohw) +#define _ecp_nistz256_sub BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sub) #define _fiat_curve25519_adx_mul BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_curve25519_adx_mul) #define _fiat_curve25519_adx_square BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_curve25519_adx_square) #define _fiat_p256_adx_mul BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_p256_adx_mul) @@ -509,14 +533,38 @@ #define chacha20_poly1305_seal BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal) #define chacha20_poly1305_seal_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal_avx2) #define chacha20_poly1305_seal_sse41 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal_sse41) +#define ecp_nistz256_div_by_2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_div_by_2) +#define ecp_nistz256_mul_by_2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_by_2) +#define ecp_nistz256_mul_by_3 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_by_3) #define ecp_nistz256_mul_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont) +#define ecp_nistz256_mul_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont_adx) +#define ecp_nistz256_mul_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont_nohw) +#define ecp_nistz256_neg BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_neg) #define ecp_nistz256_ord_mul_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont) #define ecp_nistz256_ord_mul_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont_adx) #define ecp_nistz256_ord_mul_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont_nohw) #define ecp_nistz256_ord_sqr_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont) #define ecp_nistz256_ord_sqr_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont_adx) #define ecp_nistz256_ord_sqr_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont_nohw) +#define ecp_nistz256_point_add BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add) +#define ecp_nistz256_point_add_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_adx) +#define ecp_nistz256_point_add_affine BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine) +#define ecp_nistz256_point_add_affine_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine_adx) +#define ecp_nistz256_point_add_affine_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine_nohw) +#define ecp_nistz256_point_add_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_nohw) +#define ecp_nistz256_point_double BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double) +#define ecp_nistz256_point_double_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double_adx) +#define ecp_nistz256_point_double_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double_nohw) +#define ecp_nistz256_select_w5 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5) +#define ecp_nistz256_select_w5_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5_avx2) +#define ecp_nistz256_select_w5_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5_nohw) +#define ecp_nistz256_select_w7 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7) +#define ecp_nistz256_select_w7_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7_avx2) +#define ecp_nistz256_select_w7_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7_nohw) #define ecp_nistz256_sqr_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont) +#define ecp_nistz256_sqr_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont_adx) +#define ecp_nistz256_sqr_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont_nohw) +#define ecp_nistz256_sub BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sub) #define fiat_curve25519_adx_mul BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_curve25519_adx_mul) #define fiat_curve25519_adx_square BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_curve25519_adx_square) #define fiat_p256_adx_mul BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_p256_adx_mul)
diff --git a/include/openssl/prefix_symbols_internal_c.h b/include/openssl/prefix_symbols_internal_c.h index ef12452..6e84071 100644 --- a/include/openssl/prefix_symbols_internal_c.h +++ b/include/openssl/prefix_symbols_internal_c.h
@@ -231,14 +231,38 @@ #define chacha20_poly1305_seal BORINGSSL_ADD_PREFIX(chacha20_poly1305_seal) #define chacha20_poly1305_seal_avx2 BORINGSSL_ADD_PREFIX(chacha20_poly1305_seal_avx2) #define chacha20_poly1305_seal_sse41 BORINGSSL_ADD_PREFIX(chacha20_poly1305_seal_sse41) +#define ecp_nistz256_div_by_2 BORINGSSL_ADD_PREFIX(ecp_nistz256_div_by_2) +#define ecp_nistz256_mul_by_2 BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_by_2) +#define ecp_nistz256_mul_by_3 BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_by_3) #define ecp_nistz256_mul_mont BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_mont) +#define ecp_nistz256_mul_mont_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_mont_adx) +#define ecp_nistz256_mul_mont_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_mont_nohw) +#define ecp_nistz256_neg BORINGSSL_ADD_PREFIX(ecp_nistz256_neg) #define ecp_nistz256_ord_mul_mont BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_mul_mont) #define ecp_nistz256_ord_mul_mont_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_mul_mont_adx) #define ecp_nistz256_ord_mul_mont_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_mul_mont_nohw) #define ecp_nistz256_ord_sqr_mont BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_sqr_mont) #define ecp_nistz256_ord_sqr_mont_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_sqr_mont_adx) #define ecp_nistz256_ord_sqr_mont_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_sqr_mont_nohw) +#define ecp_nistz256_point_add BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add) +#define ecp_nistz256_point_add_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_adx) +#define ecp_nistz256_point_add_affine BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_affine) +#define ecp_nistz256_point_add_affine_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_affine_adx) +#define ecp_nistz256_point_add_affine_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_affine_nohw) +#define ecp_nistz256_point_add_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_nohw) +#define ecp_nistz256_point_double BORINGSSL_ADD_PREFIX(ecp_nistz256_point_double) +#define ecp_nistz256_point_double_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_point_double_adx) +#define ecp_nistz256_point_double_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_point_double_nohw) +#define ecp_nistz256_select_w5 BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w5) +#define ecp_nistz256_select_w5_avx2 BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w5_avx2) +#define ecp_nistz256_select_w5_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w5_nohw) +#define ecp_nistz256_select_w7 BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w7) +#define ecp_nistz256_select_w7_avx2 BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w7_avx2) +#define ecp_nistz256_select_w7_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w7_nohw) #define ecp_nistz256_sqr_mont BORINGSSL_ADD_PREFIX(ecp_nistz256_sqr_mont) +#define ecp_nistz256_sqr_mont_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_sqr_mont_adx) +#define ecp_nistz256_sqr_mont_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_sqr_mont_nohw) +#define ecp_nistz256_sub BORINGSSL_ADD_PREFIX(ecp_nistz256_sub) #define fiat_curve25519_adx_mul BORINGSSL_ADD_PREFIX(fiat_curve25519_adx_mul) #define fiat_curve25519_adx_square BORINGSSL_ADD_PREFIX(fiat_curve25519_adx_square) #define fiat_p256_adx_mul BORINGSSL_ADD_PREFIX(fiat_p256_adx_mul)
diff --git a/third_party/fiat/p256_64.h b/third_party/fiat/p256_64.h index ee07a45..a14d60b 100644 --- a/third_party/fiat/p256_64.h +++ b/third_party/fiat/p256_64.h
@@ -1,3 +1,23 @@ +#include <openssl/base.h> +#include "bedrock_unverified_platform.c.inc" +#include "p256_field_64.br.c.inc" +#include "../../crypto/internal.h" + +#if !defined(OPENSSL_NO_ASM) && (defined(__ELF__) || defined(__APPLE__)) && \ + defined(OPENSSL_X86_64) && !defined(OPENSSL_NANOLIBC) +// These functions are only available with gas and SysV ABI, used by Apple and +// ELF-based platforms. Unlike most of our SysV assembly, they currently rely on +// the SysV redzone. This trips one target which looks like it targets SysV but +// has no redzone. This happens to define `OPENSSL_NANOLIBC`, so gate on that. +// +// TODO(crbug.com/522255483): Come up with a clearer story for the redzone +// situation. +extern "C" { +void fiat_p256_adx_mul(uint64_t*, const uint64_t*, const uint64_t*); +void fiat_p256_adx_sqr(uint64_t*, const uint64_t*); +} +#endif + /* Autogenerated: 'src/ExtractionOCaml/word_by_word_montgomery' --inline --static --use-value-barrier p256 64 '2^256 - 2^224 + 2^192 + 2^96 - 1' mul square add sub opp from_montgomery to_montgomery nonzero selectznz to_bytes from_bytes one msat divstep divstep_precomp */ /* curve description: p256 */ /* machine_wordsize = 64 (from "64") */ @@ -165,6 +185,14 @@ * */ static FIAT_P256_FIAT_INLINE void fiat_p256_mul(fiat_p256_montgomery_domain_field_element out1, const fiat_p256_montgomery_domain_field_element arg1, const fiat_p256_montgomery_domain_field_element arg2) { + // NOTE: edited by hand, see third_party/fiat/README.md +#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(__x86_64__) + if (bssl::CRYPTO_is_BMI1_capable() && bssl::CRYPTO_is_BMI2_capable() && + bssl::CRYPTO_is_ADX_capable()) { + fiat_p256_adx_mul(out1, arg1, arg2); + return; + } +#endif uint64_t x1; uint64_t x2; uint64_t x3; @@ -472,6 +500,14 @@ * */ static FIAT_P256_FIAT_INLINE void fiat_p256_square(fiat_p256_montgomery_domain_field_element out1, const fiat_p256_montgomery_domain_field_element arg1) { + // NOTE: edited by hand, see third_party/fiat/README.md +#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(__x86_64__) + if (bssl::CRYPTO_is_BMI1_capable() && bssl::CRYPTO_is_BMI2_capable() && + bssl::CRYPTO_is_ADX_capable()) { + fiat_p256_adx_sqr(out1, arg1); + return; + } +#endif uint64_t x1; uint64_t x2; uint64_t x3;
diff --git a/third_party/fiat/p256_64_msvc.h b/third_party/fiat/p256_64_msvc.h index abed672..aff43d3 100644 --- a/third_party/fiat/p256_64_msvc.h +++ b/third_party/fiat/p256_64_msvc.h
@@ -17,6 +17,8 @@ /* twos_complement_eval z = let x1 := z[0] + (z[1] << 64) + (z[2] << 128) + (z[3] << 192) in */ /* if x1 & (2^256-1) < 2^255 then x1 & (2^256-1) else (x1 & (2^256-1)) - 2^256 */ +#include "bedrock_unverified_platform.c.inc" +#include "p256_field_64.br.c.inc" #include <stdint.h> #include <intrin.h> #if defined(_M_X64)
diff --git a/third_party/fiat/p256_field.c.inc b/third_party/fiat/p256_field.c.inc index e69de29..bbd25de 100644 --- a/third_party/fiat/p256_field.c.inc +++ b/third_party/fiat/p256_field.c.inc
@@ -0,0 +1,26 @@ +#include <openssl/base.h> +#include "bedrock_unverified_platform.c.inc" + +#if defined(BORINGSSL_HAS_UINT128) +#include "p256_64.h" +#elif defined(OPENSSL_64_BIT) +#include "p256_64_msvc.h" +#else +#include "p256_field_32.br.c.inc" +#include "p256_32.h" +// the 32-bit Bedrock-generated field halving calls Fiat-C code for add, sub +static inline void p256_coord_add(br_word_t out, br_word_t x, br_word_t y) { + fiat_p256_add((uint32_t*)out, (const uint32_t*)x, (const uint32_t*)y); +} +static inline void p256_coord_sub(br_word_t out, br_word_t x, br_word_t y) { + fiat_p256_sub((uint32_t*)out, (const uint32_t*)x, (const uint32_t*)y); +} +#endif + +// the Bedrock-generated point operations call Fiat-C or Fiat-x86 mul, sqr +static inline void p256_coord_mul(br_word_t out, br_word_t x, br_word_t y) { + fiat_p256_mul((br_word_t*)out, (const br_word_t*)x, (const br_word_t*)y); +} +static inline void p256_coord_sqr(br_word_t out, br_word_t x) { + fiat_p256_square((br_word_t*)out, (const br_word_t*)x); +}
diff --git a/third_party/fiat/p256_point.br.c.inc b/third_party/fiat/p256_point.br.c.inc index 4928034..117547b 100644 --- a/third_party/fiat/p256_point.br.c.inc +++ b/third_party/fiat/p256_point.br.c.inc
@@ -110,57 +110,3 @@ } br_memcpy(p_out, p_sel, (br_word_t)96); } - -static inline br_word_t p256_point_add_affine_nz_nz_neq(br_word_t p_out, br_word_t p_P, br_word_t p_Q) { - br_word_t z1z1, Hsqr, ok, different_x, different_y, u2, Hcub, r, h, s2; - uint8_t _br_stackalloc_z1z1[32] = {0}; z1z1 = (br_word_t)&_br_stackalloc_z1z1; - uint8_t _br_stackalloc_u2[32] = {0}; u2 = (br_word_t)&_br_stackalloc_u2; - uint8_t _br_stackalloc_h[32] = {0}; h = (br_word_t)&_br_stackalloc_h; - uint8_t _br_stackalloc_s2[32] = {0}; s2 = (br_word_t)&_br_stackalloc_s2; - uint8_t _br_stackalloc_r[32] = {0}; r = (br_word_t)&_br_stackalloc_r; - uint8_t _br_stackalloc_Hsqr[32] = {0}; Hsqr = (br_word_t)&_br_stackalloc_Hsqr; - uint8_t _br_stackalloc_Hcub[32] = {0}; Hcub = (br_word_t)&_br_stackalloc_Hcub; - p256_coord_sqr(z1z1, (p_P+32)+32); - p256_coord_mul(u2, p_Q, z1z1); - p256_coord_sub(h, u2, p_P); - p256_coord_mul(s2, (p_P+32)+32, z1z1); - p256_coord_mul((p_out+32)+32, h, (p_P+32)+32); - p256_coord_mul(s2, s2, p_Q+32); - p256_coord_sub(r, s2, p_P+32); - p256_coord_sqr(Hsqr, h); - p256_coord_sqr(p_out, r); - p256_coord_mul(Hcub, Hsqr, h); - p256_coord_mul(u2, p_P, Hsqr); - different_x = p256_coord_nonzero(Hcub); - different_y = p256_coord_nonzero(p_out); - ok = br_value_barrier(different_x|different_y); - p256_coord_sub(p_out, p_out, Hcub); - p256_coord_sub(p_out, p_out, u2); - p256_coord_sub(p_out, p_out, u2); - p256_coord_sub(h, u2, p_out); - p256_coord_mul(s2, Hcub, p_P+32); - p256_coord_mul(h, h, r); - p256_coord_sub(p_out+32, h, s2); - return ok; -} - -static inline void p256_point_add_affinenz_conditional_vartime_if_doubling(br_word_t p_out, br_word_t p_P, br_word_t p_Q, br_word_t c) { - br_word_t nzQ, p_tmp, zeroP, zeroQ, ok, p_buf; - zeroP = p256_point_iszero(p_P); - nzQ = br_broadcast_nonzero(c); - zeroQ = ~nzQ; - uint8_t _br_stackalloc_p_tmp[96] = {0}; p_tmp = (br_word_t)&_br_stackalloc_p_tmp; - ok = p256_point_add_affine_nz_nz_neq(p_tmp, p_P, p_Q); - ok = br_declassify((zeroP|zeroQ)|ok); - uint8_t _br_stackalloc_p_buf[96] = {0}; p_buf = (br_word_t)&_br_stackalloc_p_buf; - br_memset(p_buf, (br_word_t)0, (br_word_t)3*32); - br_memcxor(p_buf, p_tmp, (br_word_t)3*32, (~zeroP)&(~zeroQ)); - br_memcxor(p_buf, p_P, (br_word_t)3*32, (~zeroP)&zeroQ); - br_memcxor(p_buf, p_Q, (br_word_t)3*32, zeroP&(~zeroQ)); - if (ok) { - /*skip*/ - } else { - p256_point_double(p_buf, p_P); - } - br_memcpy(p_out, p_buf, (br_word_t)96); -}