Replace most of P-256 assembly code with C Benchmark (X64 full) Time CPU Time Old Time New CPU Old CPU New --------------------------------------------------------------------------------------------------------------------------------- BM_SpeedECDH/p256/threads:1 +0.0012 +0.0012 146195 146366 146138 146319 BM_SpeedECDSASign/p256/threads:1 -0.0509 -0.0511 61778 58631 61762 58608 BM_SpeedECDSAVerify/p256/threads:1 +0.0079 +0.0078 113813 114712 113780 114669 OVERALL_GEOMEAN -0.0143 -0.0144 0 0 0 0 Benchmark (X64 full) Time CPU Time Old Time New CPU Old CPU New --------------------------------------------------------------------------------------------------------------------------------- BM_SpeedECDH/p256/threads:1 +0.0063 +0.0066 146195 147118 146138 147101 BM_SpeedECDSASign/p256/threads:1 -0.0434 -0.0432 61778 59098 61762 59091 BM_SpeedECDSAVerify/p256/threads:1 +0.0259 +0.0259 113813 116760 113780 116725 OVERALL_GEOMEAN -0.0042 -0.0040 0 0 0 0 Benchmark (X64 small) Time CPU Time Old Time New CPU Old CPU New --------------------------------------------------------------------------------------------------------------------------------- BM_SpeedECDH/p256/threads:1 -0.0982 -0.0982 76798 69259 76769 69234 BM_SpeedECDSASign/p256/threads:1 -0.2843 -0.2843 37843 27085 37827 27074 BM_SpeedECDSAVerify/p256/threads:1 -0.2139 -0.2139 71935 56548 71909 56528 OVERALL_GEOMEAN -0.2024 -0.2024 0 0 0 0 Benchmark (AArch64 full) Time CPU Time Old Time New CPU Old CPU New --------------------------------------------------------------------------------------------------------------------------------- BM_SpeedECDH/p256/threads:1 +0.1155 +0.1160 53096 59229 52977 59123 BM_SpeedECDSASign/p256/threads:1 +0.0310 +0.0313 19131 19724 19093 19691 BM_SpeedECDSAVerify/p256/threads:1 +0.1186 +0.1190 47258 52861 47159 52773 OVERALL_GEOMEAN +0.0876 +0.0880 0 0 0 0 Benchmark (AArch64 small) Time CPU Time Old Time New CPU Old CPU New --------------------------------------------------------------------------------------------------------------------------------- BM_SpeedECDH/p256/threads:1 -0.4665 -0.4666 121953 65063 121647 64892 BM_SpeedECDSASign/p256/threads:1 -0.4952 -0.4953 50219 25349 50118 25293 BM_SpeedECDSAVerify/p256/threads:1 -0.4997 -0.4997 104284 52178 104045 52058 OVERALL_GEOMEAN -0.4811 -0.4746 0 0 0 0 Benchmark (SSE32 on X64) Time CPU Time Old Time New CPU Old CPU New --------------------------------------------------------------------------------------------------------------------------------- BM_SpeedECDH/p256/threads:1 -0.0813 -0.0813 769580 706976 769293 706712 BM_SpeedECDSASign/p256/threads:1 -0.1820 -0.1820 288068 235645 287970 235555 BM_SpeedECDSAVerify/p256/threads:1 -0.1031 -0.1032 687829 616947 687620 616690 OVERALL_GEOMEAN -0.1232 -0.1233 0 0 0 0 Co-authored-by: Miriam Polzer <mpolzer@google.com> Change-Id: Ic03a53963ae7a3a6b7d43eadf197fb59be2bc7f3 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/74627 Commit-Queue: David Benjamin <davidben@google.com> Presubmit-BoringSSL-Verified: boringssl-scoped@luci-project-accounts.iam.gserviceaccount.com <boringssl-scoped@luci-project-accounts.iam.gserviceaccount.com> Reviewed-by: David Benjamin <davidben@google.com>
diff --git a/build.json b/build.json index 358a8c3..d0eb60c 100644 --- a/build.json +++ b/build.json
@@ -62,7 +62,6 @@ "crypto/fipsmodule/ec/ec_montgomery.cc.inc", "crypto/fipsmodule/ec/felem.cc.inc", "crypto/fipsmodule/ec/oct.cc.inc", - "crypto/fipsmodule/ec/p256-nistz.cc.inc", "crypto/fipsmodule/ec/p256.cc.inc", "crypto/fipsmodule/ec/scalar.cc.inc", "crypto/fipsmodule/ec/simple.cc.inc", @@ -525,8 +524,8 @@ "crypto/fipsmodule/digest/md32_common.h", "crypto/fipsmodule/ec/builtin_curves.h", "crypto/fipsmodule/ec/internal.h", + "crypto/fipsmodule/ec/p256_internal.h", "crypto/fipsmodule/ec/p256-nistz-table.h", - "crypto/fipsmodule/ec/p256-nistz.h", "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/entropy/internal.h", @@ -568,7 +567,6 @@ "third_party/fiat/p256_32.h", "third_party/fiat/p256_64.h", "third_party/fiat/p256_64_msvc.h", - "third_party/fiat/p256_field.c.inc", "third_party/fiat/p256_field_32.br.c.inc", "third_party/fiat/p256_field_64.br.c.inc", "third_party/fiat/p256_point.br.c.inc", @@ -854,7 +852,6 @@ "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", - "crypto/fipsmodule/ec/p256-nistz_test.cc", "crypto/fipsmodule/ec/p256_test.cc", "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/entropy/jitter_test.cc", @@ -912,7 +909,7 @@ "crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt", "crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt", "crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt", - "crypto/fipsmodule/ec/p256-nistz_tests.txt", + "crypto/fipsmodule/ec/p256-tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt", "crypto/fipsmodule/keccak/keccak_tests.txt",
diff --git a/crypto/ecdh/ecdh_test.cc b/crypto/ecdh/ecdh_test.cc index f39f934..9a81f4a 100644 --- a/crypto/ecdh/ecdh_test.cc +++ b/crypto/ecdh/ecdh_test.cc
@@ -30,6 +30,7 @@ #include <openssl/nid.h> #include <openssl/sha2.h> +#include "../fipsmodule/bn/internal.h" #include "../test/file_test.h" #include "../test/test_util.h" #include "../test/wycheproof_util.h" @@ -134,6 +135,7 @@ ASSERT_TRUE(group); bssl::UniquePtr<BIGNUM> priv_key = GetWycheproofBIGNUM(t, "private", false); ASSERT_TRUE(priv_key); + bssl::bn_secret(priv_key.get()); std::vector<uint8_t> peer_spki; ASSERT_TRUE(t->GetBytes(&peer_spki, "public")); WycheproofResult result; @@ -164,6 +166,7 @@ int ret = ECDH_compute_key(actual.data(), actual.size(), EC_KEY_get0_public_key(peer_ec), key.get(), nullptr); + CONSTTIME_DECLASSIFY(actual.data(), actual.size()); if (is_valid) { EXPECT_EQ(static_cast<int>(actual.size()), ret); EXPECT_EQ(Bytes(shared), Bytes(actual.data(), static_cast<size_t>(ret)));
diff --git a/crypto/fipsmodule/bcm.cc b/crypto/fipsmodule/bcm.cc index b0eae67..7cb8410 100644 --- a/crypto/fipsmodule/bcm.cc +++ b/crypto/fipsmodule/bcm.cc
@@ -87,7 +87,6 @@ #include "ec/ec_montgomery.cc.inc" #include "ec/felem.cc.inc" #include "ec/oct.cc.inc" -#include "ec/p256-nistz.cc.inc" #include "ec/p256.cc.inc" #include "ec/scalar.cc.inc" #include "ec/simple.cc.inc"
diff --git a/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl b/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl index f844cbf..4b3016b 100644 --- a/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl +++ b/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl
@@ -15,26 +15,9 @@ # ==================================================================== -# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL -# project. +# Originally written by Andy Polyakov <appro@openssl.org> for the OpenSSL +# project. Modified for BoringSSL. # ==================================================================== -# -# ECP_NISTZ256 module for ARMv8. -# -# February 2015. -# -# Original ECP_NISTZ256 submission targeting x86_64 is detailed in -# http://eprint.iacr.org/2013/816. -# -# with/without -DECP_NISTZ256_ASM -# Apple A7 +190-360% -# Cortex-A53 +190-400% -# Cortex-A57 +190-350% -# Denver +230-400% -# -# Ranges denote minimum and maximum improvement coefficients depending -# on benchmark. Lower coefficients are for ECDSA sign, server-side -# operation. Keep in mind that +400% means 5x improvement. # The first two arguments should always be the flavour and output file path. if ($#ARGV < 1) { die "Not enough arguments provided. @@ -52,32 +35,13 @@ *STDOUT=*OUT; { -my ($rp,$ap,$bp,$bi,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3,$poly1,$poly3, - $acc0,$acc1,$acc2,$acc3,$acc4,$acc5) = +my ($rp,$bp,$bi,$a0,$a1,$a2,$a3,$poly1,$t0,$t1,$t2,$t3,$poly3, + $acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6) = map("x$_",(0..17,19,20)); - -my ($acc6,$acc7)=($ap,$bp); # used in __ecp_nistz256_sqr_mont +my $acc7=$bi; # used in ecp_nistz256_sqr_mont +my $poly1w = $poly1 =~ s/x/w/r; $code.=<<___; -.section .rodata -.align 5 -.Lpoly: -.quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 -.LRR: // 2^512 mod P precomputed for NIST P256 polynomial -.quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd -.Lone_mont: -.quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe -.Lone: -.quad 1,0,0,0 -.Lord: -.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 -.LordK: -.quad 0xccd1c8aaee00bc4f -.asciz "ECP_NISTZ256 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>" -.text - -// void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4], -// const BN_ULONG x2[4]); .globl ecp_nistz256_mul_mont .type ecp_nistz256_mul_mont,%function .align 4 @@ -87,190 +51,6 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - ldr $bi,[$bp] // bp[0] - ldp $a0,$a1,[$ap] - ldp $a2,$a3,[$ap,#16] - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - - bl __ecp_nistz256_mul_mont - - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont - -// void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_sqr_mont -.type ecp_nistz256_sqr_mont,%function -.align 4 -ecp_nistz256_sqr_mont: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-32]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - - ldp $a0,$a1,[$ap] - ldp $a2,$a3,[$ap,#16] - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - - bl __ecp_nistz256_sqr_mont - - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont - -// void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_div_by_2 -.type ecp_nistz256_div_by_2,%function -.align 4 -ecp_nistz256_div_by_2: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp $acc0,$acc1,[$ap] - ldp $acc2,$acc3,[$ap,#16] - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - - bl __ecp_nistz256_div_by_2 - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2 - -// void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_mul_by_2 -.type ecp_nistz256_mul_by_2,%function -.align 4 -ecp_nistz256_mul_by_2: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp $acc0,$acc1,[$ap] - ldp $acc2,$acc3,[$ap,#16] - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - mov $t0,$acc0 - mov $t1,$acc1 - mov $t2,$acc2 - mov $t3,$acc3 - - bl __ecp_nistz256_add_to // ret = a+a // 2*a - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2 - -// void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_mul_by_3 -.type ecp_nistz256_mul_by_3,%function -.align 4 -ecp_nistz256_mul_by_3: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp $acc0,$acc1,[$ap] - ldp $acc2,$acc3,[$ap,#16] - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - mov $t0,$acc0 - mov $t1,$acc1 - mov $t2,$acc2 - mov $t3,$acc3 - mov $a0,$acc0 - mov $a1,$acc1 - mov $a2,$acc2 - mov $a3,$acc3 - - bl __ecp_nistz256_add_to // ret = a+a // 2*a - - mov $t0,$a0 - mov $t1,$a1 - mov $t2,$a2 - mov $t3,$a3 - - bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3 - -// void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4], -// const BN_ULONG x2[4]); -.globl ecp_nistz256_sub -.type ecp_nistz256_sub,%function -.align 4 -ecp_nistz256_sub: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp $acc0,$acc1,[$ap] - ldp $acc2,$acc3,[$ap,#16] - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - - bl __ecp_nistz256_sub_from - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_sub,.-ecp_nistz256_sub - -// void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_neg -.type ecp_nistz256_neg,%function -.align 4 -ecp_nistz256_neg: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - mov $bp,$ap - mov $acc0,xzr // a = 0 - mov $acc1,xzr - mov $acc2,xzr - mov $acc3,xzr - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - - bl __ecp_nistz256_sub_from - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_neg,.-ecp_nistz256_neg - -// note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded -// to $a0-$a3 and b[0] - to $bi -.type __ecp_nistz256_mul_mont,%function -.align 4 -__ecp_nistz256_mul_mont: mul $acc0,$a0,$bi // a[0]*b[0] umulh $t0,$a0,$bi @@ -354,6 +134,7 @@ ___ } $code.=<<___; + mov $poly1w, #-1 // poly1 = 0x00000000ffffffff // last reduction subs $t2,$acc0,$t0 // "*0xffff0001" sbc $t3,$acc0,$t1 @@ -363,6 +144,7 @@ adcs $acc3,$acc4,$t3 adc $acc4,$acc5,xzr + neg $poly3,$poly1 // poly3 = 0xffffffff00000001 adds $t0,$acc0,#1 // subs $t0,$acc0,#-1 // tmp = ret-modulus sbcs $t1,$acc1,$poly1 sbcs $t2,$acc2,xzr @@ -376,14 +158,21 @@ csel $acc3,$acc3,$t3,lo stp $acc2,$acc3,[$rp,#16] + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 + AARCH64_VALIDATE_LINK_REGISTER ret -.size __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont +.size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont -// note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded -// to $a0-$a3 -.type __ecp_nistz256_sqr_mont,%function +.globl ecp_nistz256_sqr_mont +.type ecp_nistz256_sqr_mont,%function .align 4 -__ecp_nistz256_sqr_mont: +ecp_nistz256_sqr_mont: + AARCH64_SIGN_LINK_REGISTER + stp x29,x30,[sp,#-32]! + add x29,sp,#0 + stp x19,x20,[sp,#16] + // | | | | | |a1*a0| | // | | | | |a2*a0| | | // | |a3*a2|a3*a0| | | | @@ -471,12 +260,14 @@ adcs $acc2,$acc3,$t2 // +=acc[0]*0xffff0001 adc $acc3,$t3,xzr // can't overflow + mov $poly1w, #-1 // poly1 = 0x00000000ffffffff adds $acc0,$acc0,$acc4 // accumulate upper half adcs $acc1,$acc1,$acc5 adcs $acc2,$acc2,$acc6 adcs $acc3,$acc3,$acc7 adc $acc4,xzr,xzr + neg $poly3,$poly1 // poly3 = 0xffffffff00000001 adds $t0,$acc0,#1 // subs $t0,$acc0,#-1 // tmp = ret-modulus sbcs $t1,$acc1,$poly1 sbcs $t2,$acc2,xzr @@ -490,762 +281,34 @@ csel $acc3,$acc3,$t3,lo stp $acc2,$acc3,[$rp,#16] - ret -.size __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont - -// Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to -// $a0-$a3 and $t0-$t3. This is done because it's used in multiple -// contexts, e.g. in multiplication by 2 and 3... -.type __ecp_nistz256_add_to,%function -.align 4 -__ecp_nistz256_add_to: - adds $acc0,$acc0,$t0 // ret = a+b - adcs $acc1,$acc1,$t1 - adcs $acc2,$acc2,$t2 - adcs $acc3,$acc3,$t3 - adc $ap,xzr,xzr // zap $ap - - adds $t0,$acc0,#1 // subs $t0,$a0,#-1 // tmp = ret-modulus - sbcs $t1,$acc1,$poly1 - sbcs $t2,$acc2,xzr - sbcs $t3,$acc3,$poly3 - sbcs xzr,$ap,xzr // did subtraction borrow? - - csel $acc0,$acc0,$t0,lo // ret = borrow ? ret : ret-modulus - csel $acc1,$acc1,$t1,lo - csel $acc2,$acc2,$t2,lo - stp $acc0,$acc1,[$rp] - csel $acc3,$acc3,$t3,lo - stp $acc2,$acc3,[$rp,#16] - - ret -.size __ecp_nistz256_add_to,.-__ecp_nistz256_add_to - -.type __ecp_nistz256_sub_from,%function -.align 4 -__ecp_nistz256_sub_from: - ldp $t0,$t1,[$bp] - ldp $t2,$t3,[$bp,#16] - subs $acc0,$acc0,$t0 // ret = a-b - sbcs $acc1,$acc1,$t1 - sbcs $acc2,$acc2,$t2 - sbcs $acc3,$acc3,$t3 - sbc $ap,xzr,xzr // zap $ap - - subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = ret+modulus - adcs $t1,$acc1,$poly1 - adcs $t2,$acc2,xzr - adc $t3,$acc3,$poly3 - cmp $ap,xzr // did subtraction borrow? - - csel $acc0,$acc0,$t0,eq // ret = borrow ? ret+modulus : ret - csel $acc1,$acc1,$t1,eq - csel $acc2,$acc2,$t2,eq - stp $acc0,$acc1,[$rp] - csel $acc3,$acc3,$t3,eq - stp $acc2,$acc3,[$rp,#16] - - ret -.size __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from - -.type __ecp_nistz256_sub_morf,%function -.align 4 -__ecp_nistz256_sub_morf: - ldp $t0,$t1,[$bp] - ldp $t2,$t3,[$bp,#16] - subs $acc0,$t0,$acc0 // ret = b-a - sbcs $acc1,$t1,$acc1 - sbcs $acc2,$t2,$acc2 - sbcs $acc3,$t3,$acc3 - sbc $ap,xzr,xzr // zap $ap - - subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = ret+modulus - adcs $t1,$acc1,$poly1 - adcs $t2,$acc2,xzr - adc $t3,$acc3,$poly3 - cmp $ap,xzr // did subtraction borrow? - - csel $acc0,$acc0,$t0,eq // ret = borrow ? ret+modulus : ret - csel $acc1,$acc1,$t1,eq - csel $acc2,$acc2,$t2,eq - stp $acc0,$acc1,[$rp] - csel $acc3,$acc3,$t3,eq - stp $acc2,$acc3,[$rp,#16] - - ret -.size __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf - -.type __ecp_nistz256_div_by_2,%function -.align 4 -__ecp_nistz256_div_by_2: - subs $t0,$acc0,#1 // adds $t0,$a0,#-1 // tmp = a+modulus - adcs $t1,$acc1,$poly1 - adcs $t2,$acc2,xzr - adcs $t3,$acc3,$poly3 - adc $ap,xzr,xzr // zap $ap - tst $acc0,#1 // is a even? - - csel $acc0,$acc0,$t0,eq // ret = even ? a : a+modulus - csel $acc1,$acc1,$t1,eq - csel $acc2,$acc2,$t2,eq - csel $acc3,$acc3,$t3,eq - csel $ap,xzr,$ap,eq - - lsr $acc0,$acc0,#1 // ret >>= 1 - orr $acc0,$acc0,$acc1,lsl#63 - lsr $acc1,$acc1,#1 - orr $acc1,$acc1,$acc2,lsl#63 - lsr $acc2,$acc2,#1 - orr $acc2,$acc2,$acc3,lsl#63 - lsr $acc3,$acc3,#1 - stp $acc0,$acc1,[$rp] - orr $acc3,$acc3,$ap,lsl#63 - stp $acc2,$acc3,[$rp,#16] - - ret -.size __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2 -___ -######################################################################## -# following subroutines are "literal" implementation of those found in -# ecp_nistz256.c -# -######################################################################## -# void ecp_nistz256_point_double(P256_POINT *out,const P256_POINT *inp); -# -{ -my ($S,$M,$Zsqr,$tmp0)=map(32*$_,(0..3)); -# above map() describes stack layout with 4 temporary -# 256-bit vectors on top. -my ($rp_real,$ap_real) = map("x$_",(21,22)); - -$code.=<<___; -.globl ecp_nistz256_point_double -.type ecp_nistz256_point_double,%function -.align 5 -ecp_nistz256_point_double: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-96]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - sub sp,sp,#32*4 - -.Ldouble_shortcut: - ldp $acc0,$acc1,[$ap,#32] - mov $rp_real,$rp - ldp $acc2,$acc3,[$ap,#48] - mov $ap_real,$ap - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - mov $t0,$acc0 - ldr $poly3,[$poly3,#24] - mov $t1,$acc1 - ldp $a0,$a1,[$ap_real,#64] // forward load for p256_sqr_mont - mov $t2,$acc2 - mov $t3,$acc3 - ldp $a2,$a3,[$ap_real,#64+16] - add $rp,sp,#$S - bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y); - - add $rp,sp,#$Zsqr - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z); - - ldp $t0,$t1,[$ap_real] - ldp $t2,$t3,[$ap_real,#16] - mov $a0,$acc0 // put Zsqr aside for p256_sub - mov $a1,$acc1 - mov $a2,$acc2 - mov $a3,$acc3 - add $rp,sp,#$M - bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x); - - add $bp,$ap_real,#0 - mov $acc0,$a0 // restore Zsqr - mov $acc1,$a1 - ldp $a0,$a1,[sp,#$S] // forward load for p256_sqr_mont - mov $acc2,$a2 - mov $acc3,$a3 - ldp $a2,$a3,[sp,#$S+16] - add $rp,sp,#$Zsqr - bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr); - - add $rp,sp,#$S - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S); - - ldr $bi,[$ap_real,#32] - ldp $a0,$a1,[$ap_real,#64] - ldp $a2,$a3,[$ap_real,#64+16] - add $bp,$ap_real,#32 - add $rp,sp,#$tmp0 - bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y); - - mov $t0,$acc0 - mov $t1,$acc1 - ldp $a0,$a1,[sp,#$S] // forward load for p256_sqr_mont - mov $t2,$acc2 - mov $t3,$acc3 - ldp $a2,$a3,[sp,#$S+16] - add $rp,$rp_real,#64 - bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0); - - add $rp,sp,#$tmp0 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S); - - ldr $bi,[sp,#$Zsqr] // forward load for p256_mul_mont - ldp $a0,$a1,[sp,#$M] - ldp $a2,$a3,[sp,#$M+16] - add $rp,$rp_real,#32 - bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0); - - add $bp,sp,#$Zsqr - add $rp,sp,#$M - bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr); - - mov $t0,$acc0 // duplicate M - mov $t1,$acc1 - mov $t2,$acc2 - mov $t3,$acc3 - mov $a0,$acc0 // put M aside - mov $a1,$acc1 - mov $a2,$acc2 - mov $a3,$acc3 - add $rp,sp,#$M - bl __ecp_nistz256_add_to - mov $t0,$a0 // restore M - mov $t1,$a1 - ldr $bi,[$ap_real] // forward load for p256_mul_mont - mov $t2,$a2 - ldp $a0,$a1,[sp,#$S] - mov $t3,$a3 - ldp $a2,$a3,[sp,#$S+16] - bl __ecp_nistz256_add_to // p256_mul_by_3(M, M); - - add $bp,$ap_real,#0 - add $rp,sp,#$S - bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x); - - mov $t0,$acc0 - mov $t1,$acc1 - ldp $a0,$a1,[sp,#$M] // forward load for p256_sqr_mont - mov $t2,$acc2 - mov $t3,$acc3 - ldp $a2,$a3,[sp,#$M+16] - add $rp,sp,#$tmp0 - bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S); - - add $rp,$rp_real,#0 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M); - - add $bp,sp,#$tmp0 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0); - - add $bp,sp,#$S - add $rp,sp,#$S - bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x); - - ldr $bi,[sp,#$M] - mov $a0,$acc0 // copy S - mov $a1,$acc1 - mov $a2,$acc2 - mov $a3,$acc3 - add $bp,sp,#$M - bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M); - - add $bp,$rp_real,#32 - add $rp,$rp_real,#32 - bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y); - - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x29,x30,[sp],#96 + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 AARCH64_VALIDATE_LINK_REGISTER ret -.size ecp_nistz256_point_double,.-ecp_nistz256_point_double +.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont ___ -} -######################################################################## -# void ecp_nistz256_point_add(P256_POINT *out,const P256_POINT *in1, -# const P256_POINT *in2); -{ -my ($res_x,$res_y,$res_z, - $H,$Hsqr,$R,$Rsqr,$Hcub, - $U1,$U2,$S1,$S2)=map(32*$_,(0..11)); -my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr); -# above map() describes stack layout with 12 temporary -# 256-bit vectors on top. -my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp0,$temp1,$temp2)=map("x$_",(21..28)); - -$code.=<<___; -.globl ecp_nistz256_point_add -.type ecp_nistz256_point_add,%function -.align 5 -ecp_nistz256_point_add: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-96]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - sub sp,sp,#32*12 - - ldp $a0,$a1,[$bp,#64] // in2_z - ldp $a2,$a3,[$bp,#64+16] - mov $rp_real,$rp - mov $ap_real,$ap - mov $bp_real,$bp - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - orr $t0,$a0,$a1 - orr $t2,$a2,$a3 - orr $in2infty,$t0,$t2 - cmp $in2infty,#0 - csetm $in2infty,ne // ~in2infty - add $rp,sp,#$Z2sqr - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z); - - ldp $a0,$a1,[$ap_real,#64] // in1_z - ldp $a2,$a3,[$ap_real,#64+16] - orr $t0,$a0,$a1 - orr $t2,$a2,$a3 - orr $in1infty,$t0,$t2 - cmp $in1infty,#0 - csetm $in1infty,ne // ~in1infty - add $rp,sp,#$Z1sqr - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); - - ldr $bi,[$bp_real,#64] - ldp $a0,$a1,[sp,#$Z2sqr] - ldp $a2,$a3,[sp,#$Z2sqr+16] - add $bp,$bp_real,#64 - add $rp,sp,#$S1 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z); - - ldr $bi,[$ap_real,#64] - ldp $a0,$a1,[sp,#$Z1sqr] - ldp $a2,$a3,[sp,#$Z1sqr+16] - add $bp,$ap_real,#64 - add $rp,sp,#$S2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); - - ldr $bi,[$ap_real,#32] - ldp $a0,$a1,[sp,#$S1] - ldp $a2,$a3,[sp,#$S1+16] - add $bp,$ap_real,#32 - add $rp,sp,#$S1 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y); - - ldr $bi,[$bp_real,#32] - ldp $a0,$a1,[sp,#$S2] - ldp $a2,$a3,[sp,#$S2+16] - add $bp,$bp_real,#32 - add $rp,sp,#$S2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); - - add $bp,sp,#$S1 - ldr $bi,[sp,#$Z2sqr] // forward load for p256_mul_mont - ldp $a0,$a1,[$ap_real] - ldp $a2,$a3,[$ap_real,#16] - add $rp,sp,#$R - bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1); - - orr $acc0,$acc0,$acc1 // see if result is zero - orr $acc2,$acc2,$acc3 - orr $temp0,$acc0,$acc2 // ~is_equal(S1,S2) - - add $bp,sp,#$Z2sqr - add $rp,sp,#$U1 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr); - - ldr $bi,[sp,#$Z1sqr] - ldp $a0,$a1,[$bp_real] - ldp $a2,$a3,[$bp_real,#16] - add $bp,sp,#$Z1sqr - add $rp,sp,#$U2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr); - - add $bp,sp,#$U1 - ldp $a0,$a1,[sp,#$R] // forward load for p256_sqr_mont - ldp $a2,$a3,[sp,#$R+16] - add $rp,sp,#$H - bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1); - - orr $acc0,$acc0,$acc1 // see if result is zero - orr $acc2,$acc2,$acc3 - orr $acc0,$acc0,$acc2 // ~is_equal(U1,U2) - - mvn $temp1,$in1infty // -1/0 -> 0/-1 - mvn $temp2,$in2infty // -1/0 -> 0/-1 - orr $acc0,$acc0,$temp1 - orr $acc0,$acc0,$temp2 - orr $acc0,$acc0,$temp0 - cbnz $acc0,.Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2)) - -.Ladd_double: - mov $ap,$ap_real - mov $rp,$rp_real - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames - b .Ldouble_shortcut - -.align 4 -.Ladd_proceed: - add $rp,sp,#$Rsqr - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); - - ldr $bi,[$ap_real,#64] - ldp $a0,$a1,[sp,#$H] - ldp $a2,$a3,[sp,#$H+16] - add $bp,$ap_real,#64 - add $rp,sp,#$res_z - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); - - ldp $a0,$a1,[sp,#$H] - ldp $a2,$a3,[sp,#$H+16] - add $rp,sp,#$Hsqr - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); - - ldr $bi,[$bp_real,#64] - ldp $a0,$a1,[sp,#$res_z] - ldp $a2,$a3,[sp,#$res_z+16] - add $bp,$bp_real,#64 - add $rp,sp,#$res_z - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z); - - ldr $bi,[sp,#$H] - ldp $a0,$a1,[sp,#$Hsqr] - ldp $a2,$a3,[sp,#$Hsqr+16] - add $bp,sp,#$H - add $rp,sp,#$Hcub - bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); - - ldr $bi,[sp,#$Hsqr] - ldp $a0,$a1,[sp,#$U1] - ldp $a2,$a3,[sp,#$U1+16] - add $bp,sp,#$Hsqr - add $rp,sp,#$U2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr); - - mov $t0,$acc0 - mov $t1,$acc1 - mov $t2,$acc2 - mov $t3,$acc3 - add $rp,sp,#$Hsqr - bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); - - add $bp,sp,#$Rsqr - add $rp,sp,#$res_x - bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); - - add $bp,sp,#$Hcub - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); - - add $bp,sp,#$U2 - ldr $bi,[sp,#$Hcub] // forward load for p256_mul_mont - ldp $a0,$a1,[sp,#$S1] - ldp $a2,$a3,[sp,#$S1+16] - add $rp,sp,#$res_y - bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); - - add $bp,sp,#$Hcub - add $rp,sp,#$S2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub); - - ldr $bi,[sp,#$R] - ldp $a0,$a1,[sp,#$res_y] - ldp $a2,$a3,[sp,#$res_y+16] - add $bp,sp,#$R - add $rp,sp,#$res_y - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); - - add $bp,sp,#$S2 - bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); - - ldp $a0,$a1,[sp,#$res_x] // res - ldp $a2,$a3,[sp,#$res_x+16] - ldp $t0,$t1,[$bp_real] // in2 - ldp $t2,$t3,[$bp_real,#16] -___ -for($i=0;$i<64;$i+=32) { # conditional moves -$code.=<<___; - ldp $acc0,$acc1,[$ap_real,#$i] // in1 - cmp $in1infty,#0 // ~$in1intfy, remember? - ldp $acc2,$acc3,[$ap_real,#$i+16] - csel $t0,$a0,$t0,ne - csel $t1,$a1,$t1,ne - ldp $a0,$a1,[sp,#$res_x+$i+32] // res - csel $t2,$a2,$t2,ne - csel $t3,$a3,$t3,ne - cmp $in2infty,#0 // ~$in2intfy, remember? - ldp $a2,$a3,[sp,#$res_x+$i+48] - csel $acc0,$t0,$acc0,ne - csel $acc1,$t1,$acc1,ne - ldp $t0,$t1,[$bp_real,#$i+32] // in2 - csel $acc2,$t2,$acc2,ne - csel $acc3,$t3,$acc3,ne - ldp $t2,$t3,[$bp_real,#$i+48] - stp $acc0,$acc1,[$rp_real,#$i] - stp $acc2,$acc3,[$rp_real,#$i+16] -___ -} -$code.=<<___; - ldp $acc0,$acc1,[$ap_real,#$i] // in1 - cmp $in1infty,#0 // ~$in1intfy, remember? - ldp $acc2,$acc3,[$ap_real,#$i+16] - csel $t0,$a0,$t0,ne - csel $t1,$a1,$t1,ne - csel $t2,$a2,$t2,ne - csel $t3,$a3,$t3,ne - cmp $in2infty,#0 // ~$in2intfy, remember? - csel $acc0,$t0,$acc0,ne - csel $acc1,$t1,$acc1,ne - csel $acc2,$t2,$acc2,ne - csel $acc3,$t3,$acc3,ne - stp $acc0,$acc1,[$rp_real,#$i] - stp $acc2,$acc3,[$rp_real,#$i+16] - -.Ladd_done: - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_point_add,.-ecp_nistz256_point_add -___ -} - -######################################################################## -# void ecp_nistz256_point_add_affine(P256_POINT *out,const P256_POINT *in1, -# const P256_POINT_AFFINE *in2); -{ -my ($res_x,$res_y,$res_z, - $U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr)=map(32*$_,(0..9)); -my $Z1sqr = $S2; -# above map() describes stack layout with 10 temporary -# 256-bit vectors on top. -my ($rp_real,$ap_real,$bp_real,$in1infty,$in2infty,$temp)=map("x$_",(21..26)); - -$code.=<<___; -.globl ecp_nistz256_point_add_affine -.type ecp_nistz256_point_add_affine,%function -.align 5 -ecp_nistz256_point_add_affine: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - sub sp,sp,#32*10 - - mov $rp_real,$rp - mov $ap_real,$ap - mov $bp_real,$bp - adrp $poly3,:pg_hi21:.Lpoly - add $poly3,$poly3,:lo12:.Lpoly - ldr $poly1,[$poly3,#8] - ldr $poly3,[$poly3,#24] - - ldp $a0,$a1,[$ap,#64] // in1_z - ldp $a2,$a3,[$ap,#64+16] - orr $t0,$a0,$a1 - orr $t2,$a2,$a3 - orr $in1infty,$t0,$t2 - cmp $in1infty,#0 - csetm $in1infty,ne // ~in1infty - - ldp $acc0,$acc1,[$bp] // in2_x - ldp $acc2,$acc3,[$bp,#16] - ldp $t0,$t1,[$bp,#32] // in2_y - ldp $t2,$t3,[$bp,#48] - orr $acc0,$acc0,$acc1 - orr $acc2,$acc2,$acc3 - orr $t0,$t0,$t1 - orr $t2,$t2,$t3 - orr $acc0,$acc0,$acc2 - orr $t0,$t0,$t2 - orr $in2infty,$acc0,$t0 - cmp $in2infty,#0 - csetm $in2infty,ne // ~in2infty - - add $rp,sp,#$Z1sqr - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); - - mov $a0,$acc0 - mov $a1,$acc1 - mov $a2,$acc2 - mov $a3,$acc3 - ldr $bi,[$bp_real] - add $bp,$bp_real,#0 - add $rp,sp,#$U2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x); - - add $bp,$ap_real,#0 - ldr $bi,[$ap_real,#64] // forward load for p256_mul_mont - ldp $a0,$a1,[sp,#$Z1sqr] - ldp $a2,$a3,[sp,#$Z1sqr+16] - add $rp,sp,#$H - bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x); - - add $bp,$ap_real,#64 - add $rp,sp,#$S2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); - - ldr $bi,[$ap_real,#64] - ldp $a0,$a1,[sp,#$H] - ldp $a2,$a3,[sp,#$H+16] - add $bp,$ap_real,#64 - add $rp,sp,#$res_z - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); - - ldr $bi,[$bp_real,#32] - ldp $a0,$a1,[sp,#$S2] - ldp $a2,$a3,[sp,#$S2+16] - add $bp,$bp_real,#32 - add $rp,sp,#$S2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); - - add $bp,$ap_real,#32 - ldp $a0,$a1,[sp,#$H] // forward load for p256_sqr_mont - ldp $a2,$a3,[sp,#$H+16] - add $rp,sp,#$R - bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y); - - add $rp,sp,#$Hsqr - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); - - ldp $a0,$a1,[sp,#$R] - ldp $a2,$a3,[sp,#$R+16] - add $rp,sp,#$Rsqr - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); - - ldr $bi,[sp,#$H] - ldp $a0,$a1,[sp,#$Hsqr] - ldp $a2,$a3,[sp,#$Hsqr+16] - add $bp,sp,#$H - add $rp,sp,#$Hcub - bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); - - ldr $bi,[$ap_real] - ldp $a0,$a1,[sp,#$Hsqr] - ldp $a2,$a3,[sp,#$Hsqr+16] - add $bp,$ap_real,#0 - add $rp,sp,#$U2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr); - - mov $t0,$acc0 - mov $t1,$acc1 - mov $t2,$acc2 - mov $t3,$acc3 - add $rp,sp,#$Hsqr - bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); - - add $bp,sp,#$Rsqr - add $rp,sp,#$res_x - bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); - - add $bp,sp,#$Hcub - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); - - add $bp,sp,#$U2 - ldr $bi,[$ap_real,#32] // forward load for p256_mul_mont - ldp $a0,$a1,[sp,#$Hcub] - ldp $a2,$a3,[sp,#$Hcub+16] - add $rp,sp,#$res_y - bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); - - add $bp,$ap_real,#32 - add $rp,sp,#$S2 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub); - - ldr $bi,[sp,#$R] - ldp $a0,$a1,[sp,#$res_y] - ldp $a2,$a3,[sp,#$res_y+16] - add $bp,sp,#$R - add $rp,sp,#$res_y - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); - - add $bp,sp,#$S2 - bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); - - ldp $a0,$a1,[sp,#$res_x] // res - ldp $a2,$a3,[sp,#$res_x+16] - ldp $t0,$t1,[$bp_real] // in2 - ldp $t2,$t3,[$bp_real,#16] -___ -for($i=0;$i<64;$i+=32) { # conditional moves -$code.=<<___; - ldp $acc0,$acc1,[$ap_real,#$i] // in1 - cmp $in1infty,#0 // ~$in1intfy, remember? - ldp $acc2,$acc3,[$ap_real,#$i+16] - csel $t0,$a0,$t0,ne - csel $t1,$a1,$t1,ne - ldp $a0,$a1,[sp,#$res_x+$i+32] // res - csel $t2,$a2,$t2,ne - csel $t3,$a3,$t3,ne - cmp $in2infty,#0 // ~$in2intfy, remember? - ldp $a2,$a3,[sp,#$res_x+$i+48] - csel $acc0,$t0,$acc0,ne - csel $acc1,$t1,$acc1,ne - ldp $t0,$t1,[$bp_real,#$i+32] // in2 - csel $acc2,$t2,$acc2,ne - csel $acc3,$t3,$acc3,ne - ldp $t2,$t3,[$bp_real,#$i+48] - stp $acc0,$acc1,[$rp_real,#$i] - stp $acc2,$acc3,[$rp_real,#$i+16] -___ -$code.=<<___ if ($i == 0); - adrp $bp_real,:pg_hi21:.Lone_mont-64 - add $bp_real,$bp_real,:lo12:.Lone_mont-64 -___ -} -$code.=<<___; - ldp $acc0,$acc1,[$ap_real,#$i] // in1 - cmp $in1infty,#0 // ~$in1intfy, remember? - ldp $acc2,$acc3,[$ap_real,#$i+16] - csel $t0,$a0,$t0,ne - csel $t1,$a1,$t1,ne - csel $t2,$a2,$t2,ne - csel $t3,$a3,$t3,ne - cmp $in2infty,#0 // ~$in2intfy, remember? - csel $acc0,$t0,$acc0,ne - csel $acc1,$t1,$acc1,ne - csel $acc2,$t2,$acc2,ne - csel $acc3,$t3,$acc3,ne - stp $acc0,$acc1,[$rp_real,#$i] - stp $acc2,$acc3,[$rp_real,#$i+16] - - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x29,x30,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine -___ -} if (1) { +my ($rp,$ap,$bp,$bi,$a0,$a1,$a2,$a3,$t0,$t1,$t2,$t3,$poly1,$poly3, + $acc0,$acc1,$acc2,$acc3,$acc4,$acc5) = + map("x$_",(0..17,19,20)); + +my $poly1w = $poly1 =~ s/x/w/r; my ($ord0,$ord1) = ($poly1,$poly3); my ($ord2,$ord3,$ordk,$t4) = map("x$_",(21..24)); +my $acc6 = $ap; my $acc7 = $bi; $code.=<<___; -//////////////////////////////////////////////////////////////////////// +.section .rodata +.align 5 +.Lord: +.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 +.LordK: +.quad 0xccd1c8aaee00bc4f + .asciz "ECP_NISTZ256 for ARMv8, CRYPTOGAMS by <appro\@openssl.org>" + .text + // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4], // uint64_t b[4]); .globl ecp_nistz256_ord_mul_mont @@ -1268,7 +331,7 @@ ldp $ord0,$ord1,[$ordk,#0] ldp $ord2,$ord3,[$ordk,#16] - ldr $ordk,[$ordk,#32] + ldr $ordk,[$ordk,#32] // LordK mul $acc0,$a0,$bi // a[0]*b[0] umulh $t0,$a0,$bi @@ -1542,135 +605,6 @@ ___ } } -######################################################################## -# select subroutines -# These select functions are similar to those in p256-x86_64-asm.pl -# They load all points in the lookup table -# keeping in the output only the one corresponding to the input index. -{ -my ($val,$in_t)=map("x$_",(0..1)); -my ($index)=("w2"); -my ($Idx_ctr,$Val_in, $Mask_64)=("w9", "x10", "x11"); -my ($Mask)=("v3"); -my ($Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("v$_",(16..21)); -my ($T0a,$T0b,$T0c,$T0d,$T0e,$T0f)=map("v$_",(22..27)); -$code.=<<___; -//////////////////////////////////////////////////////////////////////// -// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w5 -.type ecp_nistz256_select_w5,%function -.align 4 -ecp_nistz256_select_w5: - AARCH64_VALID_CALL_TARGET - - // $Val_in := $val - // $Idx_ctr := 0; loop counter and incremented internal index - mov $Val_in, $val - mov $Idx_ctr, #0 - - // [$Ra-$Rf] := 0 - movi $Ra.16b, #0 - movi $Rb.16b, #0 - movi $Rc.16b, #0 - movi $Rd.16b, #0 - movi $Re.16b, #0 - movi $Rf.16b, #0 - -.Lselect_w5_loop: - // Loop 16 times. - - // Increment index (loop counter); tested at the end of the loop - add $Idx_ctr, $Idx_ctr, #1 - - // [$T0a-$T0f] := Load a (3*256-bit = 6*128-bit) table entry starting at $in_t - // and advance $in_t to point to the next entry - ld1 {$T0a.2d, $T0b.2d, $T0c.2d, $T0d.2d}, [$in_t],#64 - - // $Mask_64 := ($Idx_ctr == $index)? All 1s : All 0s - cmp $Idx_ctr, $index - csetm $Mask_64, eq - - // continue loading ... - ld1 {$T0e.2d, $T0f.2d}, [$in_t],#32 - - // duplicate mask_64 into Mask (all 0s or all 1s) - dup $Mask.2d, $Mask_64 - - // [$Ra-$Rd] := (Mask == all 1s)? [$T0a-$T0d] : [$Ra-$Rd] - // i.e., values in output registers will remain the same if $Idx_ctr != $index - bit $Ra.16b, $T0a.16b, $Mask.16b - bit $Rb.16b, $T0b.16b, $Mask.16b - - bit $Rc.16b, $T0c.16b, $Mask.16b - bit $Rd.16b, $T0d.16b, $Mask.16b - - bit $Re.16b, $T0e.16b, $Mask.16b - bit $Rf.16b, $T0f.16b, $Mask.16b - - // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back - tbz $Idx_ctr, #4, .Lselect_w5_loop - - // Write [$Ra-$Rf] to memory at the output pointer - st1 {$Ra.2d, $Rb.2d, $Rc.2d, $Rd.2d}, [$Val_in],#64 - st1 {$Re.2d, $Rf.2d}, [$Val_in] - - ret -.size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5 - - -//////////////////////////////////////////////////////////////////////// -// void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w7 -.type ecp_nistz256_select_w7,%function -.align 4 -ecp_nistz256_select_w7: - AARCH64_VALID_CALL_TARGET - - // $Idx_ctr := 0; loop counter and incremented internal index - mov $Idx_ctr, #0 - - // [$Ra-$Rf] := 0 - movi $Ra.16b, #0 - movi $Rb.16b, #0 - movi $Rc.16b, #0 - movi $Rd.16b, #0 - -.Lselect_w7_loop: - // Loop 64 times. - - // Increment index (loop counter); tested at the end of the loop - add $Idx_ctr, $Idx_ctr, #1 - - // [$T0a-$T0d] := Load a (2*256-bit = 4*128-bit) table entry starting at $in_t - // and advance $in_t to point to the next entry - ld1 {$T0a.2d, $T0b.2d, $T0c.2d, $T0d.2d}, [$in_t],#64 - - // $Mask_64 := ($Idx_ctr == $index)? All 1s : All 0s - cmp $Idx_ctr, $index - csetm $Mask_64, eq - - // duplicate mask_64 into Mask (all 0s or all 1s) - dup $Mask.2d, $Mask_64 - - // [$Ra-$Rd] := (Mask == all 1s)? [$T0a-$T0d] : [$Ra-$Rd] - // i.e., values in output registers will remain the same if $Idx_ctr != $index - bit $Ra.16b, $T0a.16b, $Mask.16b - bit $Rb.16b, $T0b.16b, $Mask.16b - - bit $Rc.16b, $T0c.16b, $Mask.16b - bit $Rd.16b, $T0d.16b, $Mask.16b - - // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back - tbz $Idx_ctr, #6, .Lselect_w7_loop - - // Write [$Ra-$Rd] to memory at the output pointer - st1 {$Ra.2d, $Rb.2d, $Rc.2d, $Rd.2d}, [$val] - - ret -.size ecp_nistz256_select_w7,.-ecp_nistz256_select_w7 -___ -} - foreach (split("\n",$code)) { s/\`([^\`]*)\`/eval $1/ge;
diff --git a/crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl b/crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl index 08a6a93..7a366b8 100755 --- a/crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl +++ b/crypto/fipsmodule/ec/asm/p256-x86_64-asm.pl
@@ -25,27 +25,6 @@ # 256 Bit Primes" # Further optimization by <appro@openssl.org>: -# -# this/original with/without -DECP_NISTZ256_ASM(*) -# Opteron +15-49% +150-195% -# Bulldozer +18-45% +175-240% -# P4 +24-46% +100-150% -# Westmere +18-34% +87-160% -# Sandy Bridge +14-35% +120-185% -# Ivy Bridge +11-35% +125-180% -# Haswell +10-37% +160-200% -# Broadwell +24-58% +210-270% -# Atom +20-50% +180-240% -# VIA Nano +50-160% +480-480% -# -# (*) "without -DECP_NISTZ256_ASM" refers to build with -# "enable-ec_nistp_64_gcc_128"; -# -# Ranges denote minimum and maximum improvement coefficients depending -# on benchmark. In "this/original" column lower coefficient is for -# ECDSA sign, while in "with/without" - for ECDH key agreement, and -# higher - for ECDSA sign, relatively fastest server-side operation. -# Keep in mind that +100% means 2x improvement. $flavour = shift; $output = shift; @@ -67,21 +46,6 @@ $code.=<<___; .text -# The polynomial -.section .rodata -.align 64 -.Lpoly: -.quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 - -.LOne: -.long 1,1,1,1,1,1,1,1 -.LTwo: -.long 2,2,2,2,2,2,2,2 -.LThree: -.long 3,3,3,3,3,3,3,3 -.LONE_mont: -.quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe - # Constants for computations modulo ord(p256) .Lord: .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 @@ -90,71 +54,7 @@ .text ___ -{ -my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11)); -my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13"); -my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx"); -$code.=<<___; - -################################################################################ -# void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]); -.globl ecp_nistz256_neg -.type ecp_nistz256_neg,\@function,2 -.align 32 -ecp_nistz256_neg: -.cfi_startproc - _CET_ENDBR - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 -.Lneg_body: - - xor $a0, $a0 - xor $a1, $a1 - xor $a2, $a2 - xor $a3, $a3 - xor $t4, $t4 - - sub 8*0($a_ptr), $a0 - sbb 8*1($a_ptr), $a1 - sbb 8*2($a_ptr), $a2 - mov $a0, $t0 - sbb 8*3($a_ptr), $a3 - lea .Lpoly(%rip), $a_ptr - mov $a1, $t1 - sbb \$0, $t4 - - add 8*0($a_ptr), $a0 - mov $a2, $t2 - adc 8*1($a_ptr), $a1 - adc 8*2($a_ptr), $a2 - mov $a3, $t3 - adc 8*3($a_ptr), $a3 - test $t4, $t4 - - cmovz $t0, $a0 - cmovz $t1, $a1 - mov $a0, 8*0($r_ptr) - cmovz $t2, $a2 - mov $a1, 8*1($r_ptr) - cmovz $t3, $a3 - mov $a2, 8*2($r_ptr) - mov $a3, 8*3($r_ptr) - - mov 0(%rsp),%r13 -.cfi_restore %r13 - mov 8(%rsp),%r12 -.cfi_restore %r12 - lea 16(%rsp),%rsp -.cfi_adjust_cfa_offset -16 -.Lneg_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_neg,.-ecp_nistz256_neg -___ -} { my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx"); my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15)); @@ -174,6 +74,8 @@ ecp_nistz256_ord_mul_mont_nohw: .cfi_startproc _CET_ENDBR +___ +$code.=<<___; push %rbp .cfi_push %rbp push %rbx @@ -497,6 +399,8 @@ ecp_nistz256_ord_sqr_mont_nohw: .cfi_startproc _CET_ENDBR +___ +$code.=<<___; push %rbp .cfi_push %rbp push %rbx @@ -1230,2574 +1134,7 @@ .size ecp_nistz256_ord_sqr_mont_adx,.-ecp_nistz256_ord_sqr_mont_adx ___ -$code.=<<___; -################################################################################ -# void ecp_nistz256_mul_mont( -# uint64_t res[4], -# uint64_t a[4], -# uint64_t b[4]); - -.globl ecp_nistz256_mul_mont_nohw -.type ecp_nistz256_mul_mont_nohw,\@function,3 -.align 32 -ecp_nistz256_mul_mont_nohw: -.cfi_startproc - _CET_ENDBR - push %rbp -.cfi_push %rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 -.Lmul_body: - mov $b_org, $b_ptr - mov 8*0($b_org), %rax - mov 8*0($a_ptr), $acc1 - mov 8*1($a_ptr), $acc2 - mov 8*2($a_ptr), $acc3 - mov 8*3($a_ptr), $acc4 - - call __ecp_nistz256_mul_montq - - mov 0(%rsp),%r15 -.cfi_restore %r15 - mov 8(%rsp),%r14 -.cfi_restore %r14 - mov 16(%rsp),%r13 -.cfi_restore %r13 - mov 24(%rsp),%r12 -.cfi_restore %r12 - mov 32(%rsp),%rbx -.cfi_restore %rbx - mov 40(%rsp),%rbp -.cfi_restore %rbp - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lmul_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_mul_mont_nohw,.-ecp_nistz256_mul_mont_nohw - -.type __ecp_nistz256_mul_montq,\@abi-omnipotent -.align 32 -__ecp_nistz256_mul_montq: -.cfi_startproc - ######################################################################## - # Multiply a by b[0] - mov %rax, $t1 - mulq $acc1 - mov .Lpoly+8*1(%rip),$poly1 - mov %rax, $acc0 - mov $t1, %rax - mov %rdx, $acc1 - - mulq $acc2 - mov .Lpoly+8*3(%rip),$poly3 - add %rax, $acc1 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $acc2 - - mulq $acc3 - add %rax, $acc2 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $acc3 - - mulq $acc4 - add %rax, $acc3 - mov $acc0, %rax - adc \$0, %rdx - xor $acc5, $acc5 - mov %rdx, $acc4 - - ######################################################################## - # First reduction step - # Basically now we want to multiply acc[0] by p256, - # and add the result to the acc. - # Due to the special form of p256 we do some optimizations - # - # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0] - # then we add acc[0] and get acc[0] x 2^96 - - mov $acc0, $t1 - shl \$32, $acc0 - mulq $poly3 - shr \$32, $t1 - add $acc0, $acc1 # +=acc[0]<<96 - adc $t1, $acc2 - adc %rax, $acc3 - mov 8*1($b_ptr), %rax - adc %rdx, $acc4 - adc \$0, $acc5 - xor $acc0, $acc0 - - ######################################################################## - # Multiply by b[1] - mov %rax, $t1 - mulq 8*0($a_ptr) - add %rax, $acc1 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*1($a_ptr) - add $t0, $acc2 - adc \$0, %rdx - add %rax, $acc2 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*2($a_ptr) - add $t0, $acc3 - adc \$0, %rdx - add %rax, $acc3 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*3($a_ptr) - add $t0, $acc4 - adc \$0, %rdx - add %rax, $acc4 - mov $acc1, %rax - adc %rdx, $acc5 - adc \$0, $acc0 - - ######################################################################## - # Second reduction step - mov $acc1, $t1 - shl \$32, $acc1 - mulq $poly3 - shr \$32, $t1 - add $acc1, $acc2 - adc $t1, $acc3 - adc %rax, $acc4 - mov 8*2($b_ptr), %rax - adc %rdx, $acc5 - adc \$0, $acc0 - xor $acc1, $acc1 - - ######################################################################## - # Multiply by b[2] - mov %rax, $t1 - mulq 8*0($a_ptr) - add %rax, $acc2 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*1($a_ptr) - add $t0, $acc3 - adc \$0, %rdx - add %rax, $acc3 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*2($a_ptr) - add $t0, $acc4 - adc \$0, %rdx - add %rax, $acc4 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*3($a_ptr) - add $t0, $acc5 - adc \$0, %rdx - add %rax, $acc5 - mov $acc2, %rax - adc %rdx, $acc0 - adc \$0, $acc1 - - ######################################################################## - # Third reduction step - mov $acc2, $t1 - shl \$32, $acc2 - mulq $poly3 - shr \$32, $t1 - add $acc2, $acc3 - adc $t1, $acc4 - adc %rax, $acc5 - mov 8*3($b_ptr), %rax - adc %rdx, $acc0 - adc \$0, $acc1 - xor $acc2, $acc2 - - ######################################################################## - # Multiply by b[3] - mov %rax, $t1 - mulq 8*0($a_ptr) - add %rax, $acc3 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*1($a_ptr) - add $t0, $acc4 - adc \$0, %rdx - add %rax, $acc4 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*2($a_ptr) - add $t0, $acc5 - adc \$0, %rdx - add %rax, $acc5 - mov $t1, %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq 8*3($a_ptr) - add $t0, $acc0 - adc \$0, %rdx - add %rax, $acc0 - mov $acc3, %rax - adc %rdx, $acc1 - adc \$0, $acc2 - - ######################################################################## - # Final reduction step - mov $acc3, $t1 - shl \$32, $acc3 - mulq $poly3 - shr \$32, $t1 - add $acc3, $acc4 - adc $t1, $acc5 - mov $acc4, $t0 - adc %rax, $acc0 - adc %rdx, $acc1 - mov $acc5, $t1 - adc \$0, $acc2 - - ######################################################################## - # Branch-less conditional subtraction of P - sub \$-1, $acc4 # .Lpoly[0] - mov $acc0, $t2 - sbb $poly1, $acc5 # .Lpoly[1] - sbb \$0, $acc0 # .Lpoly[2] - mov $acc1, $t3 - sbb $poly3, $acc1 # .Lpoly[3] - sbb \$0, $acc2 - - cmovc $t0, $acc4 - cmovc $t1, $acc5 - mov $acc4, 8*0($r_ptr) - cmovc $t2, $acc0 - mov $acc5, 8*1($r_ptr) - cmovc $t3, $acc1 - mov $acc0, 8*2($r_ptr) - mov $acc1, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq - -################################################################################ -# void ecp_nistz256_sqr_mont( -# uint64_t res[4], -# uint64_t a[4]); - -# we optimize the square according to S.Gueron and V.Krasnov, -# "Speeding up Big-Number Squaring" -.globl ecp_nistz256_sqr_mont_nohw -.type ecp_nistz256_sqr_mont_nohw,\@function,2 -.align 32 -ecp_nistz256_sqr_mont_nohw: -.cfi_startproc - _CET_ENDBR - push %rbp -.cfi_push %rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 -.Lsqr_body: - mov 8*0($a_ptr), %rax - mov 8*1($a_ptr), $acc6 - mov 8*2($a_ptr), $acc7 - mov 8*3($a_ptr), $acc0 - - call __ecp_nistz256_sqr_montq - - mov 0(%rsp),%r15 -.cfi_restore %r15 - mov 8(%rsp),%r14 -.cfi_restore %r14 - mov 16(%rsp),%r13 -.cfi_restore %r13 - mov 24(%rsp),%r12 -.cfi_restore %r12 - mov 32(%rsp),%rbx -.cfi_restore %rbx - mov 40(%rsp),%rbp -.cfi_restore %rbp - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lsqr_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_sqr_mont_nohw,.-ecp_nistz256_sqr_mont_nohw - -.type __ecp_nistz256_sqr_montq,\@abi-omnipotent -.align 32 -__ecp_nistz256_sqr_montq: -.cfi_startproc - mov %rax, $acc5 - mulq $acc6 # a[1]*a[0] - mov %rax, $acc1 - mov $acc7, %rax - mov %rdx, $acc2 - - mulq $acc5 # a[0]*a[2] - add %rax, $acc2 - mov $acc0, %rax - adc \$0, %rdx - mov %rdx, $acc3 - - mulq $acc5 # a[0]*a[3] - add %rax, $acc3 - mov $acc7, %rax - adc \$0, %rdx - mov %rdx, $acc4 - - ################################# - mulq $acc6 # a[1]*a[2] - add %rax, $acc3 - mov $acc0, %rax - adc \$0, %rdx - mov %rdx, $t1 - - mulq $acc6 # a[1]*a[3] - add %rax, $acc4 - mov $acc0, %rax - adc \$0, %rdx - add $t1, $acc4 - mov %rdx, $acc5 - adc \$0, $acc5 - - ################################# - mulq $acc7 # a[2]*a[3] - xor $acc7, $acc7 - add %rax, $acc5 - mov 8*0($a_ptr), %rax - mov %rdx, $acc6 - adc \$0, $acc6 - - add $acc1, $acc1 # acc1:6<<1 - adc $acc2, $acc2 - adc $acc3, $acc3 - adc $acc4, $acc4 - adc $acc5, $acc5 - adc $acc6, $acc6 - adc \$0, $acc7 - - mulq %rax - mov %rax, $acc0 - mov 8*1($a_ptr), %rax - mov %rdx, $t0 - - mulq %rax - add $t0, $acc1 - adc %rax, $acc2 - mov 8*2($a_ptr), %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq %rax - add $t0, $acc3 - adc %rax, $acc4 - mov 8*3($a_ptr), %rax - adc \$0, %rdx - mov %rdx, $t0 - - mulq %rax - add $t0, $acc5 - adc %rax, $acc6 - mov $acc0, %rax - adc %rdx, $acc7 - - mov .Lpoly+8*1(%rip), $a_ptr - mov .Lpoly+8*3(%rip), $t1 - - ########################################## - # Now the reduction - # First iteration - mov $acc0, $t0 - shl \$32, $acc0 - mulq $t1 - shr \$32, $t0 - add $acc0, $acc1 # +=acc[0]<<96 - adc $t0, $acc2 - adc %rax, $acc3 - mov $acc1, %rax - adc \$0, %rdx - - ########################################## - # Second iteration - mov $acc1, $t0 - shl \$32, $acc1 - mov %rdx, $acc0 - mulq $t1 - shr \$32, $t0 - add $acc1, $acc2 - adc $t0, $acc3 - adc %rax, $acc0 - mov $acc2, %rax - adc \$0, %rdx - - ########################################## - # Third iteration - mov $acc2, $t0 - shl \$32, $acc2 - mov %rdx, $acc1 - mulq $t1 - shr \$32, $t0 - add $acc2, $acc3 - adc $t0, $acc0 - adc %rax, $acc1 - mov $acc3, %rax - adc \$0, %rdx - - ########################################### - # Last iteration - mov $acc3, $t0 - shl \$32, $acc3 - mov %rdx, $acc2 - mulq $t1 - shr \$32, $t0 - add $acc3, $acc0 - adc $t0, $acc1 - adc %rax, $acc2 - adc \$0, %rdx - xor $acc3, $acc3 - - ############################################ - # Add the rest of the acc - add $acc0, $acc4 - adc $acc1, $acc5 - mov $acc4, $acc0 - adc $acc2, $acc6 - adc %rdx, $acc7 - mov $acc5, $acc1 - adc \$0, $acc3 - - sub \$-1, $acc4 # .Lpoly[0] - mov $acc6, $acc2 - sbb $a_ptr, $acc5 # .Lpoly[1] - sbb \$0, $acc6 # .Lpoly[2] - mov $acc7, $t0 - sbb $t1, $acc7 # .Lpoly[3] - sbb \$0, $acc3 - - cmovc $acc0, $acc4 - cmovc $acc1, $acc5 - mov $acc4, 8*0($r_ptr) - cmovc $acc2, $acc6 - mov $acc5, 8*1($r_ptr) - cmovc $t0, $acc7 - mov $acc6, 8*2($r_ptr) - mov $acc7, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq -___ - -if ($addx) { -$code.=<<___; -.globl ecp_nistz256_mul_mont_adx -.type ecp_nistz256_mul_mont_adx,\@function,3 -.align 32 -ecp_nistz256_mul_mont_adx: -.cfi_startproc - _CET_ENDBR - push %rbp -.cfi_push %rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 -.Lmulx_body: - mov $b_org, $b_ptr - mov 8*0($b_org), %rdx - mov 8*0($a_ptr), $acc1 - mov 8*1($a_ptr), $acc2 - mov 8*2($a_ptr), $acc3 - mov 8*3($a_ptr), $acc4 - lea -128($a_ptr), $a_ptr # control u-op density - - call __ecp_nistz256_mul_montx - - mov 0(%rsp),%r15 -.cfi_restore %r15 - mov 8(%rsp),%r14 -.cfi_restore %r14 - mov 16(%rsp),%r13 -.cfi_restore %r13 - mov 24(%rsp),%r12 -.cfi_restore %r12 - mov 32(%rsp),%rbx -.cfi_restore %rbx - mov 40(%rsp),%rbp -.cfi_restore %rbp - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lmulx_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_mul_mont_adx,.-ecp_nistz256_mul_mont_adx - -.type __ecp_nistz256_mul_montx,\@abi-omnipotent -.align 32 -__ecp_nistz256_mul_montx: -.cfi_startproc - ######################################################################## - # Multiply by b[0] - mulx $acc1, $acc0, $acc1 - mulx $acc2, $t0, $acc2 - mov \$32, $poly1 - xor $acc5, $acc5 # cf=0 - mulx $acc3, $t1, $acc3 - mov .Lpoly+8*3(%rip), $poly3 - adc $t0, $acc1 - mulx $acc4, $t0, $acc4 - mov $acc0, %rdx - adc $t1, $acc2 - shlx $poly1,$acc0,$t1 - adc $t0, $acc3 - shrx $poly1,$acc0,$t0 - adc \$0, $acc4 - - ######################################################################## - # First reduction step - add $t1, $acc1 - adc $t0, $acc2 - - mulx $poly3, $t0, $t1 - mov 8*1($b_ptr), %rdx - adc $t0, $acc3 - adc $t1, $acc4 - adc \$0, $acc5 - xor $acc0, $acc0 # $acc0=0,cf=0,of=0 - - ######################################################################## - # Multiply by b[1] - mulx 8*0+128($a_ptr), $t0, $t1 - adcx $t0, $acc1 - adox $t1, $acc2 - - mulx 8*1+128($a_ptr), $t0, $t1 - adcx $t0, $acc2 - adox $t1, $acc3 - - mulx 8*2+128($a_ptr), $t0, $t1 - adcx $t0, $acc3 - adox $t1, $acc4 - - mulx 8*3+128($a_ptr), $t0, $t1 - mov $acc1, %rdx - adcx $t0, $acc4 - shlx $poly1, $acc1, $t0 - adox $t1, $acc5 - shrx $poly1, $acc1, $t1 - - adcx $acc0, $acc5 - adox $acc0, $acc0 - adc \$0, $acc0 - - ######################################################################## - # Second reduction step - add $t0, $acc2 - adc $t1, $acc3 - - mulx $poly3, $t0, $t1 - mov 8*2($b_ptr), %rdx - adc $t0, $acc4 - adc $t1, $acc5 - adc \$0, $acc0 - xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0 - - ######################################################################## - # Multiply by b[2] - mulx 8*0+128($a_ptr), $t0, $t1 - adcx $t0, $acc2 - adox $t1, $acc3 - - mulx 8*1+128($a_ptr), $t0, $t1 - adcx $t0, $acc3 - adox $t1, $acc4 - - mulx 8*2+128($a_ptr), $t0, $t1 - adcx $t0, $acc4 - adox $t1, $acc5 - - mulx 8*3+128($a_ptr), $t0, $t1 - mov $acc2, %rdx - adcx $t0, $acc5 - shlx $poly1, $acc2, $t0 - adox $t1, $acc0 - shrx $poly1, $acc2, $t1 - - adcx $acc1, $acc0 - adox $acc1, $acc1 - adc \$0, $acc1 - - ######################################################################## - # Third reduction step - add $t0, $acc3 - adc $t1, $acc4 - - mulx $poly3, $t0, $t1 - mov 8*3($b_ptr), %rdx - adc $t0, $acc5 - adc $t1, $acc0 - adc \$0, $acc1 - xor $acc2, $acc2 # $acc2=0,cf=0,of=0 - - ######################################################################## - # Multiply by b[3] - mulx 8*0+128($a_ptr), $t0, $t1 - adcx $t0, $acc3 - adox $t1, $acc4 - - mulx 8*1+128($a_ptr), $t0, $t1 - adcx $t0, $acc4 - adox $t1, $acc5 - - mulx 8*2+128($a_ptr), $t0, $t1 - adcx $t0, $acc5 - adox $t1, $acc0 - - mulx 8*3+128($a_ptr), $t0, $t1 - mov $acc3, %rdx - adcx $t0, $acc0 - shlx $poly1, $acc3, $t0 - adox $t1, $acc1 - shrx $poly1, $acc3, $t1 - - adcx $acc2, $acc1 - adox $acc2, $acc2 - adc \$0, $acc2 - - ######################################################################## - # Fourth reduction step - add $t0, $acc4 - adc $t1, $acc5 - - mulx $poly3, $t0, $t1 - mov $acc4, $t2 - mov .Lpoly+8*1(%rip), $poly1 - adc $t0, $acc0 - mov $acc5, $t3 - adc $t1, $acc1 - adc \$0, $acc2 - - ######################################################################## - # Branch-less conditional subtraction of P - xor %eax, %eax - mov $acc0, $t0 - sbb \$-1, $acc4 # .Lpoly[0] - sbb $poly1, $acc5 # .Lpoly[1] - sbb \$0, $acc0 # .Lpoly[2] - mov $acc1, $t1 - sbb $poly3, $acc1 # .Lpoly[3] - sbb \$0, $acc2 - - cmovc $t2, $acc4 - cmovc $t3, $acc5 - mov $acc4, 8*0($r_ptr) - cmovc $t0, $acc0 - mov $acc5, 8*1($r_ptr) - cmovc $t1, $acc1 - mov $acc0, 8*2($r_ptr) - mov $acc1, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx - -.globl ecp_nistz256_sqr_mont_adx -.type ecp_nistz256_sqr_mont_adx,\@function,2 -.align 32 -ecp_nistz256_sqr_mont_adx: -.cfi_startproc - _CET_ENDBR - push %rbp -.cfi_push %rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 -.Lsqrx_body: - mov 8*0($a_ptr), %rdx - mov 8*1($a_ptr), $acc6 - mov 8*2($a_ptr), $acc7 - mov 8*3($a_ptr), $acc0 - lea -128($a_ptr), $a_ptr # control u-op density - - call __ecp_nistz256_sqr_montx - - mov 0(%rsp),%r15 -.cfi_restore %r15 - mov 8(%rsp),%r14 -.cfi_restore %r14 - mov 16(%rsp),%r13 -.cfi_restore %r13 - mov 24(%rsp),%r12 -.cfi_restore %r12 - mov 32(%rsp),%rbx -.cfi_restore %rbx - mov 40(%rsp),%rbp -.cfi_restore %rbp - lea 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lsqrx_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_sqr_mont_adx,.-ecp_nistz256_sqr_mont_adx - -.type __ecp_nistz256_sqr_montx,\@abi-omnipotent -.align 32 -__ecp_nistz256_sqr_montx: -.cfi_startproc - mulx $acc6, $acc1, $acc2 # a[0]*a[1] - mulx $acc7, $t0, $acc3 # a[0]*a[2] - xor %eax, %eax - adc $t0, $acc2 - mulx $acc0, $t1, $acc4 # a[0]*a[3] - mov $acc6, %rdx - adc $t1, $acc3 - adc \$0, $acc4 - xor $acc5, $acc5 # $acc5=0,cf=0,of=0 - - ################################# - mulx $acc7, $t0, $t1 # a[1]*a[2] - adcx $t0, $acc3 - adox $t1, $acc4 - - mulx $acc0, $t0, $t1 # a[1]*a[3] - mov $acc7, %rdx - adcx $t0, $acc4 - adox $t1, $acc5 - adc \$0, $acc5 - - ################################# - mulx $acc0, $t0, $acc6 # a[2]*a[3] - mov 8*0+128($a_ptr), %rdx - xor $acc7, $acc7 # $acc7=0,cf=0,of=0 - adcx $acc1, $acc1 # acc1:6<<1 - adox $t0, $acc5 - adcx $acc2, $acc2 - adox $acc7, $acc6 # of=0 - - mulx %rdx, $acc0, $t1 - mov 8*1+128($a_ptr), %rdx - adcx $acc3, $acc3 - adox $t1, $acc1 - adcx $acc4, $acc4 - mulx %rdx, $t0, $t4 - mov 8*2+128($a_ptr), %rdx - adcx $acc5, $acc5 - adox $t0, $acc2 - adcx $acc6, $acc6 - .byte 0x67 - mulx %rdx, $t0, $t1 - mov 8*3+128($a_ptr), %rdx - adox $t4, $acc3 - adcx $acc7, $acc7 - adox $t0, $acc4 - mov \$32, $a_ptr - adox $t1, $acc5 - .byte 0x67,0x67 - mulx %rdx, $t0, $t4 - mov .Lpoly+8*3(%rip), %rdx - adox $t0, $acc6 - shlx $a_ptr, $acc0, $t0 - adox $t4, $acc7 - shrx $a_ptr, $acc0, $t4 - mov %rdx,$t1 - - # reduction step 1 - add $t0, $acc1 - adc $t4, $acc2 - - mulx $acc0, $t0, $acc0 - adc $t0, $acc3 - shlx $a_ptr, $acc1, $t0 - adc \$0, $acc0 - shrx $a_ptr, $acc1, $t4 - - # reduction step 2 - add $t0, $acc2 - adc $t4, $acc3 - - mulx $acc1, $t0, $acc1 - adc $t0, $acc0 - shlx $a_ptr, $acc2, $t0 - adc \$0, $acc1 - shrx $a_ptr, $acc2, $t4 - - # reduction step 3 - add $t0, $acc3 - adc $t4, $acc0 - - mulx $acc2, $t0, $acc2 - adc $t0, $acc1 - shlx $a_ptr, $acc3, $t0 - adc \$0, $acc2 - shrx $a_ptr, $acc3, $t4 - - # reduction step 4 - add $t0, $acc0 - adc $t4, $acc1 - - mulx $acc3, $t0, $acc3 - adc $t0, $acc2 - adc \$0, $acc3 - - xor $t3, $t3 - add $acc0, $acc4 # accumulate upper half - mov .Lpoly+8*1(%rip), $a_ptr - adc $acc1, $acc5 - mov $acc4, $acc0 - adc $acc2, $acc6 - adc $acc3, $acc7 - mov $acc5, $acc1 - adc \$0, $t3 - - sub \$-1, $acc4 # .Lpoly[0] - mov $acc6, $acc2 - sbb $a_ptr, $acc5 # .Lpoly[1] - sbb \$0, $acc6 # .Lpoly[2] - mov $acc7, $acc3 - sbb $t1, $acc7 # .Lpoly[3] - sbb \$0, $t3 - - cmovc $acc0, $acc4 - cmovc $acc1, $acc5 - mov $acc4, 8*0($r_ptr) - cmovc $acc2, $acc6 - mov $acc5, 8*1($r_ptr) - cmovc $acc3, $acc7 - mov $acc6, 8*2($r_ptr) - mov $acc7, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx -___ } -} -{ -my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); -my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7)); -my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15)); -my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15)); - -$code.=<<___; -################################################################################ -# void ecp_nistz256_select_w5_nohw(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w5_nohw -.type ecp_nistz256_select_w5_nohw,\@abi-omnipotent -.align 32 -ecp_nistz256_select_w5_nohw: -.cfi_startproc - _CET_ENDBR -___ -$code.=<<___ if ($win64); - lea -0x88(%rsp), %rax -.LSEH_begin_ecp_nistz256_select_w5_nohw: - .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp - .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax) - .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax) - .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax) - .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax) - .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax) - .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax) - .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax) - .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax) - .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax) - .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax) -___ -$code.=<<___; - movdqa .LOne(%rip), $ONE - movd $index, $INDEX - - pxor $Ra, $Ra - pxor $Rb, $Rb - pxor $Rc, $Rc - pxor $Rd, $Rd - pxor $Re, $Re - pxor $Rf, $Rf - - movdqa $ONE, $M0 - pshufd \$0, $INDEX, $INDEX - - mov \$16, %rax -.Lselect_loop_sse_w5: - - movdqa $M0, $TMP0 - paddd $ONE, $M0 - pcmpeqd $INDEX, $TMP0 - - movdqa 16*0($in_t), $T0a - movdqa 16*1($in_t), $T0b - movdqa 16*2($in_t), $T0c - movdqa 16*3($in_t), $T0d - movdqa 16*4($in_t), $T0e - movdqa 16*5($in_t), $T0f - lea 16*6($in_t), $in_t - - pand $TMP0, $T0a - pand $TMP0, $T0b - por $T0a, $Ra - pand $TMP0, $T0c - por $T0b, $Rb - pand $TMP0, $T0d - por $T0c, $Rc - pand $TMP0, $T0e - por $T0d, $Rd - pand $TMP0, $T0f - por $T0e, $Re - por $T0f, $Rf - - dec %rax - jnz .Lselect_loop_sse_w5 - - movdqu $Ra, 16*0($val) - movdqu $Rb, 16*1($val) - movdqu $Rc, 16*2($val) - movdqu $Rd, 16*3($val) - movdqu $Re, 16*4($val) - movdqu $Rf, 16*5($val) -___ -$code.=<<___ if ($win64); - movaps (%rsp), %xmm6 - movaps 0x10(%rsp), %xmm7 - movaps 0x20(%rsp), %xmm8 - movaps 0x30(%rsp), %xmm9 - movaps 0x40(%rsp), %xmm10 - movaps 0x50(%rsp), %xmm11 - movaps 0x60(%rsp), %xmm12 - movaps 0x70(%rsp), %xmm13 - movaps 0x80(%rsp), %xmm14 - movaps 0x90(%rsp), %xmm15 - lea 0xa8(%rsp), %rsp -___ -$code.=<<___; - ret -.cfi_endproc -.LSEH_end_ecp_nistz256_select_w5_nohw: -.size ecp_nistz256_select_w5_nohw,.-ecp_nistz256_select_w5_nohw - -################################################################################ -# void ecp_nistz256_select_w7_nohw(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w7_nohw -.type ecp_nistz256_select_w7_nohw,\@abi-omnipotent -.align 32 -ecp_nistz256_select_w7_nohw: -.cfi_startproc - _CET_ENDBR -___ -$code.=<<___ if ($win64); - lea -0x88(%rsp), %rax -.LSEH_begin_ecp_nistz256_select_w7_nohw: - .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp - .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax) - .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax) - .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax) - .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax) - .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax) - .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax) - .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax) - .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax) - .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax) - .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax) -___ -$code.=<<___; - movdqa .LOne(%rip), $M0 - movd $index, $INDEX - - pxor $Ra, $Ra - pxor $Rb, $Rb - pxor $Rc, $Rc - pxor $Rd, $Rd - - movdqa $M0, $ONE - pshufd \$0, $INDEX, $INDEX - mov \$64, %rax - -.Lselect_loop_sse_w7: - movdqa $M0, $TMP0 - paddd $ONE, $M0 - movdqa 16*0($in_t), $T0a - movdqa 16*1($in_t), $T0b - pcmpeqd $INDEX, $TMP0 - movdqa 16*2($in_t), $T0c - movdqa 16*3($in_t), $T0d - lea 16*4($in_t), $in_t - - pand $TMP0, $T0a - pand $TMP0, $T0b - por $T0a, $Ra - pand $TMP0, $T0c - por $T0b, $Rb - pand $TMP0, $T0d - por $T0c, $Rc - prefetcht0 255($in_t) - por $T0d, $Rd - - dec %rax - jnz .Lselect_loop_sse_w7 - - movdqu $Ra, 16*0($val) - movdqu $Rb, 16*1($val) - movdqu $Rc, 16*2($val) - movdqu $Rd, 16*3($val) -___ -$code.=<<___ if ($win64); - movaps (%rsp), %xmm6 - movaps 0x10(%rsp), %xmm7 - movaps 0x20(%rsp), %xmm8 - movaps 0x30(%rsp), %xmm9 - movaps 0x40(%rsp), %xmm10 - movaps 0x50(%rsp), %xmm11 - movaps 0x60(%rsp), %xmm12 - movaps 0x70(%rsp), %xmm13 - movaps 0x80(%rsp), %xmm14 - movaps 0x90(%rsp), %xmm15 - lea 0xa8(%rsp), %rsp -___ -$code.=<<___; - ret -.cfi_endproc -.LSEH_end_ecp_nistz256_select_w7_nohw: -.size ecp_nistz256_select_w7_nohw,.-ecp_nistz256_select_w7_nohw -___ -} -if ($avx>1) { -my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); -my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4)); -my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9)); -my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14)); - -$code.=<<___; -################################################################################ -# void ecp_nistz256_select_w5_avx2(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w5_avx2 -.type ecp_nistz256_select_w5_avx2,\@abi-omnipotent -.align 32 -ecp_nistz256_select_w5_avx2: -.cfi_startproc - _CET_ENDBR - vzeroupper -___ -$code.=<<___ if ($win64); - lea -0x88(%rsp), %rax - mov %rsp,%r11 -.LSEH_begin_ecp_nistz256_select_w5_avx2: - .byte 0x48,0x8d,0x60,0xe0 # lea -0x20(%rax), %rsp - .byte 0xc5,0xf8,0x29,0x70,0xe0 # vmovaps %xmm6, -0x20(%rax) - .byte 0xc5,0xf8,0x29,0x78,0xf0 # vmovaps %xmm7, -0x10(%rax) - .byte 0xc5,0x78,0x29,0x40,0x00 # vmovaps %xmm8, 8(%rax) - .byte 0xc5,0x78,0x29,0x48,0x10 # vmovaps %xmm9, 0x10(%rax) - .byte 0xc5,0x78,0x29,0x50,0x20 # vmovaps %xmm10, 0x20(%rax) - .byte 0xc5,0x78,0x29,0x58,0x30 # vmovaps %xmm11, 0x30(%rax) - .byte 0xc5,0x78,0x29,0x60,0x40 # vmovaps %xmm12, 0x40(%rax) - .byte 0xc5,0x78,0x29,0x68,0x50 # vmovaps %xmm13, 0x50(%rax) - .byte 0xc5,0x78,0x29,0x70,0x60 # vmovaps %xmm14, 0x60(%rax) - .byte 0xc5,0x78,0x29,0x78,0x70 # vmovaps %xmm15, 0x70(%rax) -___ -$code.=<<___; - vmovdqa .LTwo(%rip), $TWO - - vpxor $Ra, $Ra, $Ra - vpxor $Rb, $Rb, $Rb - vpxor $Rc, $Rc, $Rc - - vmovdqa .LOne(%rip), $M0 - vmovdqa .LTwo(%rip), $M1 - - vmovd $index, %xmm1 - vpermd $INDEX, $Ra, $INDEX - - mov \$8, %rax -.Lselect_loop_avx2_w5: - - vmovdqa 32*0($in_t), $T0a - vmovdqa 32*1($in_t), $T0b - vmovdqa 32*2($in_t), $T0c - - vmovdqa 32*3($in_t), $T1a - vmovdqa 32*4($in_t), $T1b - vmovdqa 32*5($in_t), $T1c - - vpcmpeqd $INDEX, $M0, $TMP0 - vpcmpeqd $INDEX, $M1, $TMP1 - - vpaddd $TWO, $M0, $M0 - vpaddd $TWO, $M1, $M1 - lea 32*6($in_t), $in_t - - vpand $TMP0, $T0a, $T0a - vpand $TMP0, $T0b, $T0b - vpand $TMP0, $T0c, $T0c - vpand $TMP1, $T1a, $T1a - vpand $TMP1, $T1b, $T1b - vpand $TMP1, $T1c, $T1c - - vpxor $T0a, $Ra, $Ra - vpxor $T0b, $Rb, $Rb - vpxor $T0c, $Rc, $Rc - vpxor $T1a, $Ra, $Ra - vpxor $T1b, $Rb, $Rb - vpxor $T1c, $Rc, $Rc - - dec %rax - jnz .Lselect_loop_avx2_w5 - - vmovdqu $Ra, 32*0($val) - vmovdqu $Rb, 32*1($val) - vmovdqu $Rc, 32*2($val) - vzeroupper -___ -$code.=<<___ if ($win64); - movaps (%rsp), %xmm6 - movaps 0x10(%rsp), %xmm7 - movaps 0x20(%rsp), %xmm8 - movaps 0x30(%rsp), %xmm9 - movaps 0x40(%rsp), %xmm10 - movaps 0x50(%rsp), %xmm11 - movaps 0x60(%rsp), %xmm12 - movaps 0x70(%rsp), %xmm13 - movaps 0x80(%rsp), %xmm14 - movaps 0x90(%rsp), %xmm15 - lea (%r11), %rsp -___ -$code.=<<___; - ret -.cfi_endproc -.LSEH_end_ecp_nistz256_select_w5_avx2: -.size ecp_nistz256_select_w5_avx2,.-ecp_nistz256_select_w5_avx2 -___ -} -if ($avx>1) { -my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx"); -my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3)); -my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7)); -my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11)); -my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15)); - -$code.=<<___; - -################################################################################ -# void ecp_nistz256_select_w7_avx2(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w7_avx2 -.type ecp_nistz256_select_w7_avx2,\@abi-omnipotent -.align 32 -ecp_nistz256_select_w7_avx2: -.cfi_startproc - _CET_ENDBR - vzeroupper -___ -$code.=<<___ if ($win64); - mov %rsp,%r11 - lea -0x88(%rsp), %rax -.LSEH_begin_ecp_nistz256_select_w7_avx2: - .byte 0x48,0x8d,0x60,0xe0 # lea -0x20(%rax), %rsp - .byte 0xc5,0xf8,0x29,0x70,0xe0 # vmovaps %xmm6, -0x20(%rax) - .byte 0xc5,0xf8,0x29,0x78,0xf0 # vmovaps %xmm7, -0x10(%rax) - .byte 0xc5,0x78,0x29,0x40,0x00 # vmovaps %xmm8, 8(%rax) - .byte 0xc5,0x78,0x29,0x48,0x10 # vmovaps %xmm9, 0x10(%rax) - .byte 0xc5,0x78,0x29,0x50,0x20 # vmovaps %xmm10, 0x20(%rax) - .byte 0xc5,0x78,0x29,0x58,0x30 # vmovaps %xmm11, 0x30(%rax) - .byte 0xc5,0x78,0x29,0x60,0x40 # vmovaps %xmm12, 0x40(%rax) - .byte 0xc5,0x78,0x29,0x68,0x50 # vmovaps %xmm13, 0x50(%rax) - .byte 0xc5,0x78,0x29,0x70,0x60 # vmovaps %xmm14, 0x60(%rax) - .byte 0xc5,0x78,0x29,0x78,0x70 # vmovaps %xmm15, 0x70(%rax) -___ -$code.=<<___; - vmovdqa .LThree(%rip), $THREE - - vpxor $Ra, $Ra, $Ra - vpxor $Rb, $Rb, $Rb - - vmovdqa .LOne(%rip), $M0 - vmovdqa .LTwo(%rip), $M1 - vmovdqa .LThree(%rip), $M2 - - vmovd $index, %xmm1 - vpermd $INDEX, $Ra, $INDEX - # Skip index = 0, because it is implicitly the point at infinity - - mov \$21, %rax -.Lselect_loop_avx2_w7: - - vmovdqa 32*0($in_t), $T0a - vmovdqa 32*1($in_t), $T0b - - vmovdqa 32*2($in_t), $T1a - vmovdqa 32*3($in_t), $T1b - - vmovdqa 32*4($in_t), $T2a - vmovdqa 32*5($in_t), $T2b - - vpcmpeqd $INDEX, $M0, $TMP0 - vpcmpeqd $INDEX, $M1, $TMP1 - vpcmpeqd $INDEX, $M2, $TMP2 - - vpaddd $THREE, $M0, $M0 - vpaddd $THREE, $M1, $M1 - vpaddd $THREE, $M2, $M2 - lea 32*6($in_t), $in_t - - vpand $TMP0, $T0a, $T0a - vpand $TMP0, $T0b, $T0b - vpand $TMP1, $T1a, $T1a - vpand $TMP1, $T1b, $T1b - vpand $TMP2, $T2a, $T2a - vpand $TMP2, $T2b, $T2b - - vpxor $T0a, $Ra, $Ra - vpxor $T0b, $Rb, $Rb - vpxor $T1a, $Ra, $Ra - vpxor $T1b, $Rb, $Rb - vpxor $T2a, $Ra, $Ra - vpxor $T2b, $Rb, $Rb - - dec %rax - jnz .Lselect_loop_avx2_w7 - - - vmovdqa 32*0($in_t), $T0a - vmovdqa 32*1($in_t), $T0b - - vpcmpeqd $INDEX, $M0, $TMP0 - - vpand $TMP0, $T0a, $T0a - vpand $TMP0, $T0b, $T0b - - vpxor $T0a, $Ra, $Ra - vpxor $T0b, $Rb, $Rb - - vmovdqu $Ra, 32*0($val) - vmovdqu $Rb, 32*1($val) - vzeroupper -___ -$code.=<<___ if ($win64); - movaps (%rsp), %xmm6 - movaps 0x10(%rsp), %xmm7 - movaps 0x20(%rsp), %xmm8 - movaps 0x30(%rsp), %xmm9 - movaps 0x40(%rsp), %xmm10 - movaps 0x50(%rsp), %xmm11 - movaps 0x60(%rsp), %xmm12 - movaps 0x70(%rsp), %xmm13 - movaps 0x80(%rsp), %xmm14 - movaps 0x90(%rsp), %xmm15 - lea (%r11), %rsp -___ -$code.=<<___; - ret -.cfi_endproc -.LSEH_end_ecp_nistz256_select_w7_avx2: -.size ecp_nistz256_select_w7_avx2,.-ecp_nistz256_select_w7_avx2 -___ -} else { -$code.=<<___; -.globl ecp_nistz256_select_w7_avx2 -.type ecp_nistz256_select_w7_avx2,\@function,3 -.align 32 -ecp_nistz256_select_w7_avx2: - _CET_ENDBR - .byte 0x0f,0x0b # ud2 - ret -.size ecp_nistz256_select_w7_avx2,.-ecp_nistz256_select_w7_avx2 -___ -} -{{{ -######################################################################## -# This block implements higher level point_double, point_add and -# point_add_affine. The key to performance in this case is to allow -# out-of-order execution logic to overlap computations from next step -# with tail processing from current step. By using tailored calling -# sequence we minimize inter-step overhead to give processor better -# shot at overlapping operations... -# -# You will notice that input data is copied to stack. Trouble is that -# there are no registers to spare for holding original pointers and -# reloading them, pointers, would create undesired dependencies on -# effective addresses calculation paths. In other words it's too done -# to favour out-of-order execution logic. -# <appro@openssl.org> - -my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx"); -my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15)); -my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4); -my ($poly1,$poly3)=($acc6,$acc7); - -sub load_for_mul () { -my ($a,$b,$src0) = @_; -my $bias = $src0 eq "%rax" ? 0 : -128; - -" mov $b, $src0 - lea $b, $b_ptr - mov 8*0+$a, $acc1 - mov 8*1+$a, $acc2 - lea $bias+$a, $a_ptr - mov 8*2+$a, $acc3 - mov 8*3+$a, $acc4" -} - -sub load_for_sqr () { -my ($a,$src0) = @_; -my $bias = $src0 eq "%rax" ? 0 : -128; - -" mov 8*0+$a, $src0 - mov 8*1+$a, $acc6 - lea $bias+$a, $a_ptr - mov 8*2+$a, $acc7 - mov 8*3+$a, $acc0" -} - - { -######################################################################## -# operate in 4-5-0-1 "name space" that matches multiplication output -# -my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); - -$code.=<<___; -.type __ecp_nistz256_add_toq,\@abi-omnipotent -.align 32 -__ecp_nistz256_add_toq: -.cfi_startproc - xor $t4,$t4 - add 8*0($b_ptr), $a0 - adc 8*1($b_ptr), $a1 - mov $a0, $t0 - adc 8*2($b_ptr), $a2 - adc 8*3($b_ptr), $a3 - mov $a1, $t1 - adc \$0, $t4 - - sub \$-1, $a0 - mov $a2, $t2 - sbb $poly1, $a1 - sbb \$0, $a2 - mov $a3, $t3 - sbb $poly3, $a3 - sbb \$0, $t4 - - cmovc $t0, $a0 - cmovc $t1, $a1 - mov $a0, 8*0($r_ptr) - cmovc $t2, $a2 - mov $a1, 8*1($r_ptr) - cmovc $t3, $a3 - mov $a2, 8*2($r_ptr) - mov $a3, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq - -.type __ecp_nistz256_sub_fromq,\@abi-omnipotent -.align 32 -__ecp_nistz256_sub_fromq: -.cfi_startproc - sub 8*0($b_ptr), $a0 - sbb 8*1($b_ptr), $a1 - mov $a0, $t0 - sbb 8*2($b_ptr), $a2 - sbb 8*3($b_ptr), $a3 - mov $a1, $t1 - sbb $t4, $t4 - - add \$-1, $a0 - mov $a2, $t2 - adc $poly1, $a1 - adc \$0, $a2 - mov $a3, $t3 - adc $poly3, $a3 - test $t4, $t4 - - cmovz $t0, $a0 - cmovz $t1, $a1 - mov $a0, 8*0($r_ptr) - cmovz $t2, $a2 - mov $a1, 8*1($r_ptr) - cmovz $t3, $a3 - mov $a2, 8*2($r_ptr) - mov $a3, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq - -.type __ecp_nistz256_subq,\@abi-omnipotent -.align 32 -__ecp_nistz256_subq: -.cfi_startproc - sub $a0, $t0 - sbb $a1, $t1 - mov $t0, $a0 - sbb $a2, $t2 - sbb $a3, $t3 - mov $t1, $a1 - sbb $t4, $t4 - - add \$-1, $t0 - mov $t2, $a2 - adc $poly1, $t1 - adc \$0, $t2 - mov $t3, $a3 - adc $poly3, $t3 - test $t4, $t4 - - cmovnz $t0, $a0 - cmovnz $t1, $a1 - cmovnz $t2, $a2 - cmovnz $t3, $a3 - - ret -.cfi_endproc -.size __ecp_nistz256_subq,.-__ecp_nistz256_subq - -.type __ecp_nistz256_mul_by_2q,\@abi-omnipotent -.align 32 -__ecp_nistz256_mul_by_2q: -.cfi_startproc - xor $t4, $t4 - add $a0, $a0 # a0:a3+a0:a3 - adc $a1, $a1 - mov $a0, $t0 - adc $a2, $a2 - adc $a3, $a3 - mov $a1, $t1 - adc \$0, $t4 - - sub \$-1, $a0 - mov $a2, $t2 - sbb $poly1, $a1 - sbb \$0, $a2 - mov $a3, $t3 - sbb $poly3, $a3 - sbb \$0, $t4 - - cmovc $t0, $a0 - cmovc $t1, $a1 - mov $a0, 8*0($r_ptr) - cmovc $t2, $a2 - mov $a1, 8*1($r_ptr) - cmovc $t3, $a3 - mov $a2, 8*2($r_ptr) - mov $a3, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q -___ - } -sub gen_double () { - my $x = shift; - my ($src0,$sfx,$bias); - my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4)); - - if ($x ne "x") { - $src0 = "%rax"; - $sfx = "_nohw"; - $bias = 0; - } else { - $src0 = "%rdx"; - $sfx = "_adx"; - $bias = 128; - } -$code.=<<___; -.globl ecp_nistz256_point_double$sfx -.type ecp_nistz256_point_double$sfx,\@function,2 -.align 32 -ecp_nistz256_point_double$sfx: -.cfi_startproc - _CET_ENDBR - push %rbp -.cfi_push %rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 - sub \$32*5+8, %rsp -.cfi_adjust_cfa_offset 32*5+8 -.Lpoint_double${x}_body: - -.Lpoint_double_shortcut$x: - movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x - mov $a_ptr, $b_ptr # backup copy - movdqu 0x10($a_ptr), %xmm1 - mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order - mov 0x20+8*1($a_ptr), $acc5 - mov 0x20+8*2($a_ptr), $acc0 - mov 0x20+8*3($a_ptr), $acc1 - mov .Lpoly+8*1(%rip), $poly1 - mov .Lpoly+8*3(%rip), $poly3 - movdqa %xmm0, $in_x(%rsp) - movdqa %xmm1, $in_x+0x10(%rsp) - lea 0x20($r_ptr), $acc2 - lea 0x40($r_ptr), $acc3 - movq $r_ptr, %xmm0 - movq $acc2, %xmm1 - movq $acc3, %xmm2 - - lea $S(%rsp), $r_ptr - call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y); - - mov 0x40+8*0($a_ptr), $src0 - mov 0x40+8*1($a_ptr), $acc6 - mov 0x40+8*2($a_ptr), $acc7 - mov 0x40+8*3($a_ptr), $acc0 - lea 0x40-$bias($a_ptr), $a_ptr - lea $Zsqr(%rsp), $r_ptr - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z); - - `&load_for_sqr("$S(%rsp)", "$src0")` - lea $S(%rsp), $r_ptr - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S); - - mov 0x20($b_ptr), $src0 # $b_ptr is still valid - mov 0x40+8*0($b_ptr), $acc1 - mov 0x40+8*1($b_ptr), $acc2 - mov 0x40+8*2($b_ptr), $acc3 - mov 0x40+8*3($b_ptr), $acc4 - lea 0x40-$bias($b_ptr), $a_ptr - lea 0x20($b_ptr), $b_ptr - movq %xmm2, $r_ptr - call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y); - call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z); - - mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order - mov $in_x+8*1(%rsp), $acc5 - lea $Zsqr(%rsp), $b_ptr - mov $in_x+8*2(%rsp), $acc0 - mov $in_x+8*3(%rsp), $acc1 - lea $M(%rsp), $r_ptr - call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr); - - mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order - mov $in_x+8*1(%rsp), $acc5 - lea $Zsqr(%rsp), $b_ptr - mov $in_x+8*2(%rsp), $acc0 - mov $in_x+8*3(%rsp), $acc1 - lea $Zsqr(%rsp), $r_ptr - call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr); - - `&load_for_sqr("$S(%rsp)", "$src0")` - movq %xmm1, $r_ptr - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S); -___ -{ -######## ecp_nistz256_div_by_2(res_y, res_y); ########################## -# operate in 4-5-6-7 "name space" that matches squaring output -# -my ($poly1,$poly3)=($a_ptr,$t1); -my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2); - -$code.=<<___; - xor $t4, $t4 - mov $a0, $t0 - add \$-1, $a0 - mov $a1, $t1 - adc $poly1, $a1 - mov $a2, $t2 - adc \$0, $a2 - mov $a3, $t3 - adc $poly3, $a3 - adc \$0, $t4 - xor $a_ptr, $a_ptr # borrow $a_ptr - test \$1, $t0 - - cmovz $t0, $a0 - cmovz $t1, $a1 - cmovz $t2, $a2 - cmovz $t3, $a3 - cmovz $a_ptr, $t4 - - mov $a1, $t0 # a0:a3>>1 - shr \$1, $a0 - shl \$63, $t0 - mov $a2, $t1 - shr \$1, $a1 - or $t0, $a0 - shl \$63, $t1 - mov $a3, $t2 - shr \$1, $a2 - or $t1, $a1 - shl \$63, $t2 - mov $a0, 8*0($r_ptr) - shr \$1, $a3 - mov $a1, 8*1($r_ptr) - shl \$63, $t4 - or $t2, $a2 - or $t4, $a3 - mov $a2, 8*2($r_ptr) - mov $a3, 8*3($r_ptr) -___ -} -$code.=<<___; - `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")` - lea $M(%rsp), $r_ptr - call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr); - - lea $tmp0(%rsp), $r_ptr - call __ecp_nistz256_mul_by_2$x - - lea $M(%rsp), $b_ptr - lea $M(%rsp), $r_ptr - call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M); - - `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")` - lea $S(%rsp), $r_ptr - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x); - - lea $tmp0(%rsp), $r_ptr - call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S); - - `&load_for_sqr("$M(%rsp)", "$src0")` - movq %xmm0, $r_ptr - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M); - - lea $tmp0(%rsp), $b_ptr - mov $acc6, $acc0 # harmonize sqr output and sub input - mov $acc7, $acc1 - mov $a_ptr, $poly1 - mov $t1, $poly3 - call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0); - - mov $S+8*0(%rsp), $t0 - mov $S+8*1(%rsp), $t1 - mov $S+8*2(%rsp), $t2 - mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order - lea $S(%rsp), $r_ptr - call __ecp_nistz256_sub$x # p256_sub(S, S, res_x); - - mov $M(%rsp), $src0 - lea $M(%rsp), $b_ptr - mov $acc4, $acc6 # harmonize sub output and mul input - xor %ecx, %ecx - mov $acc4, $S+8*0(%rsp) # have to save:-( - mov $acc5, $acc2 - mov $acc5, $S+8*1(%rsp) - cmovz $acc0, $acc3 - mov $acc0, $S+8*2(%rsp) - lea $S-$bias(%rsp), $a_ptr - cmovz $acc1, $acc4 - mov $acc1, $S+8*3(%rsp) - mov $acc6, $acc1 - lea $S(%rsp), $r_ptr - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M); - - movq %xmm1, $b_ptr - movq %xmm1, $r_ptr - call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y); - - lea 32*5+56(%rsp), %rsi -.cfi_def_cfa %rsi,8 - mov -48(%rsi),%r15 -.cfi_restore %r15 - mov -40(%rsi),%r14 -.cfi_restore %r14 - mov -32(%rsi),%r13 -.cfi_restore %r13 - mov -24(%rsi),%r12 -.cfi_restore %r12 - mov -16(%rsi),%rbx -.cfi_restore %rbx - mov -8(%rsi),%rbp -.cfi_restore %rbp - lea (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Lpoint_double${x}_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx -___ -} -&gen_double("q"); - -sub gen_add () { - my $x = shift; - my ($src0,$sfx,$bias); - my ($H,$Hsqr,$R,$Rsqr,$Hcub, - $U1,$U2,$S1,$S2, - $res_x,$res_y,$res_z, - $in1_x,$in1_y,$in1_z, - $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17)); - my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr); - - if ($x ne "x") { - $src0 = "%rax"; - $sfx = "_nohw"; - $bias = 0; - } else { - $src0 = "%rdx"; - $sfx = "_adx"; - $bias = 128; - } -$code.=<<___; -.globl ecp_nistz256_point_add$sfx -.type ecp_nistz256_point_add$sfx,\@function,3 -.align 32 -ecp_nistz256_point_add$sfx: -.cfi_startproc - _CET_ENDBR - push %rbp -.cfi_push %rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 - sub \$32*18+8, %rsp -.cfi_adjust_cfa_offset 32*18+8 -.Lpoint_add${x}_body: - - movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr - movdqu 0x10($a_ptr), %xmm1 - movdqu 0x20($a_ptr), %xmm2 - movdqu 0x30($a_ptr), %xmm3 - movdqu 0x40($a_ptr), %xmm4 - movdqu 0x50($a_ptr), %xmm5 - mov $a_ptr, $b_ptr # reassign - mov $b_org, $a_ptr # reassign - movdqa %xmm0, $in1_x(%rsp) - movdqa %xmm1, $in1_x+0x10(%rsp) - movdqa %xmm2, $in1_y(%rsp) - movdqa %xmm3, $in1_y+0x10(%rsp) - movdqa %xmm4, $in1_z(%rsp) - movdqa %xmm5, $in1_z+0x10(%rsp) - por %xmm4, %xmm5 - - movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr - pshufd \$0xb1, %xmm5, %xmm3 - movdqu 0x10($a_ptr), %xmm1 - movdqu 0x20($a_ptr), %xmm2 - por %xmm3, %xmm5 - movdqu 0x30($a_ptr), %xmm3 - mov 0x40+8*0($a_ptr), $src0 # load original in2_z - mov 0x40+8*1($a_ptr), $acc6 - mov 0x40+8*2($a_ptr), $acc7 - mov 0x40+8*3($a_ptr), $acc0 - movdqa %xmm0, $in2_x(%rsp) - pshufd \$0x1e, %xmm5, %xmm4 - movdqa %xmm1, $in2_x+0x10(%rsp) - movdqu 0x40($a_ptr),%xmm0 # in2_z again - movdqu 0x50($a_ptr),%xmm1 - movdqa %xmm2, $in2_y(%rsp) - movdqa %xmm3, $in2_y+0x10(%rsp) - por %xmm4, %xmm5 - pxor %xmm4, %xmm4 - por %xmm0, %xmm1 - movq $r_ptr, %xmm0 # save $r_ptr - - lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid - mov $src0, $in2_z+8*0(%rsp) # make in2_z copy - mov $acc6, $in2_z+8*1(%rsp) - mov $acc7, $in2_z+8*2(%rsp) - mov $acc0, $in2_z+8*3(%rsp) - lea $Z2sqr(%rsp), $r_ptr # Z2^2 - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z); - - pcmpeqd %xmm4, %xmm5 - pshufd \$0xb1, %xmm1, %xmm4 - por %xmm1, %xmm4 - pshufd \$0, %xmm5, %xmm5 # in1infty - pshufd \$0x1e, %xmm4, %xmm3 - por %xmm3, %xmm4 - pxor %xmm3, %xmm3 - pcmpeqd %xmm3, %xmm4 - pshufd \$0, %xmm4, %xmm4 # in2infty - mov 0x40+8*0($b_ptr), $src0 # load original in1_z - mov 0x40+8*1($b_ptr), $acc6 - mov 0x40+8*2($b_ptr), $acc7 - mov 0x40+8*3($b_ptr), $acc0 - movq $b_ptr, %xmm1 - - lea 0x40-$bias($b_ptr), $a_ptr - lea $Z1sqr(%rsp), $r_ptr # Z1^2 - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z); - - `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")` - lea $S1(%rsp), $r_ptr # S1 = Z2^3 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z); - - `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")` - lea $S2(%rsp), $r_ptr # S2 = Z1^3 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z); - - `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")` - lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y); - - `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")` - lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y); - - lea $S1(%rsp), $b_ptr - lea $R(%rsp), $r_ptr # R = S2 - S1 - call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1); - - or $acc5, $acc4 # see if result is zero - movdqa %xmm4, %xmm2 - or $acc0, $acc4 - or $acc1, $acc4 - por %xmm5, %xmm2 # in1infty || in2infty - movq $acc4, %xmm3 - - `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")` - lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr); - - `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")` - lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr); - - lea $U1(%rsp), $b_ptr - lea $H(%rsp), $r_ptr # H = U2 - U1 - call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1); - - or $acc5, $acc4 # see if result is zero - or $acc0, $acc4 - or $acc1, $acc4 # !is_equal(U1, U2) - - movq %xmm2, $acc0 - movq %xmm3, $acc1 - or $acc0, $acc4 - .byte 0x3e # predict taken - jnz .Ladd_proceed$x # !is_equal(U1, U2) || in1infty || in2infty - - # We now know A = B or A = -B and neither is infinity. Compare the - # y-coordinates via S1 and S2. - test $acc1, $acc1 - jz .Ladd_double$x # is_equal(S1, S2) - - # A = -B, so the result is infinity. - # - # TODO(davidben): Does .Ladd_proceed handle this case? It seems to, in - # which case we should eliminate this special-case and simplify the - # timing analysis. - movq %xmm0, $r_ptr # restore $r_ptr - pxor %xmm0, %xmm0 - movdqu %xmm0, 0x00($r_ptr) - movdqu %xmm0, 0x10($r_ptr) - movdqu %xmm0, 0x20($r_ptr) - movdqu %xmm0, 0x30($r_ptr) - movdqu %xmm0, 0x40($r_ptr) - movdqu %xmm0, 0x50($r_ptr) - jmp .Ladd_done$x - -.align 32 -.Ladd_double$x: - movq %xmm1, $a_ptr # restore $a_ptr - movq %xmm0, $r_ptr # restore $r_ptr - add \$`32*(18-5)`, %rsp # difference in frame sizes -.cfi_adjust_cfa_offset `-32*(18-5)` - jmp .Lpoint_double_shortcut$x -.cfi_adjust_cfa_offset `32*(18-5)` - -.align 32 -.Ladd_proceed$x: - `&load_for_sqr("$R(%rsp)", "$src0")` - lea $Rsqr(%rsp), $r_ptr # R^2 - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R); - - `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")` - lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z); - - `&load_for_sqr("$H(%rsp)", "$src0")` - lea $Hsqr(%rsp), $r_ptr # H^2 - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H); - - `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")` - lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z); - - `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")` - lea $Hcub(%rsp), $r_ptr # H^3 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H); - - `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")` - lea $U2(%rsp), $r_ptr # U1*H^2 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr); -___ -{ -####################################################################### -# operate in 4-5-0-1 "name space" that matches multiplication output -# -my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); -my ($poly1, $poly3)=($acc6,$acc7); - -$code.=<<___; - #lea $U2(%rsp), $a_ptr - #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2 - #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); - - xor $t4, $t4 - add $acc0, $acc0 # a0:a3+a0:a3 - lea $Rsqr(%rsp), $a_ptr - adc $acc1, $acc1 - mov $acc0, $t0 - adc $acc2, $acc2 - adc $acc3, $acc3 - mov $acc1, $t1 - adc \$0, $t4 - - sub \$-1, $acc0 - mov $acc2, $t2 - sbb $poly1, $acc1 - sbb \$0, $acc2 - mov $acc3, $t3 - sbb $poly3, $acc3 - sbb \$0, $t4 - - cmovc $t0, $acc0 - mov 8*0($a_ptr), $t0 - cmovc $t1, $acc1 - mov 8*1($a_ptr), $t1 - cmovc $t2, $acc2 - mov 8*2($a_ptr), $t2 - cmovc $t3, $acc3 - mov 8*3($a_ptr), $t3 - - call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr); - - lea $Hcub(%rsp), $b_ptr - lea $res_x(%rsp), $r_ptr - call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub); - - mov $U2+8*0(%rsp), $t0 - mov $U2+8*1(%rsp), $t1 - mov $U2+8*2(%rsp), $t2 - mov $U2+8*3(%rsp), $t3 - lea $res_y(%rsp), $r_ptr - - call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x); - - mov $acc0, 8*0($r_ptr) # save the result, as - mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't - mov $acc2, 8*2($r_ptr) - mov $acc3, 8*3($r_ptr) -___ -} -$code.=<<___; - `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")` - lea $S2(%rsp), $r_ptr - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub); - - `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")` - lea $res_y(%rsp), $r_ptr - call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y); - - lea $S2(%rsp), $b_ptr - lea $res_y(%rsp), $r_ptr - call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2); - - movq %xmm0, $r_ptr # restore $r_ptr - - movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty); - movdqa %xmm5, %xmm1 - pandn $res_z(%rsp), %xmm0 - movdqa %xmm5, %xmm2 - pandn $res_z+0x10(%rsp), %xmm1 - movdqa %xmm5, %xmm3 - pand $in2_z(%rsp), %xmm2 - pand $in2_z+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - - movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty); - movdqa %xmm4, %xmm1 - pandn %xmm2, %xmm0 - movdqa %xmm4, %xmm2 - pandn %xmm3, %xmm1 - movdqa %xmm4, %xmm3 - pand $in1_z(%rsp), %xmm2 - pand $in1_z+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - movdqu %xmm2, 0x40($r_ptr) - movdqu %xmm3, 0x50($r_ptr) - - movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty); - movdqa %xmm5, %xmm1 - pandn $res_x(%rsp), %xmm0 - movdqa %xmm5, %xmm2 - pandn $res_x+0x10(%rsp), %xmm1 - movdqa %xmm5, %xmm3 - pand $in2_x(%rsp), %xmm2 - pand $in2_x+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - - movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty); - movdqa %xmm4, %xmm1 - pandn %xmm2, %xmm0 - movdqa %xmm4, %xmm2 - pandn %xmm3, %xmm1 - movdqa %xmm4, %xmm3 - pand $in1_x(%rsp), %xmm2 - pand $in1_x+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - movdqu %xmm2, 0x00($r_ptr) - movdqu %xmm3, 0x10($r_ptr) - - movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty); - movdqa %xmm5, %xmm1 - pandn $res_y(%rsp), %xmm0 - movdqa %xmm5, %xmm2 - pandn $res_y+0x10(%rsp), %xmm1 - movdqa %xmm5, %xmm3 - pand $in2_y(%rsp), %xmm2 - pand $in2_y+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - - movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty); - movdqa %xmm4, %xmm1 - pandn %xmm2, %xmm0 - movdqa %xmm4, %xmm2 - pandn %xmm3, %xmm1 - movdqa %xmm4, %xmm3 - pand $in1_y(%rsp), %xmm2 - pand $in1_y+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - movdqu %xmm2, 0x20($r_ptr) - movdqu %xmm3, 0x30($r_ptr) - -.Ladd_done$x: - lea 32*18+56(%rsp), %rsi -.cfi_def_cfa %rsi,8 - mov -48(%rsi),%r15 -.cfi_restore %r15 - mov -40(%rsi),%r14 -.cfi_restore %r14 - mov -32(%rsi),%r13 -.cfi_restore %r13 - mov -24(%rsi),%r12 -.cfi_restore %r12 - mov -16(%rsi),%rbx -.cfi_restore %rbx - mov -8(%rsi),%rbp -.cfi_restore %rbp - lea (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Lpoint_add${x}_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx -___ -} -&gen_add("q"); - -sub gen_add_affine () { - my $x = shift; - my ($src0,$sfx,$bias); - my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr, - $res_x,$res_y,$res_z, - $in1_x,$in1_y,$in1_z, - $in2_x,$in2_y)=map(32*$_,(0..14)); - my $Z1sqr = $S2; - - if ($x ne "x") { - $src0 = "%rax"; - $sfx = "_nohw"; - $bias = 0; - } else { - $src0 = "%rdx"; - $sfx = "_adx"; - $bias = 128; - } -$code.=<<___; -.globl ecp_nistz256_point_add_affine$sfx -.type ecp_nistz256_point_add_affine$sfx,\@function,3 -.align 32 -ecp_nistz256_point_add_affine$sfx: -.cfi_startproc - _CET_ENDBR - push %rbp -.cfi_push %rbp - push %rbx -.cfi_push %rbx - push %r12 -.cfi_push %r12 - push %r13 -.cfi_push %r13 - push %r14 -.cfi_push %r14 - push %r15 -.cfi_push %r15 - sub \$32*15+8, %rsp -.cfi_adjust_cfa_offset 32*15+8 -.Ladd_affine${x}_body: - - movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr - mov $b_org, $b_ptr # reassign - movdqu 0x10($a_ptr), %xmm1 - movdqu 0x20($a_ptr), %xmm2 - movdqu 0x30($a_ptr), %xmm3 - movdqu 0x40($a_ptr), %xmm4 - movdqu 0x50($a_ptr), %xmm5 - mov 0x40+8*0($a_ptr), $src0 # load original in1_z - mov 0x40+8*1($a_ptr), $acc6 - mov 0x40+8*2($a_ptr), $acc7 - mov 0x40+8*3($a_ptr), $acc0 - movdqa %xmm0, $in1_x(%rsp) - movdqa %xmm1, $in1_x+0x10(%rsp) - movdqa %xmm2, $in1_y(%rsp) - movdqa %xmm3, $in1_y+0x10(%rsp) - movdqa %xmm4, $in1_z(%rsp) - movdqa %xmm5, $in1_z+0x10(%rsp) - por %xmm4, %xmm5 - - movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr - pshufd \$0xb1, %xmm5, %xmm3 - movdqu 0x10($b_ptr), %xmm1 - movdqu 0x20($b_ptr), %xmm2 - por %xmm3, %xmm5 - movdqu 0x30($b_ptr), %xmm3 - movdqa %xmm0, $in2_x(%rsp) - pshufd \$0x1e, %xmm5, %xmm4 - movdqa %xmm1, $in2_x+0x10(%rsp) - por %xmm0, %xmm1 - movq $r_ptr, %xmm0 # save $r_ptr - movdqa %xmm2, $in2_y(%rsp) - movdqa %xmm3, $in2_y+0x10(%rsp) - por %xmm2, %xmm3 - por %xmm4, %xmm5 - pxor %xmm4, %xmm4 - por %xmm1, %xmm3 - - lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid - lea $Z1sqr(%rsp), $r_ptr # Z1^2 - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z); - - pcmpeqd %xmm4, %xmm5 - pshufd \$0xb1, %xmm3, %xmm4 - mov 0x00($b_ptr), $src0 # $b_ptr is still valid - #lea 0x00($b_ptr), $b_ptr - mov $acc4, $acc1 # harmonize sqr output and mul input - por %xmm3, %xmm4 - pshufd \$0, %xmm5, %xmm5 # in1infty - pshufd \$0x1e, %xmm4, %xmm3 - mov $acc5, $acc2 - por %xmm3, %xmm4 - pxor %xmm3, %xmm3 - mov $acc6, $acc3 - pcmpeqd %xmm3, %xmm4 - pshufd \$0, %xmm4, %xmm4 # in2infty - - lea $Z1sqr-$bias(%rsp), $a_ptr - mov $acc7, $acc4 - lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x); - - lea $in1_x(%rsp), $b_ptr - lea $H(%rsp), $r_ptr # H = U2 - U1 - call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x); - - `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")` - lea $S2(%rsp), $r_ptr # S2 = Z1^3 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z); - - `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")` - lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z); - - `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")` - lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y); - - lea $in1_y(%rsp), $b_ptr - lea $R(%rsp), $r_ptr # R = S2 - S1 - call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y); - - `&load_for_sqr("$H(%rsp)", "$src0")` - lea $Hsqr(%rsp), $r_ptr # H^2 - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H); - - `&load_for_sqr("$R(%rsp)", "$src0")` - lea $Rsqr(%rsp), $r_ptr # R^2 - call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R); - - `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")` - lea $Hcub(%rsp), $r_ptr # H^3 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H); - - `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")` - lea $U2(%rsp), $r_ptr # U1*H^2 - call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr); -___ -{ -####################################################################### -# operate in 4-5-0-1 "name space" that matches multiplication output -# -my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); -my ($poly1, $poly3)=($acc6,$acc7); - -$code.=<<___; - #lea $U2(%rsp), $a_ptr - #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2 - #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2); - - xor $t4, $t4 - add $acc0, $acc0 # a0:a3+a0:a3 - lea $Rsqr(%rsp), $a_ptr - adc $acc1, $acc1 - mov $acc0, $t0 - adc $acc2, $acc2 - adc $acc3, $acc3 - mov $acc1, $t1 - adc \$0, $t4 - - sub \$-1, $acc0 - mov $acc2, $t2 - sbb $poly1, $acc1 - sbb \$0, $acc2 - mov $acc3, $t3 - sbb $poly3, $acc3 - sbb \$0, $t4 - - cmovc $t0, $acc0 - mov 8*0($a_ptr), $t0 - cmovc $t1, $acc1 - mov 8*1($a_ptr), $t1 - cmovc $t2, $acc2 - mov 8*2($a_ptr), $t2 - cmovc $t3, $acc3 - mov 8*3($a_ptr), $t3 - - call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr); - - lea $Hcub(%rsp), $b_ptr - lea $res_x(%rsp), $r_ptr - call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub); - - mov $U2+8*0(%rsp), $t0 - mov $U2+8*1(%rsp), $t1 - mov $U2+8*2(%rsp), $t2 - mov $U2+8*3(%rsp), $t3 - lea $H(%rsp), $r_ptr - - call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x); - - mov $acc0, 8*0($r_ptr) # save the result, as - mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't - mov $acc2, 8*2($r_ptr) - mov $acc3, 8*3($r_ptr) -___ -} -$code.=<<___; - `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")` - lea $S2(%rsp), $r_ptr - call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y); - - `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")` - lea $H(%rsp), $r_ptr - call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R); - - lea $S2(%rsp), $b_ptr - lea $res_y(%rsp), $r_ptr - call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2); - - movq %xmm0, $r_ptr # restore $r_ptr - - movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty); - movdqa %xmm5, %xmm1 - pandn $res_z(%rsp), %xmm0 - movdqa %xmm5, %xmm2 - pandn $res_z+0x10(%rsp), %xmm1 - movdqa %xmm5, %xmm3 - pand .LONE_mont(%rip), %xmm2 - pand .LONE_mont+0x10(%rip), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - - movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty); - movdqa %xmm4, %xmm1 - pandn %xmm2, %xmm0 - movdqa %xmm4, %xmm2 - pandn %xmm3, %xmm1 - movdqa %xmm4, %xmm3 - pand $in1_z(%rsp), %xmm2 - pand $in1_z+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - movdqu %xmm2, 0x40($r_ptr) - movdqu %xmm3, 0x50($r_ptr) - - movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty); - movdqa %xmm5, %xmm1 - pandn $res_x(%rsp), %xmm0 - movdqa %xmm5, %xmm2 - pandn $res_x+0x10(%rsp), %xmm1 - movdqa %xmm5, %xmm3 - pand $in2_x(%rsp), %xmm2 - pand $in2_x+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - - movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty); - movdqa %xmm4, %xmm1 - pandn %xmm2, %xmm0 - movdqa %xmm4, %xmm2 - pandn %xmm3, %xmm1 - movdqa %xmm4, %xmm3 - pand $in1_x(%rsp), %xmm2 - pand $in1_x+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - movdqu %xmm2, 0x00($r_ptr) - movdqu %xmm3, 0x10($r_ptr) - - movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty); - movdqa %xmm5, %xmm1 - pandn $res_y(%rsp), %xmm0 - movdqa %xmm5, %xmm2 - pandn $res_y+0x10(%rsp), %xmm1 - movdqa %xmm5, %xmm3 - pand $in2_y(%rsp), %xmm2 - pand $in2_y+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - - movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty); - movdqa %xmm4, %xmm1 - pandn %xmm2, %xmm0 - movdqa %xmm4, %xmm2 - pandn %xmm3, %xmm1 - movdqa %xmm4, %xmm3 - pand $in1_y(%rsp), %xmm2 - pand $in1_y+0x10(%rsp), %xmm3 - por %xmm0, %xmm2 - por %xmm1, %xmm3 - movdqu %xmm2, 0x20($r_ptr) - movdqu %xmm3, 0x30($r_ptr) - - lea 32*15+56(%rsp), %rsi -.cfi_def_cfa %rsi,8 - mov -48(%rsi),%r15 -.cfi_restore %r15 - mov -40(%rsi),%r14 -.cfi_restore %r14 - mov -32(%rsi),%r13 -.cfi_restore %r13 - mov -24(%rsi),%r12 -.cfi_restore %r12 - mov -16(%rsi),%rbx -.cfi_restore %rbx - mov -8(%rsi),%rbp -.cfi_restore %rbp - lea (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Ladd_affine${x}_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx -___ -} -&gen_add_affine("q"); - -######################################################################## -# AD*X magic -# -if ($addx) { { -######################################################################## -# operate in 4-5-0-1 "name space" that matches multiplication output -# -my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3); - -$code.=<<___; -.type __ecp_nistz256_add_tox,\@abi-omnipotent -.align 32 -__ecp_nistz256_add_tox: -.cfi_startproc - xor $t4, $t4 - adc 8*0($b_ptr), $a0 - adc 8*1($b_ptr), $a1 - mov $a0, $t0 - adc 8*2($b_ptr), $a2 - adc 8*3($b_ptr), $a3 - mov $a1, $t1 - adc \$0, $t4 - - xor $t3, $t3 - sbb \$-1, $a0 - mov $a2, $t2 - sbb $poly1, $a1 - sbb \$0, $a2 - mov $a3, $t3 - sbb $poly3, $a3 - sbb \$0, $t4 - - cmovc $t0, $a0 - cmovc $t1, $a1 - mov $a0, 8*0($r_ptr) - cmovc $t2, $a2 - mov $a1, 8*1($r_ptr) - cmovc $t3, $a3 - mov $a2, 8*2($r_ptr) - mov $a3, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox - -.type __ecp_nistz256_sub_fromx,\@abi-omnipotent -.align 32 -__ecp_nistz256_sub_fromx: -.cfi_startproc - xor $t4, $t4 - sbb 8*0($b_ptr), $a0 - sbb 8*1($b_ptr), $a1 - mov $a0, $t0 - sbb 8*2($b_ptr), $a2 - sbb 8*3($b_ptr), $a3 - mov $a1, $t1 - sbb \$0, $t4 - - xor $t3, $t3 - adc \$-1, $a0 - mov $a2, $t2 - adc $poly1, $a1 - adc \$0, $a2 - mov $a3, $t3 - adc $poly3, $a3 - - bt \$0, $t4 - cmovnc $t0, $a0 - cmovnc $t1, $a1 - mov $a0, 8*0($r_ptr) - cmovnc $t2, $a2 - mov $a1, 8*1($r_ptr) - cmovnc $t3, $a3 - mov $a2, 8*2($r_ptr) - mov $a3, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx - -.type __ecp_nistz256_subx,\@abi-omnipotent -.align 32 -__ecp_nistz256_subx: -.cfi_startproc - xor $t4, $t4 - sbb $a0, $t0 - sbb $a1, $t1 - mov $t0, $a0 - sbb $a2, $t2 - sbb $a3, $t3 - mov $t1, $a1 - sbb \$0, $t4 - - xor $a3 ,$a3 - adc \$-1, $t0 - mov $t2, $a2 - adc $poly1, $t1 - adc \$0, $t2 - mov $t3, $a3 - adc $poly3, $t3 - - bt \$0, $t4 - cmovc $t0, $a0 - cmovc $t1, $a1 - cmovc $t2, $a2 - cmovc $t3, $a3 - - ret -.cfi_endproc -.size __ecp_nistz256_subx,.-__ecp_nistz256_subx - -.type __ecp_nistz256_mul_by_2x,\@abi-omnipotent -.align 32 -__ecp_nistz256_mul_by_2x: -.cfi_startproc - xor $t4, $t4 - adc $a0, $a0 # a0:a3+a0:a3 - adc $a1, $a1 - mov $a0, $t0 - adc $a2, $a2 - adc $a3, $a3 - mov $a1, $t1 - adc \$0, $t4 - - xor $t3, $t3 - sbb \$-1, $a0 - mov $a2, $t2 - sbb $poly1, $a1 - sbb \$0, $a2 - mov $a3, $t3 - sbb $poly3, $a3 - sbb \$0, $t4 - - cmovc $t0, $a0 - cmovc $t1, $a1 - mov $a0, 8*0($r_ptr) - cmovc $t2, $a2 - mov $a1, 8*1($r_ptr) - cmovc $t3, $a3 - mov $a2, 8*2($r_ptr) - mov $a3, 8*3($r_ptr) - - ret -.cfi_endproc -.size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x -___ - } -&gen_double("x"); -&gen_add("x"); -&gen_add_affine("x"); -} -}}} # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, # CONTEXT *context,DISPATCHER_CONTEXT *disp) @@ -3942,10 +1279,6 @@ .section .pdata .align 4 - .rva .LSEH_begin_ecp_nistz256_neg - .rva .LSEH_end_ecp_nistz256_neg - .rva .LSEH_info_ecp_nistz256_neg - .rva .LSEH_begin_ecp_nistz256_ord_mul_mont_nohw .rva .LSEH_end_ecp_nistz256_ord_mul_mont_nohw .rva .LSEH_info_ecp_nistz256_ord_mul_mont_nohw @@ -3964,75 +1297,9 @@ .rva .LSEH_info_ecp_nistz256_ord_sqr_mont_adx ___ $code.=<<___; - .rva .LSEH_begin_ecp_nistz256_mul_mont_nohw - .rva .LSEH_end_ecp_nistz256_mul_mont_nohw - .rva .LSEH_info_ecp_nistz256_mul_mont_nohw - - .rva .LSEH_begin_ecp_nistz256_sqr_mont_nohw - .rva .LSEH_end_ecp_nistz256_sqr_mont_nohw - .rva .LSEH_info_ecp_nistz256_sqr_mont_nohw -___ -$code.=<<___ if ($addx); - .rva .LSEH_begin_ecp_nistz256_mul_mont_adx - .rva .LSEH_end_ecp_nistz256_mul_mont_adx - .rva .LSEH_info_ecp_nistz256_mul_mont_adx - - .rva .LSEH_begin_ecp_nistz256_sqr_mont_adx - .rva .LSEH_end_ecp_nistz256_sqr_mont_adx - .rva .LSEH_info_ecp_nistz256_sqr_mont_adx -___ -$code.=<<___; - .rva .LSEH_begin_ecp_nistz256_select_w5_nohw - .rva .LSEH_end_ecp_nistz256_select_w5_nohw - .rva .LSEH_info_ecp_nistz256_select_wX_nohw - - .rva .LSEH_begin_ecp_nistz256_select_w7_nohw - .rva .LSEH_end_ecp_nistz256_select_w7_nohw - .rva .LSEH_info_ecp_nistz256_select_wX_nohw -___ -$code.=<<___ if ($avx>1); - .rva .LSEH_begin_ecp_nistz256_select_w5_avx2 - .rva .LSEH_end_ecp_nistz256_select_w5_avx2 - .rva .LSEH_info_ecp_nistz256_select_wX_avx2 - - .rva .LSEH_begin_ecp_nistz256_select_w7_avx2 - .rva .LSEH_end_ecp_nistz256_select_w7_avx2 - .rva .LSEH_info_ecp_nistz256_select_wX_avx2 -___ -$code.=<<___; - .rva .LSEH_begin_ecp_nistz256_point_double_nohw - .rva .LSEH_end_ecp_nistz256_point_double_nohw - .rva .LSEH_info_ecp_nistz256_point_double_nohw - - .rva .LSEH_begin_ecp_nistz256_point_add_nohw - .rva .LSEH_end_ecp_nistz256_point_add_nohw - .rva .LSEH_info_ecp_nistz256_point_add_nohw - - .rva .LSEH_begin_ecp_nistz256_point_add_affine_nohw - .rva .LSEH_end_ecp_nistz256_point_add_affine_nohw - .rva .LSEH_info_ecp_nistz256_point_add_affine_nohw -___ -$code.=<<___ if ($addx); - .rva .LSEH_begin_ecp_nistz256_point_double_adx - .rva .LSEH_end_ecp_nistz256_point_double_adx - .rva .LSEH_info_ecp_nistz256_point_double_adx - - .rva .LSEH_begin_ecp_nistz256_point_add_adx - .rva .LSEH_end_ecp_nistz256_point_add_adx - .rva .LSEH_info_ecp_nistz256_point_add_adx - - .rva .LSEH_begin_ecp_nistz256_point_add_affine_adx - .rva .LSEH_end_ecp_nistz256_point_add_affine_adx - .rva .LSEH_info_ecp_nistz256_point_add_affine_adx -___ -$code.=<<___; .section .xdata .align 8 -.LSEH_info_ecp_nistz256_neg: - .byte 9,0,0,0 - .rva short_handler - .rva .Lneg_body,.Lneg_epilogue # HandlerData[] .LSEH_info_ecp_nistz256_ord_mul_mont_nohw: .byte 9,0,0,0 .rva full_handler @@ -4056,98 +1323,7 @@ .rva .Lord_sqrx_body,.Lord_sqrx_epilogue # HandlerData[] .long 48,0 ___ -$code.=<<___; -.LSEH_info_ecp_nistz256_mul_mont_nohw: - .byte 9,0,0,0 - .rva full_handler - .rva .Lmul_body,.Lmul_epilogue # HandlerData[] - .long 48,0 -.LSEH_info_ecp_nistz256_sqr_mont_nohw: - .byte 9,0,0,0 - .rva full_handler - .rva .Lsqr_body,.Lsqr_epilogue # HandlerData[] - .long 48,0 -___ -$code.=<<___ if ($addx); -.LSEH_info_ecp_nistz256_mul_mont_adx: - .byte 9,0,0,0 - .rva full_handler - .rva .Lmulx_body,.Lmulx_epilogue # HandlerData[] - .long 48,0 -.LSEH_info_ecp_nistz256_sqr_mont_adx: - .byte 9,0,0,0 - .rva full_handler - .rva .Lsqrx_body,.Lsqrx_epilogue # HandlerData[] - .long 48,0 -___ -$code.=<<___; -.LSEH_info_ecp_nistz256_select_wX_nohw: - .byte 0x01,0x33,0x16,0x00 - .byte 0x33,0xf8,0x09,0x00 #movaps 0x90(rsp),xmm15 - .byte 0x2e,0xe8,0x08,0x00 #movaps 0x80(rsp),xmm14 - .byte 0x29,0xd8,0x07,0x00 #movaps 0x70(rsp),xmm13 - .byte 0x24,0xc8,0x06,0x00 #movaps 0x60(rsp),xmm12 - .byte 0x1f,0xb8,0x05,0x00 #movaps 0x50(rsp),xmm11 - .byte 0x1a,0xa8,0x04,0x00 #movaps 0x40(rsp),xmm10 - .byte 0x15,0x98,0x03,0x00 #movaps 0x30(rsp),xmm9 - .byte 0x10,0x88,0x02,0x00 #movaps 0x20(rsp),xmm8 - .byte 0x0c,0x78,0x01,0x00 #movaps 0x10(rsp),xmm7 - .byte 0x08,0x68,0x00,0x00 #movaps 0x00(rsp),xmm6 - .byte 0x04,0x01,0x15,0x00 #sub rsp,0xa8 - .align 8 -___ -$code.=<<___ if ($avx>1); -.LSEH_info_ecp_nistz256_select_wX_avx2: - .byte 0x01,0x36,0x17,0x0b - .byte 0x36,0xf8,0x09,0x00 # vmovaps 0x90(rsp),xmm15 - .byte 0x31,0xe8,0x08,0x00 # vmovaps 0x80(rsp),xmm14 - .byte 0x2c,0xd8,0x07,0x00 # vmovaps 0x70(rsp),xmm13 - .byte 0x27,0xc8,0x06,0x00 # vmovaps 0x60(rsp),xmm12 - .byte 0x22,0xb8,0x05,0x00 # vmovaps 0x50(rsp),xmm11 - .byte 0x1d,0xa8,0x04,0x00 # vmovaps 0x40(rsp),xmm10 - .byte 0x18,0x98,0x03,0x00 # vmovaps 0x30(rsp),xmm9 - .byte 0x13,0x88,0x02,0x00 # vmovaps 0x20(rsp),xmm8 - .byte 0x0e,0x78,0x01,0x00 # vmovaps 0x10(rsp),xmm7 - .byte 0x09,0x68,0x00,0x00 # vmovaps 0x00(rsp),xmm6 - .byte 0x04,0x01,0x15,0x00 # sub rsp,0xa8 - .byte 0x00,0xb3,0x00,0x00 # set_frame r11 - .align 8 -___ -$code.=<<___; -.LSEH_info_ecp_nistz256_point_double_nohw: - .byte 9,0,0,0 - .rva full_handler - .rva .Lpoint_doubleq_body,.Lpoint_doubleq_epilogue # HandlerData[] - .long 32*5+56,0 -.LSEH_info_ecp_nistz256_point_add_nohw: - .byte 9,0,0,0 - .rva full_handler - .rva .Lpoint_addq_body,.Lpoint_addq_epilogue # HandlerData[] - .long 32*18+56,0 -.LSEH_info_ecp_nistz256_point_add_affine_nohw: - .byte 9,0,0,0 - .rva full_handler - .rva .Ladd_affineq_body,.Ladd_affineq_epilogue # HandlerData[] - .long 32*15+56,0 -___ -$code.=<<___ if ($addx); -.align 8 -.LSEH_info_ecp_nistz256_point_double_adx: - .byte 9,0,0,0 - .rva full_handler - .rva .Lpoint_doublex_body,.Lpoint_doublex_epilogue # HandlerData[] - .long 32*5+56,0 -.LSEH_info_ecp_nistz256_point_add_adx: - .byte 9,0,0,0 - .rva full_handler - .rva .Lpoint_addx_body,.Lpoint_addx_epilogue # HandlerData[] - .long 32*18+56,0 -.LSEH_info_ecp_nistz256_point_add_affine_adx: - .byte 9,0,0,0 - .rva full_handler - .rva .Ladd_affinex_body,.Ladd_affinex_epilogue # HandlerData[] - .long 32*15+56,0 -___ + } $code =~ s/\`([^\`]*)\`/eval $1/gem;
diff --git a/crypto/fipsmodule/ec/ec.cc.inc b/crypto/fipsmodule/ec/ec.cc.inc index 5c795d9..e23d436 100644 --- a/crypto/fipsmodule/ec/ec.cc.inc +++ b/crypto/fipsmodule/ec/ec.cc.inc
@@ -99,13 +99,7 @@ ec_group_init_static_mont(&out->order, std::size(kP256Order), kP256Order, kP256OrderRR, kP256OrderN0); -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ - !defined(OPENSSL_SMALL) - out->meth = EC_GFp_nistz256_method(); -#else out->meth = EC_GFp_nistp256_method(); -#endif out->generator.group = out; OPENSSL_memcpy(out->generator.raw.X.words, kP256MontGX, sizeof(kP256MontGX)); OPENSSL_memcpy(out->generator.raw.Y.words, kP256MontGY, sizeof(kP256MontGY)); @@ -768,7 +762,7 @@ // Check the result is on the curve to defend against fault attacks or bugs. // This has negligible cost compared to the multiplication. - if (!ec_GFp_simple_is_on_curve(group, r)) { + if (!constant_time_declassify_int(ec_GFp_simple_is_on_curve(group, r))) { OPENSSL_PUT_ERROR(EC, ERR_R_INTERNAL_ERROR); return 0; }
diff --git a/crypto/fipsmodule/ec/internal.h b/crypto/fipsmodule/ec/internal.h index 2261f20..14fe006 100644 --- a/crypto/fipsmodule/ec/internal.h +++ b/crypto/fipsmodule/ec/internal.h
@@ -666,10 +666,6 @@ const EC_METHOD *EC_GFp_nistp256_method(); -// EC_GFp_nistz256_method is a GFp method using montgomery multiplication, with -// x86-64 optimized P256. See http://eprint.iacr.org/2013/816. -const EC_METHOD *EC_GFp_nistz256_method(); - // An EC_WRAPPED_SCALAR is an `EC_SCALAR` with a parallel `BIGNUM` // representation. It exists to support the `EC_KEY_get0_private_key` API. typedef struct {
diff --git a/crypto/fipsmodule/ec/p256-nistz.cc.inc b/crypto/fipsmodule/ec/p256-nistz.cc.inc deleted file mode 100644 index 0e2d282..0000000 --- a/crypto/fipsmodule/ec/p256-nistz.cc.inc +++ /dev/null
@@ -1,740 +0,0 @@ -// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. -// Copyright (c) 2014, Intel Corporation. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1) -// (1) Intel Corporation, Israel Development Center, Haifa, Israel -// (2) University of Haifa, Israel -// -// Reference: -// S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with -// 256 Bit Primes" - -#include <openssl/ec.h> - -#include <assert.h> -#include <stdint.h> -#include <string.h> - -#include <openssl/bn.h> -#include <openssl/crypto.h> -#include <openssl/err.h> - -#include "../../internal.h" -#include "../bn/internal.h" -#include "../delocate.h" -#include "internal.h" -#include "p256-nistz.h" - - -using namespace bssl; - -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ - !defined(OPENSSL_SMALL) - -typedef P256_POINT_AFFINE PRECOMP256_ROW[64]; - -// One converted into the Montgomery domain -static const BN_ULONG ONE_MONT[P256_LIMBS] = { - TOBN(0x00000000, 0x00000001), - TOBN(0xffffffff, 0x00000000), - TOBN(0xffffffff, 0xffffffff), - TOBN(0x00000000, 0xfffffffe), -}; - -// Precomputed tables for the default generator -#include "p256-nistz-table.h" - -// Recode window to a signed digit, see `ec_GFp_nistp_recode_scalar_bits` in -// util.c for details -static crypto_word_t booth_recode_w5(crypto_word_t in) { - crypto_word_t s, d; - - s = ~((in >> 5) - 1); - d = (1 << 6) - in - 1; - d = (d & s) | (in & ~s); - d = (d >> 1) + (d & 1); - - return (d << 1) + (s & 1); -} - -static crypto_word_t booth_recode_w7(crypto_word_t in) { - crypto_word_t s, d; - - s = ~((in >> 7) - 1); - d = (1 << 8) - in - 1; - d = (d & s) | (in & ~s); - d = (d >> 1) + (d & 1); - - return (d << 1) + (s & 1); -} - -// copy_conditional copies `src` to `dst` if `move` is one and leaves it as-is -// if `move` is zero. -// -// WARNING: this breaks the usual convention of constant-time functions -// returning masks. -static void copy_conditional(BN_ULONG dst[P256_LIMBS], - const BN_ULONG src[P256_LIMBS], BN_ULONG move) { - BN_ULONG mask1 = ((BN_ULONG)0) - move; - BN_ULONG mask2 = ~mask1; - - dst[0] = (src[0] & mask1) ^ (dst[0] & mask2); - dst[1] = (src[1] & mask1) ^ (dst[1] & mask2); - dst[2] = (src[2] & mask1) ^ (dst[2] & mask2); - dst[3] = (src[3] & mask1) ^ (dst[3] & mask2); - if (P256_LIMBS == 8) { - dst[4] = (src[4] & mask1) ^ (dst[4] & mask2); - dst[5] = (src[5] & mask1) ^ (dst[5] & mask2); - dst[6] = (src[6] & mask1) ^ (dst[6] & mask2); - dst[7] = (src[7] & mask1) ^ (dst[7] & mask2); - } -} - -// is_not_zero returns one iff in != 0 and zero otherwise. -// -// WARNING: this breaks the usual convention of constant-time functions -// returning masks. -// -// (define-fun is_not_zero ((in (_ BitVec 64))) (_ BitVec 64) -// (bvlshr (bvor in (bvsub #x0000000000000000 in)) #x000000000000003f) -// ) -// -// (declare-fun x () (_ BitVec 64)) -// -// (assert (and (= x #x0000000000000000) (= (is_not_zero x) -// #x0000000000000001))) (check-sat) -// -// (assert (and (not (= x #x0000000000000000)) (= (is_not_zero x) -// #x0000000000000000))) (check-sat) -// -static BN_ULONG is_not_zero(BN_ULONG in) { - in |= (0 - in); - in >>= BN_BITS2 - 1; - return in; -} - -#if defined(OPENSSL_X86_64) -// Dispatch between CPU variations. The "_adx" suffixed functions use MULX in -// addition to ADCX/ADOX. MULX is part of BMI2, not ADX, so we must check both -// capabilities. -static void ecp_nistz256_mul_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]) { - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - ecp_nistz256_mul_mont_adx(res, a, b); - } else { - ecp_nistz256_mul_mont_nohw(res, a, b); - } -} - -static void ecp_nistz256_sqr_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS]) { - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - ecp_nistz256_sqr_mont_adx(res, a); - } else { - ecp_nistz256_sqr_mont_nohw(res, a); - } -} - -static void ecp_nistz256_ord_mul_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]) { - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - ecp_nistz256_ord_mul_mont_adx(res, a, b); - } else { - ecp_nistz256_ord_mul_mont_nohw(res, a, b); - } -} - -static void ecp_nistz256_ord_sqr_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - BN_ULONG rep) { - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - ecp_nistz256_ord_sqr_mont_adx(res, a, rep); - } else { - ecp_nistz256_ord_sqr_mont_nohw(res, a, rep); - } -} - -static void ecp_nistz256_select_w5(P256_POINT *val, const P256_POINT in_t[16], - int index) { - if (CRYPTO_is_AVX2_capable()) { - ecp_nistz256_select_w5_avx2(val, in_t, index); - } else { - ecp_nistz256_select_w5_nohw(val, in_t, index); - } -} - -static void ecp_nistz256_select_w7(P256_POINT_AFFINE *val, - const P256_POINT_AFFINE in_t[64], - int index) { - if (CRYPTO_is_AVX2_capable()) { - ecp_nistz256_select_w7_avx2(val, in_t, index); - } else { - ecp_nistz256_select_w7_nohw(val, in_t, index); - } -} - -static void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a) { - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - ecp_nistz256_point_double_adx(r, a); - } else { - ecp_nistz256_point_double_nohw(r, a); - } -} - -static void ecp_nistz256_point_add(P256_POINT *r, const P256_POINT *a, - const P256_POINT *b) { - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - ecp_nistz256_point_add_adx(r, a, b); - } else { - ecp_nistz256_point_add_nohw(r, a, b); - } -} - -static void ecp_nistz256_point_add_affine(P256_POINT *r, const P256_POINT *a, - const P256_POINT_AFFINE *b) { - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - ecp_nistz256_point_add_affine_adx(r, a, b); - } else { - ecp_nistz256_point_add_affine_nohw(r, a, b); - } -} -#endif // OPENSSL_X86_64 - -// ecp_nistz256_from_mont sets `res` to `in`, converted from Montgomery domain -// by multiplying with 1. -static void ecp_nistz256_from_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG in[P256_LIMBS]) { - static const BN_ULONG ONE[P256_LIMBS] = {1}; - ecp_nistz256_mul_mont(res, in, ONE); -} - -// ecp_nistz256_mod_inverse_sqr_mont sets `r` to (`in` * 2^-256)^-2 * 2^256 mod -// p. That is, `r` is the modular inverse square of `in` for input and output in -// the Montgomery domain. -static void ecp_nistz256_mod_inverse_sqr_mont(BN_ULONG r[P256_LIMBS], - const BN_ULONG in[P256_LIMBS]) { - // This implements the addition chain described in - // https://briansmith.org/ecc-inversion-addition-chains-01#p256_field_inversion - BN_ULONG x2[P256_LIMBS], x3[P256_LIMBS], x6[P256_LIMBS], x12[P256_LIMBS], - x15[P256_LIMBS], x30[P256_LIMBS], x32[P256_LIMBS]; - ecp_nistz256_sqr_mont(x2, in); // 2^2 - 2^1 - ecp_nistz256_mul_mont(x2, x2, in); // 2^2 - 2^0 - - ecp_nistz256_sqr_mont(x3, x2); // 2^3 - 2^1 - ecp_nistz256_mul_mont(x3, x3, in); // 2^3 - 2^0 - - ecp_nistz256_sqr_mont(x6, x3); - for (int i = 1; i < 3; i++) { - ecp_nistz256_sqr_mont(x6, x6); - } // 2^6 - 2^3 - ecp_nistz256_mul_mont(x6, x6, x3); // 2^6 - 2^0 - - ecp_nistz256_sqr_mont(x12, x6); - for (int i = 1; i < 6; i++) { - ecp_nistz256_sqr_mont(x12, x12); - } // 2^12 - 2^6 - ecp_nistz256_mul_mont(x12, x12, x6); // 2^12 - 2^0 - - ecp_nistz256_sqr_mont(x15, x12); - for (int i = 1; i < 3; i++) { - ecp_nistz256_sqr_mont(x15, x15); - } // 2^15 - 2^3 - ecp_nistz256_mul_mont(x15, x15, x3); // 2^15 - 2^0 - - ecp_nistz256_sqr_mont(x30, x15); - for (int i = 1; i < 15; i++) { - ecp_nistz256_sqr_mont(x30, x30); - } // 2^30 - 2^15 - ecp_nistz256_mul_mont(x30, x30, x15); // 2^30 - 2^0 - - ecp_nistz256_sqr_mont(x32, x30); - ecp_nistz256_sqr_mont(x32, x32); // 2^32 - 2^2 - ecp_nistz256_mul_mont(x32, x32, x2); // 2^32 - 2^0 - - BN_ULONG ret[P256_LIMBS]; - ecp_nistz256_sqr_mont(ret, x32); - for (int i = 1; i < 31 + 1; i++) { - ecp_nistz256_sqr_mont(ret, ret); - } // 2^64 - 2^32 - ecp_nistz256_mul_mont(ret, ret, in); // 2^64 - 2^32 + 2^0 - - for (int i = 0; i < 96 + 32; i++) { - ecp_nistz256_sqr_mont(ret, ret); - } // 2^192 - 2^160 + 2^128 - ecp_nistz256_mul_mont(ret, ret, x32); // 2^192 - 2^160 + 2^128 + 2^32 - 2^0 - - for (int i = 0; i < 32; i++) { - ecp_nistz256_sqr_mont(ret, ret); - } // 2^224 - 2^192 + 2^160 + 2^64 - 2^32 - ecp_nistz256_mul_mont(ret, ret, x32); // 2^224 - 2^192 + 2^160 + 2^64 - 2^0 - - for (int i = 0; i < 30; i++) { - ecp_nistz256_sqr_mont(ret, ret); - } // 2^254 - 2^222 + 2^190 + 2^94 - 2^30 - ecp_nistz256_mul_mont(ret, ret, x30); // 2^254 - 2^222 + 2^190 + 2^94 - 2^0 - - ecp_nistz256_sqr_mont(ret, ret); - ecp_nistz256_sqr_mont(r, ret); // 2^256 - 2^224 + 2^192 + 2^96 - 2^2 -} - -// r = p * p_scalar -static void ecp_nistz256_windowed_mul(const EC_GROUP *group, P256_POINT *r, - const EC_JACOBIAN *p, - const EC_SCALAR *p_scalar) { - assert(p != nullptr); - assert(p_scalar != nullptr); - assert(group->field.N.width == P256_LIMBS); - - static const size_t kWindowSize = 5; - static const crypto_word_t kMask = (1 << (5 /* kWindowSize */ + 1)) - 1; - - // A `P256_POINT` is (3 * 32) = 96 bytes, and the 64-byte alignment should - // add no more than 63 bytes of overhead. Thus, `table` should require - // ~1599 ((96 * 16) + 63) bytes of stack space. - alignas(64) P256_POINT table[16]; - uint8_t p_str[33]; - OPENSSL_memcpy(p_str, p_scalar->words, 32); - p_str[32] = 0; - - // table[0] is implicitly (0,0,0) (the point at infinity), therefore it is - // not stored. All other values are actually stored with an offset of -1 in - // table. - P256_POINT *row = table; - assert(group->field.N.width == P256_LIMBS); - OPENSSL_memcpy(row[1 - 1].X, p->X.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(row[1 - 1].Y, p->Y.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(row[1 - 1].Z, p->Z.words, P256_LIMBS * sizeof(BN_ULONG)); - - ecp_nistz256_point_double(&row[2 - 1], &row[1 - 1]); - ecp_nistz256_point_add(&row[3 - 1], &row[2 - 1], &row[1 - 1]); - ecp_nistz256_point_double(&row[4 - 1], &row[2 - 1]); - ecp_nistz256_point_double(&row[6 - 1], &row[3 - 1]); - ecp_nistz256_point_double(&row[8 - 1], &row[4 - 1]); - ecp_nistz256_point_double(&row[12 - 1], &row[6 - 1]); - ecp_nistz256_point_add(&row[5 - 1], &row[4 - 1], &row[1 - 1]); - ecp_nistz256_point_add(&row[7 - 1], &row[6 - 1], &row[1 - 1]); - ecp_nistz256_point_add(&row[9 - 1], &row[8 - 1], &row[1 - 1]); - ecp_nistz256_point_add(&row[13 - 1], &row[12 - 1], &row[1 - 1]); - ecp_nistz256_point_double(&row[14 - 1], &row[7 - 1]); - ecp_nistz256_point_double(&row[10 - 1], &row[5 - 1]); - ecp_nistz256_point_add(&row[15 - 1], &row[14 - 1], &row[1 - 1]); - ecp_nistz256_point_add(&row[11 - 1], &row[10 - 1], &row[1 - 1]); - ecp_nistz256_point_double(&row[16 - 1], &row[8 - 1]); - - BN_ULONG tmp[P256_LIMBS]; - alignas(32) P256_POINT h; - size_t index = 255; - crypto_word_t wvalue = p_str[(index - 1) / 8]; - wvalue = (wvalue >> ((index - 1) % 8)) & kMask; - - ecp_nistz256_select_w5(r, table, booth_recode_w5(wvalue) >> 1); - - while (index >= 5) { - if (index != 255) { - size_t off = (index - 1) / 8; - - wvalue = (crypto_word_t)p_str[off] | (crypto_word_t)p_str[off + 1] << 8; - wvalue = (wvalue >> ((index - 1) % 8)) & kMask; - - wvalue = booth_recode_w5(wvalue); - - ecp_nistz256_select_w5(&h, table, wvalue >> 1); - - ecp_nistz256_neg(tmp, h.Y); - copy_conditional(h.Y, tmp, (wvalue & 1)); - - ecp_nistz256_point_add(r, r, &h); - } - - index -= kWindowSize; - - ecp_nistz256_point_double(r, r); - ecp_nistz256_point_double(r, r); - ecp_nistz256_point_double(r, r); - ecp_nistz256_point_double(r, r); - ecp_nistz256_point_double(r, r); - } - - // Final window - wvalue = p_str[0]; - wvalue = (wvalue << 1) & kMask; - - wvalue = booth_recode_w5(wvalue); - - ecp_nistz256_select_w5(&h, table, wvalue >> 1); - - ecp_nistz256_neg(tmp, h.Y); - copy_conditional(h.Y, tmp, wvalue & 1); - - ecp_nistz256_point_add(r, r, &h); -} - -static crypto_word_t calc_first_wvalue(size_t *index, const uint8_t p_str[33]) { - static const size_t kWindowSize = 7; - static const crypto_word_t kMask = (1 << (7 /* kWindowSize */ + 1)) - 1; - *index = kWindowSize; - - crypto_word_t wvalue = (p_str[0] << 1) & kMask; - return booth_recode_w7(wvalue); -} - -static crypto_word_t calc_wvalue(size_t *index, const uint8_t p_str[33]) { - static const size_t kWindowSize = 7; - static const crypto_word_t kMask = (1 << (7 /* kWindowSize */ + 1)) - 1; - - const size_t off = (*index - 1) / 8; - crypto_word_t wvalue = - (crypto_word_t)p_str[off] | (crypto_word_t)p_str[off + 1] << 8; - wvalue = (wvalue >> ((*index - 1) % 8)) & kMask; - *index += kWindowSize; - - return booth_recode_w7(wvalue); -} - -static void ecp_nistz256_point_mul(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *p, - const EC_SCALAR *scalar) { - alignas(32) P256_POINT out; - ecp_nistz256_windowed_mul(group, &out, p, scalar); - - assert(group->field.N.width == P256_LIMBS); - OPENSSL_memcpy(r->X.words, out.X, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Y.words, out.Y, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Z.words, out.Z, P256_LIMBS * sizeof(BN_ULONG)); -} - -static void ecp_nistz256_point_mul_base(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_SCALAR *scalar) { - uint8_t p_str[33]; - OPENSSL_memcpy(p_str, scalar->words, 32); - p_str[32] = 0; - - // First window - size_t index = 0; - crypto_word_t wvalue = calc_first_wvalue(&index, p_str); - - alignas(32) P256_POINT_AFFINE t; - alignas(32) P256_POINT p; - ecp_nistz256_select_w7(&t, ecp_nistz256_precomputed[0], wvalue >> 1); - ecp_nistz256_neg(p.Z, t.Y); - copy_conditional(t.Y, p.Z, wvalue & 1); - - // Convert `t` from affine to Jacobian coordinates. We set Z to zero if `t` - // is infinity and `ONE_MONT` otherwise. `t` was computed from the table, so - // it is infinity iff `wvalue >> 1` is zero. - OPENSSL_memcpy(p.X, t.X, sizeof(p.X)); - OPENSSL_memcpy(p.Y, t.Y, sizeof(p.Y)); - OPENSSL_memset(p.Z, 0, sizeof(p.Z)); - copy_conditional(p.Z, ONE_MONT, is_not_zero(wvalue >> 1)); - - for (int i = 1; i < 37; i++) { - wvalue = calc_wvalue(&index, p_str); - - ecp_nistz256_select_w7(&t, ecp_nistz256_precomputed[i], wvalue >> 1); - - alignas(32) BN_ULONG neg_Y[P256_LIMBS]; - ecp_nistz256_neg(neg_Y, t.Y); - copy_conditional(t.Y, neg_Y, wvalue & 1); - - // Note `ecp_nistz256_point_add_affine` does not work if `p` and `t` are the - // same non-infinity point. - ecp_nistz256_point_add_affine(&p, &p, &t); - } - - assert(group->field.N.width == P256_LIMBS); - OPENSSL_memcpy(r->X.words, p.X, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Y.words, p.Y, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Z.words, p.Z, P256_LIMBS * sizeof(BN_ULONG)); -} - -static void ecp_nistz256_points_mul_public(const EC_GROUP *group, - EC_JACOBIAN *r, - const EC_SCALAR *g_scalar, - const EC_JACOBIAN *p_, - const EC_SCALAR *p_scalar) { - assert(p_ != nullptr && p_scalar != nullptr && g_scalar != nullptr); - - alignas(32) P256_POINT p; - uint8_t p_str[33]; - OPENSSL_memcpy(p_str, g_scalar->words, 32); - p_str[32] = 0; - - // First window - size_t index = 0; - size_t wvalue = calc_first_wvalue(&index, p_str); - - // Convert `p` from affine to Jacobian coordinates. We set Z to zero if `p` - // is infinity and `ONE_MONT` otherwise. `p` was computed from the table, so - // it is infinity iff `wvalue >> 1` is zero. - if ((wvalue >> 1) != 0) { - OPENSSL_memcpy(p.X, &ecp_nistz256_precomputed[0][(wvalue >> 1) - 1].X, - sizeof(p.X)); - OPENSSL_memcpy(p.Y, &ecp_nistz256_precomputed[0][(wvalue >> 1) - 1].Y, - sizeof(p.Y)); - OPENSSL_memcpy(p.Z, ONE_MONT, sizeof(p.Z)); - } else { - OPENSSL_memset(p.X, 0, sizeof(p.X)); - OPENSSL_memset(p.Y, 0, sizeof(p.Y)); - OPENSSL_memset(p.Z, 0, sizeof(p.Z)); - } - - if ((wvalue & 1) == 1) { - ecp_nistz256_neg(p.Y, p.Y); - } - - for (int i = 1; i < 37; i++) { - wvalue = calc_wvalue(&index, p_str); - if ((wvalue >> 1) == 0) { - continue; - } - - alignas(32) P256_POINT_AFFINE t; - OPENSSL_memcpy(&t, &ecp_nistz256_precomputed[i][(wvalue >> 1) - 1], - sizeof(t)); - if ((wvalue & 1) == 1) { - ecp_nistz256_neg(t.Y, t.Y); - } - - // Note `ecp_nistz256_point_add_affine` does not work if `p` and `t` are - // the same non-infinity point, so it is important that we compute the - // `g_scalar` term before the `p_scalar` term. - ecp_nistz256_point_add_affine(&p, &p, &t); - } - - alignas(32) P256_POINT tmp; - ecp_nistz256_windowed_mul(group, &tmp, p_, p_scalar); - ecp_nistz256_point_add(&p, &p, &tmp); - - assert(group->field.N.width == P256_LIMBS); - OPENSSL_memcpy(r->X.words, p.X, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Y.words, p.Y, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Z.words, p.Z, P256_LIMBS * sizeof(BN_ULONG)); -} - -static int ecp_nistz256_get_affine(const EC_GROUP *group, - const EC_JACOBIAN *point, EC_FELEM *x, - EC_FELEM *y) { - if (constant_time_declassify_int( - ec_GFp_simple_is_at_infinity(group, point))) { - OPENSSL_PUT_ERROR(EC, EC_R_POINT_AT_INFINITY); - return 0; - } - - BN_ULONG z_inv2[P256_LIMBS]; - assert(group->field.N.width == P256_LIMBS); - ecp_nistz256_mod_inverse_sqr_mont(z_inv2, point->Z.words); - - if (x != nullptr) { - ecp_nistz256_mul_mont(x->words, z_inv2, point->X.words); - } - - if (y != nullptr) { - ecp_nistz256_sqr_mont(z_inv2, z_inv2); // z^-4 - ecp_nistz256_mul_mont(y->words, point->Y.words, point->Z.words); // y * z - ecp_nistz256_mul_mont(y->words, y->words, z_inv2); // y * z^-3 - } - - return 1; -} - -static void ecp_nistz256_add(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *a_, const EC_JACOBIAN *b_) { - P256_POINT a, b; - OPENSSL_memcpy(a.X, a_->X.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(a.Y, a_->Y.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(a.Z, a_->Z.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(b.X, b_->X.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(b.Y, b_->Y.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(b.Z, b_->Z.words, P256_LIMBS * sizeof(BN_ULONG)); - ecp_nistz256_point_add(&a, &a, &b); - OPENSSL_memcpy(r->X.words, a.X, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Y.words, a.Y, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Z.words, a.Z, P256_LIMBS * sizeof(BN_ULONG)); -} - -static void ecp_nistz256_dbl(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *a_) { - P256_POINT a; - OPENSSL_memcpy(a.X, a_->X.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(a.Y, a_->Y.words, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(a.Z, a_->Z.words, P256_LIMBS * sizeof(BN_ULONG)); - ecp_nistz256_point_double(&a, &a); - OPENSSL_memcpy(r->X.words, a.X, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Y.words, a.Y, P256_LIMBS * sizeof(BN_ULONG)); - OPENSSL_memcpy(r->Z.words, a.Z, P256_LIMBS * sizeof(BN_ULONG)); -} - -static void ecp_nistz256_inv0_mod_ord(const EC_GROUP *group, EC_SCALAR *out, - const EC_SCALAR *in) { - // table[i] stores a power of `in` corresponding to the matching enum value. - enum { - // The following indices specify the power in binary. - i_1 = 0, - i_10, - i_11, - i_101, - i_111, - i_1010, - i_1111, - i_10101, - i_101010, - i_101111, - // The following indices specify 2^N-1, or N ones in a row. - i_x6, - i_x8, - i_x16, - i_x32 - }; - BN_ULONG table[15][P256_LIMBS]; - - // https://briansmith.org/ecc-inversion-addition-chains-01#p256_scalar_inversion - // - // Even though this code path spares 12 squarings, 4.5%, and 13 - // multiplications, 25%, the overall sign operation is not that much faster, - // not more that 2%. Most of the performance of this function comes from the - // scalar operations. - - // Pre-calculate powers. - OPENSSL_memcpy(table[i_1], in->words, P256_LIMBS * sizeof(BN_ULONG)); - - ecp_nistz256_ord_sqr_mont(table[i_10], table[i_1], 1); - - ecp_nistz256_ord_mul_mont(table[i_11], table[i_1], table[i_10]); - - ecp_nistz256_ord_mul_mont(table[i_101], table[i_11], table[i_10]); - - ecp_nistz256_ord_mul_mont(table[i_111], table[i_101], table[i_10]); - - ecp_nistz256_ord_sqr_mont(table[i_1010], table[i_101], 1); - - ecp_nistz256_ord_mul_mont(table[i_1111], table[i_1010], table[i_101]); - - ecp_nistz256_ord_sqr_mont(table[i_10101], table[i_1010], 1); - ecp_nistz256_ord_mul_mont(table[i_10101], table[i_10101], table[i_1]); - - ecp_nistz256_ord_sqr_mont(table[i_101010], table[i_10101], 1); - - ecp_nistz256_ord_mul_mont(table[i_101111], table[i_101010], table[i_101]); - - ecp_nistz256_ord_mul_mont(table[i_x6], table[i_101010], table[i_10101]); - - ecp_nistz256_ord_sqr_mont(table[i_x8], table[i_x6], 2); - ecp_nistz256_ord_mul_mont(table[i_x8], table[i_x8], table[i_11]); - - ecp_nistz256_ord_sqr_mont(table[i_x16], table[i_x8], 8); - ecp_nistz256_ord_mul_mont(table[i_x16], table[i_x16], table[i_x8]); - - ecp_nistz256_ord_sqr_mont(table[i_x32], table[i_x16], 16); - ecp_nistz256_ord_mul_mont(table[i_x32], table[i_x32], table[i_x16]); - - // Compute `in` raised to the order-2. - ecp_nistz256_ord_sqr_mont(out->words, table[i_x32], 64); - ecp_nistz256_ord_mul_mont(out->words, out->words, table[i_x32]); - static const struct { - uint8_t p, i; - } kChain[27] = {{32, i_x32}, {6, i_101111}, {5, i_111}, {4, i_11}, - {5, i_1111}, {5, i_10101}, {4, i_101}, {3, i_101}, - {3, i_101}, {5, i_111}, {9, i_101111}, {6, i_1111}, - {2, i_1}, {5, i_1}, {6, i_1111}, {5, i_111}, - {4, i_111}, {5, i_111}, {5, i_101}, {3, i_11}, - {10, i_101111}, {2, i_11}, {5, i_11}, {5, i_11}, - {3, i_1}, {7, i_10101}, {6, i_1111}}; - for (const auto &step : kChain) { - ecp_nistz256_ord_sqr_mont(out->words, out->words, step.p); - ecp_nistz256_ord_mul_mont(out->words, out->words, table[step.i]); - } -} - -static int ecp_nistz256_scalar_to_montgomery_inv_vartime(const EC_GROUP *group, - EC_SCALAR *out, - const EC_SCALAR *in) { -#if defined(OPENSSL_X86_64) - if (!CRYPTO_is_AVX_capable()) { - // No AVX support; fallback to generic code. - return ec_simple_scalar_to_montgomery_inv_vartime(group, out, in); - } -#endif - - assert(group->order.N.width == P256_LIMBS); - if (!beeu_mod_inverse_vartime(out->words, in->words, group->order.N.d)) { - return 0; - } - - // The result should be returned in the Montgomery domain. - ec_scalar_to_montgomery(group, out, out); - return 1; -} - -static int ecp_nistz256_cmp_x_coordinate(const EC_GROUP *group, - const EC_JACOBIAN *p, - const EC_SCALAR *r) { - if (ec_GFp_simple_is_at_infinity(group, p)) { - return 0; - } - - assert(group->order.N.width == P256_LIMBS); - assert(group->field.N.width == P256_LIMBS); - - // We wish to compare X/Z^2 with r. This is equivalent to comparing X with - // r*Z^2. Note that X and Z are represented in Montgomery form, while r is - // not. - BN_ULONG r_Z2[P256_LIMBS], Z2_mont[P256_LIMBS], X[P256_LIMBS]; - ecp_nistz256_mul_mont(Z2_mont, p->Z.words, p->Z.words); - ecp_nistz256_mul_mont(r_Z2, r->words, Z2_mont); - ecp_nistz256_from_mont(X, p->X.words); - - if (OPENSSL_memcmp(r_Z2, X, sizeof(r_Z2)) == 0) { - return 1; - } - - // During signing the x coefficient is reduced modulo the group order. - // Therefore there is a small possibility, less than 1/2^128, that group_order - // < p.x < P. in that case we need not only to compare against `r` but also to - // compare against r+group_order. - BN_ULONG carry = bn_add_words(r_Z2, r->words, group->order.N.d, P256_LIMBS); - if (carry == 0 && bn_less_than_words(r_Z2, group->field.N.d, P256_LIMBS)) { - // r + group_order < p, so compare (r + group_order) * Z^2 against X. - ecp_nistz256_mul_mont(r_Z2, r_Z2, Z2_mont); - if (OPENSSL_memcmp(r_Z2, X, sizeof(r_Z2)) == 0) { - return 1; - } - } - - return 0; -} - -BSSL_NAMESPACE_BEGIN - -DEFINE_METHOD_FUNCTION(EC_METHOD, EC_GFp_nistz256_method) { - out->point_get_affine_coordinates = ecp_nistz256_get_affine; - out->add = ecp_nistz256_add; - out->dbl = ecp_nistz256_dbl; - out->mul = ecp_nistz256_point_mul; - out->mul_base = ecp_nistz256_point_mul_base; - out->mul_public = ecp_nistz256_points_mul_public; - out->scalar_inv0_montgomery = ecp_nistz256_inv0_mod_ord; - out->scalar_to_montgomery_inv_vartime = - ecp_nistz256_scalar_to_montgomery_inv_vartime; - out->cmp_x_coordinate = ecp_nistz256_cmp_x_coordinate; -} - -BSSL_NAMESPACE_END - -#endif /* !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ - !defined(OPENSSL_SMALL) */
diff --git a/crypto/fipsmodule/ec/p256-nistz.h b/crypto/fipsmodule/ec/p256-nistz.h deleted file mode 100644 index 43da01c..0000000 --- a/crypto/fipsmodule/ec/p256-nistz.h +++ /dev/null
@@ -1,207 +0,0 @@ -// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. -// Copyright (c) 2014, Intel Corporation. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1) -// (1) Intel Corporation, Israel Development Center, Haifa, Israel -// (2) University of Haifa, Israel -// -// Reference: -// S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with -// 256 Bit Primes" - -#ifndef OPENSSL_HEADER_CRYPTO_FIPSMODULE_EC_P256_NISTZ_H -#define OPENSSL_HEADER_CRYPTO_FIPSMODULE_EC_P256_NISTZ_H - -#include <openssl/base.h> - -#include <openssl/bn.h> - -#include "../bn/internal.h" - -#if defined(__cplusplus) -extern "C" { -#endif - - -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ - !defined(OPENSSL_SMALL) - -// P-256 field operations. -// -// An element mod P in P-256 is represented as a little-endian array of -// `P256_LIMBS` `BN_ULONG`s, spanning the full range of values. -// -// The following functions take fully-reduced inputs mod P and give -// fully-reduced outputs. They may be used in-place. - -#define P256_LIMBS (256 / BN_BITS2) - -// ecp_nistz256_neg sets `res` to -`a` mod P. -void ecp_nistz256_neg(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]); - -// ecp_nistz256_mul_mont sets `res` to `a` * `b` * 2^-256 mod P. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_mul_mont_nohw(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]); -void ecp_nistz256_mul_mont_adx(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]); -#else -void ecp_nistz256_mul_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]); -#endif - -// ecp_nistz256_sqr_mont sets `res` to `a` * `a` * 2^-256 mod P. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_sqr_mont_nohw(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS]); -void ecp_nistz256_sqr_mont_adx(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS]); -#else -void ecp_nistz256_sqr_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS]); -#endif - - -// P-256 scalar operations. -// -// The following functions compute modulo N, where N is the order of P-256. They -// take fully-reduced inputs and give fully-reduced outputs. - -// ecp_nistz256_ord_mul_mont sets `res` to `a` * `b` where inputs and outputs -// are in Montgomery form. That is, `res` is `a` * `b` * 2^-256 mod N. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_ord_mul_mont_nohw(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]); -void ecp_nistz256_ord_mul_mont_adx(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]); -#else -void ecp_nistz256_ord_mul_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]); -#endif - -// ecp_nistz256_ord_sqr_mont sets `res` to `a`^(2*`rep`) where inputs and -// outputs are in Montgomery form. That is, `res` is -// (`a` * 2^-256)^(2*`rep`) * 2^256 mod N. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_ord_sqr_mont_nohw(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], BN_ULONG rep); -void ecp_nistz256_ord_sqr_mont_adx(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], BN_ULONG rep); -#else -void ecp_nistz256_ord_sqr_mont(BN_ULONG res[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], BN_ULONG rep); -#endif - -// beeu_mod_inverse_vartime sets out = a^-1 mod p using a Euclidean algorithm. -// Assumption: 0 < a < p < 2^(256) and p is odd. -int beeu_mod_inverse_vartime(BN_ULONG out[P256_LIMBS], - const BN_ULONG a[P256_LIMBS], - const BN_ULONG p[P256_LIMBS]); - - -// P-256 point operations. -// -// The following functions may be used in-place. All coordinates are in the -// Montgomery domain. - -// A P256_POINT represents a P-256 point in Jacobian coordinates. -typedef struct { - BN_ULONG X[P256_LIMBS]; - BN_ULONG Y[P256_LIMBS]; - BN_ULONG Z[P256_LIMBS]; -} P256_POINT; - -// A P256_POINT_AFFINE represents a P-256 point in affine coordinates. Infinity -// is encoded as (0, 0). -typedef struct { - BN_ULONG X[P256_LIMBS]; - BN_ULONG Y[P256_LIMBS]; -} P256_POINT_AFFINE; - -// ecp_nistz256_select_w5 sets `*val` to `in_t[index-1]` if 1 <= `index` <= 16 -// and all zeros (the point at infinity) if `index` is 0. This is done in -// constant time. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_select_w5_nohw(P256_POINT *val, const P256_POINT in_t[16], - int index); -void ecp_nistz256_select_w5_avx2(P256_POINT *val, const P256_POINT in_t[16], - int index); -#else -void ecp_nistz256_select_w5(P256_POINT *val, const P256_POINT in_t[16], - int index); -#endif - -// ecp_nistz256_select_w7 sets `*val` to `in_t[index-1]` if 1 <= `index` <= 64 -// and all zeros (the point at infinity) if `index` is 0. This is done in -// constant time. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_select_w7_nohw(P256_POINT_AFFINE *val, - const P256_POINT_AFFINE in_t[64], int index); -void ecp_nistz256_select_w7_avx2(P256_POINT_AFFINE *val, - const P256_POINT_AFFINE in_t[64], int index); -#else -void ecp_nistz256_select_w7(P256_POINT_AFFINE *val, - const P256_POINT_AFFINE in_t[64], int index); -#endif - -// ecp_nistz256_point_double sets `r` to `a` doubled. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_point_double_nohw(P256_POINT *r, const P256_POINT *a); -void ecp_nistz256_point_double_adx(P256_POINT *r, const P256_POINT *a); -#else -void ecp_nistz256_point_double(P256_POINT *r, const P256_POINT *a); -#endif - -// ecp_nistz256_point_add adds `a` to `b` and places the result in `r`. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_point_add_nohw(P256_POINT *r, const P256_POINT *a, - const P256_POINT *b); -void ecp_nistz256_point_add_adx(P256_POINT *r, const P256_POINT *a, - const P256_POINT *b); -#else -void ecp_nistz256_point_add(P256_POINT *r, const P256_POINT *a, - const P256_POINT *b); -#endif - -// ecp_nistz256_point_add_affine adds `a` to `b` and places the result in -// `r`. `a` and `b` must not represent the same point unless they are both -// infinity. -#if defined(OPENSSL_X86_64) -void ecp_nistz256_point_add_affine_adx(P256_POINT *r, const P256_POINT *a, - const P256_POINT_AFFINE *b); -void ecp_nistz256_point_add_affine_nohw(P256_POINT *r, const P256_POINT *a, - const P256_POINT_AFFINE *b); -#else -void ecp_nistz256_point_add_affine(P256_POINT *r, const P256_POINT *a, - const P256_POINT_AFFINE *b); -#endif - -#endif /* !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ - !defined(OPENSSL_SMALL) */ - - -#if defined(__cplusplus) -} // extern C -#endif - -#endif // OPENSSL_HEADER_CRYPTO_FIPSMODULE_EC_P256_NISTZ_H
diff --git a/crypto/fipsmodule/ec/p256-nistz_test.cc b/crypto/fipsmodule/ec/p256-nistz_test.cc deleted file mode 100644 index e32894e..0000000 --- a/crypto/fipsmodule/ec/p256-nistz_test.cc +++ /dev/null
@@ -1,690 +0,0 @@ -// Copyright 2016 The BoringSSL Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include <openssl/base.h> - -#include <stdio.h> -#include <string.h> - -#include <gtest/gtest.h> - -#include <openssl/bn.h> -#include <openssl/ec.h> -#include <openssl/mem.h> -#include <openssl/nid.h> - -#include "../../internal.h" -#include "../../test/abi_test.h" -#include "../../test/file_test.h" -#include "../../test/test_util.h" -#include "../bn/internal.h" -#include "internal.h" -#include "p256-nistz.h" - - -BSSL_NAMESPACE_BEGIN -namespace { - -// Disable tests if BORINGSSL_SHARED_LIBRARY is defined. These tests need access -// to internal functions. -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) && \ - !defined(OPENSSL_SMALL) && !defined(BORINGSSL_SHARED_LIBRARY) - -struct P256NistzSelectImpl { - const char *name; - void (*select_w5)(P256_POINT *val, const P256_POINT in_t[16], int index); - void (*select_w7)(P256_POINT_AFFINE *val, const P256_POINT_AFFINE in_t[64], - int index); -}; - -static std::vector<P256NistzSelectImpl> AllP256NistzSelectImpls() { - std::vector<P256NistzSelectImpl> impls; -#if defined(OPENSSL_X86_64) - impls.push_back({ - "NoHW", - ecp_nistz256_select_w5_nohw, - ecp_nistz256_select_w7_nohw, - }); - if (CRYPTO_is_AVX2_capable()) { - impls.push_back({ - "AVX2", - ecp_nistz256_select_w5_nohw, - ecp_nistz256_select_w7_nohw, - }); - } -#else - impls.push_back({ - "Impl", - ecp_nistz256_select_w5, - ecp_nistz256_select_w7, - }); -#endif - return impls; -} - -class P256NistzSelectImplTest : public testing::TestWithParam<P256NistzSelectImpl> { - protected: - const P256NistzSelectImpl &impl() const { return GetParam(); } -}; - -INSTANTIATE_TEST_SUITE_P(All, P256NistzSelectImplTest, - testing::ValuesIn(AllP256NistzSelectImpls()), - [](const testing::TestParamInfo<P256NistzSelectImpl> ¶ms) - -> std::string { return params.param.name; }); - -TEST_P(P256NistzSelectImplTest, SelectW5) { - // Fill a table with some garbage input. - alignas(64) P256_POINT table[16]; - for (size_t i = 0; i < 16; i++) { - OPENSSL_memset(table[i].X, static_cast<uint8_t>(3 * i), sizeof(table[i].X)); - OPENSSL_memset(table[i].Y, static_cast<uint8_t>(3 * i + 1), - sizeof(table[i].Y)); - OPENSSL_memset(table[i].Z, static_cast<uint8_t>(3 * i + 2), - sizeof(table[i].Z)); - } - - for (int i = 0; i <= 16; i++) { - P256_POINT val; - impl().select_w5(&val, table, i); - - P256_POINT expected; - if (i == 0) { - OPENSSL_memset(&expected, 0, sizeof(expected)); - } else { - expected = table[i-1]; - } - - EXPECT_EQ(Bytes(reinterpret_cast<const char *>(&expected), sizeof(expected)), - Bytes(reinterpret_cast<const char *>(&val), sizeof(val))); - } - - // This is a constant-time function, so it is only necessary to instrument one - // index for ABI checking. - P256_POINT val; - CHECK_ABI(impl().select_w5, &val, table, 7); -} - -TEST_P(P256NistzSelectImplTest, SelectW7) { - // Fill a table with some garbage input. - alignas(64) P256_POINT_AFFINE table[64]; - for (size_t i = 0; i < 64; i++) { - OPENSSL_memset(table[i].X, static_cast<uint8_t>(2 * i), sizeof(table[i].X)); - OPENSSL_memset(table[i].Y, static_cast<uint8_t>(2 * i + 1), - sizeof(table[i].Y)); - } - - for (int i = 0; i <= 64; i++) { - P256_POINT_AFFINE val; - impl().select_w7(&val, table, i); - - P256_POINT_AFFINE expected; - if (i == 0) { - OPENSSL_memset(&expected, 0, sizeof(expected)); - } else { - expected = table[i-1]; - } - - EXPECT_EQ(Bytes(reinterpret_cast<const char *>(&expected), sizeof(expected)), - Bytes(reinterpret_cast<const char *>(&val), sizeof(val))); - } - - // This is a constant-time function, so it is only necessary to instrument one - // index for ABI checking. - P256_POINT_AFFINE val; - CHECK_ABI(impl().select_w7, &val, table, 42); -} - - -static std::string FieldElementToString(const BN_ULONG a[P256_LIMBS]) { - std::string ret; - for (size_t i = P256_LIMBS-1; i < P256_LIMBS; i--) { - char buf[2 * BN_BYTES + 1]; - snprintf(buf, sizeof(buf), BN_HEX_FMT2, a[i]); - ret += buf; - } - return ret; -} - -static testing::AssertionResult ExpectFieldElementsEqual( - const char *expected_expr, const char *actual_expr, - const BN_ULONG expected[P256_LIMBS], const BN_ULONG actual[P256_LIMBS]) { - if (OPENSSL_memcmp(expected, actual, sizeof(BN_ULONG) * P256_LIMBS) == 0) { - return testing::AssertionSuccess(); - } - - return testing::AssertionFailure() - << "Expected: " << FieldElementToString(expected) << " (" - << expected_expr << ")\n" - << "Actual: " << FieldElementToString(actual) << " (" << actual_expr - << ")"; -} - -#define EXPECT_FIELD_ELEMENTS_EQUAL(a, b) \ - EXPECT_PRED_FORMAT2(ExpectFieldElementsEqual, a, b) - -// P-256 scalars and field elements are the same size. -#define EXPECT_SCALARS_EQUAL(a, b) EXPECT_FIELD_ELEMENTS_EQUAL(a, b) - -TEST(P256_NistzTest, BEEU) { -#if defined(OPENSSL_X86_64) - if (!CRYPTO_is_AVX_capable()) { - // No AVX support; cannot run the BEEU code. - return; - } -#endif - - const EC_GROUP *group = EC_group_p256(); - BN_ULONG order_words[P256_LIMBS]; - ASSERT_TRUE( - bn_copy_words(order_words, P256_LIMBS, EC_GROUP_get0_order(group))); - - BN_ULONG in[P256_LIMBS], out[P256_LIMBS]; - EC_SCALAR in_scalar, out_scalar, result; - OPENSSL_memset(in, 0, sizeof(in)); - - // Trying to find the inverse of zero should fail. - ASSERT_FALSE(beeu_mod_inverse_vartime(out, in, order_words)); - // This is not a constant-time function, so instrument both zero and a few - // inputs below. - ASSERT_FALSE(CHECK_ABI(beeu_mod_inverse_vartime, out, in, order_words)); - - BN_ULONG kOne[P256_LIMBS] = {}; - kOne[0] = 1; - - auto beeu_test = [&] { - EXPECT_TRUE(bn_less_than_words(in, order_words, P256_LIMBS)); - ASSERT_TRUE(beeu_mod_inverse_vartime(out, in, order_words)); - EXPECT_TRUE(bn_less_than_words(out, order_words, P256_LIMBS)); - - // Calculate out*in and confirm that it equals one, modulo the order. We - // only have Montgomery multiplication, but if we convert one input, and not - // the other, this gives the result outside the Montgomery domain. - OPENSSL_memcpy(in_scalar.words, in, sizeof(in)); - OPENSSL_memcpy(out_scalar.words, out, sizeof(out)); - ec_scalar_to_montgomery(group, &in_scalar, &in_scalar); - ec_scalar_mul_montgomery(group, &result, &in_scalar, &out_scalar); - EXPECT_SCALARS_EQUAL(kOne, result.words); - - // Invert the result and expect to get back to the original value. - ASSERT_TRUE(beeu_mod_inverse_vartime(out, out, order_words)); - EXPECT_SCALARS_EQUAL(in, out); - }; - - for (BN_ULONG i = 1; i < 2000; i++) { - SCOPED_TRACE(i); - in[0] = i; - if (i >= 1000) { - in[1] = i << 8; - in[2] = i << 32; - in[3] = i << 48; - } else { - in[1] = in[2] = in[3] = 0; - } - - beeu_test(); - if (i < 5) { - EXPECT_TRUE(CHECK_ABI(beeu_mod_inverse_vartime, out, in, order_words)); - } - } - - for (int i = 0; i < 255; i++) { - SCOPED_TRACE(i); - // Test `in` = 2^i. - OPENSSL_memset(in, 0, sizeof(in)); - in[i / BN_BITS2] = BN_ULONG{1} << (i % BN_BITS2); - beeu_test(); - - // Test `in` = N - 2^i. - bn_sub_words(in, order_words, in, P256_LIMBS); - beeu_test(); - } -} - -static bool GetFieldElement(FileTest *t, BN_ULONG out[P256_LIMBS], - const char *name) { - std::vector<uint8_t> bytes; - if (!t->GetBytes(&bytes, name)) { - return false; - } - - if (bytes.size() != BN_BYTES * P256_LIMBS) { - ADD_FAILURE() << "Invalid length: " << name; - return false; - } - - // `byte` contains bytes in big-endian while `out` should contain `BN_ULONG`s - // in little-endian. - OPENSSL_memset(out, 0, P256_LIMBS * sizeof(BN_ULONG)); - for (size_t i = 0; i < bytes.size(); i++) { - out[P256_LIMBS - 1 - (i / BN_BYTES)] <<= 8; - out[P256_LIMBS - 1 - (i / BN_BYTES)] |= bytes[i]; - } - - return true; -} - -static bool PointToAffine(P256_POINT_AFFINE *out, const P256_POINT *in) { - static const uint8_t kP[] = { - 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, - 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, - }; - - UniquePtr<BIGNUM> x(BN_new()), y(BN_new()), z(BN_new()); - UniquePtr<BIGNUM> p(BN_bin2bn(kP, sizeof(kP), nullptr)); - if (!x || !y || !z || !p || !bn_set_words(x.get(), in->X, P256_LIMBS) || - !bn_set_words(y.get(), in->Y, P256_LIMBS) || - !bn_set_words(z.get(), in->Z, P256_LIMBS)) { - return false; - } - - // Coordinates must be fully-reduced. - if (BN_cmp(x.get(), p.get()) >= 0 || - BN_cmp(y.get(), p.get()) >= 0 || - BN_cmp(z.get(), p.get()) >= 0) { - return false; - } - - if (BN_is_zero(z.get())) { - // The point at infinity is represented as (0, 0). - OPENSSL_memset(out, 0, sizeof(P256_POINT_AFFINE)); - return true; - } - - UniquePtr<BN_CTX> ctx(BN_CTX_new()); - UniquePtr<BN_MONT_CTX> mont(BN_MONT_CTX_new_for_modulus(p.get(), ctx.get())); - if (!ctx || !mont || - // Invert Z. - !BN_from_montgomery(z.get(), z.get(), mont.get(), ctx.get()) || - !BN_mod_inverse(z.get(), z.get(), p.get(), ctx.get()) || - !BN_to_montgomery(z.get(), z.get(), mont.get(), ctx.get()) || - // Convert (X, Y, Z) to (X/Z^2, Y/Z^3). - !BN_mod_mul_montgomery(x.get(), x.get(), z.get(), mont.get(), - ctx.get()) || - !BN_mod_mul_montgomery(x.get(), x.get(), z.get(), mont.get(), - ctx.get()) || - !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), - ctx.get()) || - !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), - ctx.get()) || - !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), - ctx.get()) || - !bn_copy_words(out->X, P256_LIMBS, x.get()) || - !bn_copy_words(out->Y, P256_LIMBS, y.get())) { - return false; - } - return true; -} - -static testing::AssertionResult ExpectPointsEqual( - const char *expected_expr, const char *actual_expr, - const P256_POINT_AFFINE *expected, const P256_POINT *actual) { - // There are multiple representations of the same `P256_POINT`, so convert to - // `P256_POINT_AFFINE` and compare. - P256_POINT_AFFINE affine; - if (!PointToAffine(&affine, actual)) { - return testing::AssertionFailure() - << "Could not convert " << actual_expr << " to affine: (" - << FieldElementToString(actual->X) << ", " - << FieldElementToString(actual->Y) << ", " - << FieldElementToString(actual->Z) << ")"; - } - - if (OPENSSL_memcmp(expected, &affine, sizeof(P256_POINT_AFFINE)) != 0) { - return testing::AssertionFailure() - << "Expected: (" << FieldElementToString(expected->X) << ", " - << FieldElementToString(expected->Y) << ") (" << expected_expr - << "; affine)\n" - << "Actual: (" << FieldElementToString(affine.X) << ", " - << FieldElementToString(affine.Y) << ") (" << actual_expr << ")"; - } - - return testing::AssertionSuccess(); -} - -#define EXPECT_POINTS_EQUAL(a, b) EXPECT_PRED_FORMAT2(ExpectPointsEqual, a, b) - -struct P256NistzImpl { - const char *name; - void (*neg)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]); - void (*mul_mont)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]); - void (*sqr_mont)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS]); - void (*ord_mul_mont)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS], - const BN_ULONG b[P256_LIMBS]); - void (*ord_sqr_mont)(BN_ULONG res[P256_LIMBS], const BN_ULONG a[P256_LIMBS], - BN_ULONG rep); - void (*point_double)(P256_POINT *r, const P256_POINT *a); - void (*point_add)(P256_POINT *r, const P256_POINT *a, const P256_POINT *b); - void (*point_add_affine)(P256_POINT *r, const P256_POINT *a, - const P256_POINT_AFFINE *b); -}; - -static std::vector<P256NistzImpl> AllP256NistzImpls() { - std::vector<P256NistzImpl> impls; -#if defined(OPENSSL_X86_64) - impls.push_back({ - "NoHW", - ecp_nistz256_neg, - ecp_nistz256_mul_mont_nohw, - ecp_nistz256_sqr_mont_nohw, - ecp_nistz256_ord_mul_mont_nohw, - ecp_nistz256_ord_sqr_mont_nohw, - ecp_nistz256_point_double_nohw, - ecp_nistz256_point_add_nohw, - ecp_nistz256_point_add_affine_nohw, - }); - if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { - impls.push_back({ - "ADX", - ecp_nistz256_neg, - ecp_nistz256_mul_mont_adx, - ecp_nistz256_sqr_mont_adx, - ecp_nistz256_ord_mul_mont_adx, - ecp_nistz256_ord_sqr_mont_adx, - ecp_nistz256_point_double_adx, - ecp_nistz256_point_add_adx, - ecp_nistz256_point_add_affine_adx, - }); - } -#else - impls.push_back({ - "Impl", - ecp_nistz256_neg, - ecp_nistz256_mul_mont, - ecp_nistz256_sqr_mont, - ecp_nistz256_ord_mul_mont, - ecp_nistz256_ord_sqr_mont, - ecp_nistz256_point_double, - ecp_nistz256_point_add, - ecp_nistz256_point_add_affine, - }); -#endif - return impls; -} - -class P256NistzImplTest : public testing::TestWithParam<P256NistzImpl> { - protected: - const P256NistzImpl &impl() const { return GetParam(); } -}; - -INSTANTIATE_TEST_SUITE_P(All, P256NistzImplTest, - testing::ValuesIn(AllP256NistzImpls()), - [](const testing::TestParamInfo<P256NistzImpl> ¶ms) - -> std::string { return params.param.name; }); - -static void TestNegate(FileTest *t, const P256NistzImpl &impl) { - BN_ULONG a[P256_LIMBS], b[P256_LIMBS]; - ASSERT_TRUE(GetFieldElement(t, a, "A")); - ASSERT_TRUE(GetFieldElement(t, b, "B")); - - // Test that -A = B. - BN_ULONG ret[P256_LIMBS]; - impl.neg(ret, a); - EXPECT_FIELD_ELEMENTS_EQUAL(b, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - impl.neg(ret, ret /* a */); - EXPECT_FIELD_ELEMENTS_EQUAL(b, ret); - - // Test that -B = A. - impl.neg(ret, b); - EXPECT_FIELD_ELEMENTS_EQUAL(a, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - impl.neg(ret, ret /* b */); - EXPECT_FIELD_ELEMENTS_EQUAL(a, ret); -} - -static void TestMulMont(FileTest *t, const P256NistzImpl &impl) { - BN_ULONG a[P256_LIMBS], b[P256_LIMBS], result[P256_LIMBS]; - ASSERT_TRUE(GetFieldElement(t, a, "A")); - ASSERT_TRUE(GetFieldElement(t, b, "B")); - ASSERT_TRUE(GetFieldElement(t, result, "Result")); - - BN_ULONG ret[P256_LIMBS]; - impl.mul_mont(ret, a, b); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - impl.mul_mont(ret, b, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - impl.mul_mont(ret, ret /* a */, b); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - impl.mul_mont(ret, b, ret); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - impl.mul_mont(ret, a, ret /* b */); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - impl.mul_mont(ret, ret /* b */, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { - impl.sqr_mont(ret, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - impl.sqr_mont(ret, ret /* a */); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - } -} - -static void TestPointAdd(FileTest *t, const P256NistzImpl &impl) { - P256_POINT a, b; - P256_POINT_AFFINE result; - ASSERT_TRUE(GetFieldElement(t, a.X, "A.X")); - ASSERT_TRUE(GetFieldElement(t, a.Y, "A.Y")); - ASSERT_TRUE(GetFieldElement(t, a.Z, "A.Z")); - ASSERT_TRUE(GetFieldElement(t, b.X, "B.X")); - ASSERT_TRUE(GetFieldElement(t, b.Y, "B.Y")); - ASSERT_TRUE(GetFieldElement(t, b.Z, "B.Z")); - ASSERT_TRUE(GetFieldElement(t, result.X, "Result.X")); - ASSERT_TRUE(GetFieldElement(t, result.Y, "Result.Y")); - - P256_POINT ret; - impl.point_add(&ret, &a, &b); - EXPECT_POINTS_EQUAL(&result, &ret); - - impl.point_add(&ret, &b, &a); - EXPECT_POINTS_EQUAL(&result, &ret); - - OPENSSL_memcpy(&ret, &a, sizeof(ret)); - impl.point_add(&ret, &ret /* a */, &b); - EXPECT_POINTS_EQUAL(&result, &ret); - - OPENSSL_memcpy(&ret, &a, sizeof(ret)); - impl.point_add(&ret, &b, &ret /* a */); - EXPECT_POINTS_EQUAL(&result, &ret); - - OPENSSL_memcpy(&ret, &b, sizeof(ret)); - impl.point_add(&ret, &a, &ret /* b */); - EXPECT_POINTS_EQUAL(&result, &ret); - - OPENSSL_memcpy(&ret, &b, sizeof(ret)); - impl.point_add(&ret, &ret /* b */, &a); - EXPECT_POINTS_EQUAL(&result, &ret); - - P256_POINT_AFFINE a_affine, b_affine, infinity; - OPENSSL_memset(&infinity, 0, sizeof(infinity)); - ASSERT_TRUE(PointToAffine(&a_affine, &a)); - ASSERT_TRUE(PointToAffine(&b_affine, &b)); - - // ecp_nistz256_point_add_affine does not work when a == b unless doubling the - // point at infinity. - if (OPENSSL_memcmp(&a_affine, &b_affine, sizeof(a_affine)) != 0 || - OPENSSL_memcmp(&a_affine, &infinity, sizeof(a_affine)) == 0) { - impl.point_add_affine(&ret, &a, &b_affine); - EXPECT_POINTS_EQUAL(&result, &ret); - - OPENSSL_memcpy(&ret, &a, sizeof(ret)); - impl.point_add_affine(&ret, &ret /* a */, &b_affine); - EXPECT_POINTS_EQUAL(&result, &ret); - - impl.point_add_affine(&ret, &b, &a_affine); - EXPECT_POINTS_EQUAL(&result, &ret); - - OPENSSL_memcpy(&ret, &b, sizeof(ret)); - impl.point_add_affine(&ret, &ret /* b */, &a_affine); - EXPECT_POINTS_EQUAL(&result, &ret); - } - - if (OPENSSL_memcmp(&a, &b, sizeof(a)) == 0) { - impl.point_double(&ret, &a); - EXPECT_POINTS_EQUAL(&result, &ret); - - ret = a; - impl.point_double(&ret, &ret /* a */); - EXPECT_POINTS_EQUAL(&result, &ret); - } -} - -static void TestOrdMulMont(FileTest *t, const P256NistzImpl &impl) { - // This test works on scalars rather than field elements, but the - // representation is the same. - BN_ULONG a[P256_LIMBS], b[P256_LIMBS], result[P256_LIMBS]; - ASSERT_TRUE(GetFieldElement(t, a, "A")); - ASSERT_TRUE(GetFieldElement(t, b, "B")); - ASSERT_TRUE(GetFieldElement(t, result, "Result")); - - BN_ULONG ret[P256_LIMBS]; - impl.ord_mul_mont(ret, a, b); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - impl.ord_mul_mont(ret, b, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - impl.ord_mul_mont(ret, ret /* a */, b); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - impl.ord_mul_mont(ret, b, ret); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - impl.ord_mul_mont(ret, a, ret /* b */); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, b, sizeof(ret)); - impl.ord_mul_mont(ret, ret /* b */, a); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { - impl.ord_sqr_mont(ret, a, 1); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - - OPENSSL_memcpy(ret, a, sizeof(ret)); - impl.ord_sqr_mont(ret, ret /* a */, 1); - EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); - } -} - -TEST_P(P256NistzImplTest, TestVectors) { - return FileTestGTest("crypto/fipsmodule/ec/p256-nistz_tests.txt", - [&](FileTest *t) { - if (t->GetParameter() == "Negate") { - TestNegate(t, impl()); - } else if (t->GetParameter() == "MulMont") { - TestMulMont(t, impl()); - } else if (t->GetParameter() == "PointAdd") { - TestPointAdd(t, impl()); - } else if (t->GetParameter() == "OrdMulMont") { - TestOrdMulMont(t, impl()); - } else { - FAIL() << "Unknown test type:" << t->GetParameter(); - } - }); -} - -// Instrument the functions covered in TestVectors for ABI checking. -TEST_P(P256NistzImplTest, ABI) { - BN_ULONG a[P256_LIMBS], b[P256_LIMBS], c[P256_LIMBS]; - OPENSSL_memset(a, 0x01, sizeof(a)); - // These functions are all branchless, so it is only necessary to - // instrument one call each for ABI checking. - CHECK_ABI(impl().neg, b, a); - CHECK_ABI(impl().mul_mont, c, a, b); - CHECK_ABI(impl().sqr_mont, c, a); - CHECK_ABI(impl().ord_mul_mont, c, a, b); - - // Check a few different loop counts. - CHECK_ABI(impl().ord_sqr_mont, b, a, 1); - CHECK_ABI(impl().ord_sqr_mont, b, a, 3); - - // Point addition has some special cases around infinity and doubling. Test - // a few different scenarios. - static const P256_POINT kA = { - {TOBN(0x60559ac7, 0xc8d0d89d), TOBN(0x6cda3400, 0x545f7e2c), - TOBN(0x9b5159e0, 0x323e6048), TOBN(0xcb8dea33, 0x27057fe6)}, - {TOBN(0x81a2d3bc, 0xc93a2d53), TOBN(0x81f40762, 0xa4f33ccf), - TOBN(0xc3c3300a, 0xa8ad50ea), TOBN(0x553de89b, 0x31719830)}, - {TOBN(0x3fd9470f, 0xb277d181), TOBN(0xc191b8d5, 0x6376f206), - TOBN(0xb2572c1f, 0x45eda26f), TOBN(0x4589e40d, 0xf2efc546)}, - }; - static const P256_POINT kB = { - {TOBN(0x3cf0b0aa, 0x92054341), TOBN(0xb949bb80, 0xdab57807), - TOBN(0x99de6814, 0xefd21b3e), TOBN(0x32ad5649, 0x7c6c6e83)}, - {TOBN(0x06afaa02, 0x688399e0), TOBN(0x75f2d096, 0x2a3ce65c), - TOBN(0xf6a31eb7, 0xca0244b3), TOBN(0x57b33b7a, 0xcfeee75e)}, - {TOBN(0x7617d2e0, 0xb4f1d35f), TOBN(0xa922cb10, 0x7f592b65), - TOBN(0x12fd6c7a, 0x51a2f474), TOBN(0x337d5e1e, 0xc2fc711b)}, - }; - // This file represents Jacobian infinity as (*, *, 0). - static const P256_POINT kInfinity = { - {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, - {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, - {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, - }; - - P256_POINT p; - CHECK_ABI(impl().point_add, &p, &kA, &kB); - CHECK_ABI(impl().point_add, &p, &kA, &kA); - OPENSSL_memcpy(&p, &kA, sizeof(P256_POINT)); - impl().neg(p.Y, p.Y); - CHECK_ABI(impl().point_add, &p, &kA, &p); // A + -A - CHECK_ABI(impl().point_add, &p, &kA, &kInfinity); - CHECK_ABI(impl().point_add, &p, &kInfinity, &kA); - CHECK_ABI(impl().point_add, &p, &kInfinity, &kInfinity); - CHECK_ABI(impl().point_double, &p, &kA); - CHECK_ABI(impl().point_double, &p, &kInfinity); - - static const P256_POINT_AFFINE kC = { - {TOBN(0x7e3ad339, 0xfb3fa5f0), TOBN(0x559d669d, 0xe3a047b2), - TOBN(0x8883b298, 0x7042e595), TOBN(0xfabada65, 0x7e477f08)}, - {TOBN(0xd9cfceb8, 0xda1c3e85), TOBN(0x80863761, 0x0ce6d6bc), - TOBN(0xa8409d84, 0x66034f02), TOBN(0x05519925, 0x31a68d55)}, - }; - // This file represents affine infinity as (0, 0). - static const P256_POINT_AFFINE kInfinityAffine = { - {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, - {TOBN(0, 0), TOBN(0, 0), TOBN(0, 0), TOBN(0, 0)}, - }; - - CHECK_ABI(impl().point_add_affine, &p, &kA, &kC); - CHECK_ABI(impl().point_add_affine, &p, &kA, &kInfinityAffine); - CHECK_ABI(impl().point_add_affine, &p, &kInfinity, &kInfinityAffine); - CHECK_ABI(impl().point_add_affine, &p, &kInfinity, &kC); -} - -#endif - -} // namespace -BSSL_NAMESPACE_END
diff --git a/crypto/fipsmodule/ec/p256-nistz_tests.txt b/crypto/fipsmodule/ec/p256-tests.txt similarity index 100% rename from crypto/fipsmodule/ec/p256-nistz_tests.txt rename to crypto/fipsmodule/ec/p256-tests.txt
diff --git a/crypto/fipsmodule/ec/p256.cc.inc b/crypto/fipsmodule/ec/p256.cc.inc index a67923d..6ce9471 100644 --- a/crypto/fipsmodule/ec/p256.cc.inc +++ b/crypto/fipsmodule/ec/p256.cc.inc
@@ -1,4 +1,6 @@ // Copyright 2020 The BoringSSL Authors +// Copyright 2014-2016 The OpenSSL Project Authors. All Rights Reserved. +// Copyright (c) 2014, Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,81 +14,346 @@ // See the License for the specific language governing permissions and // limitations under the License. -// An implementation of the NIST P-256 elliptic curve point multiplication. -// 256-bit Montgomery form for 64 and 32-bit. Field operations are generated by -// Fiat, which lives in //third_party/fiat. - #include <openssl/base.h> #include <openssl/bn.h> #include <openssl/ec.h> -#include <openssl/err.h> #include <openssl/mem.h> #include <assert.h> -#include <string.h> #include <iterator> #include "../../internal.h" #include "../delocate.h" -#include "./internal.h" +#include "internal.h" -#include "../../../third_party/fiat/p256_field.c.inc" -#include "../../../third_party/fiat/p256_point.br.c.inc" - +#include "p256_internal.h" using namespace bssl; -// utility functions, handwritten - -#if defined(OPENSSL_64_BIT) -#define FIAT_P256_NLIMBS 4 -typedef uint64_t fiat_p256_limb_t; -typedef uint64_t fiat_p256_felem[FIAT_P256_NLIMBS]; -static const fiat_p256_felem fiat_p256_one = {0x1, 0xffffffff00000000, - 0xffffffffffffffff, 0xfffffffe}; -#else // 64BIT; else 32BIT -#define FIAT_P256_NLIMBS 8 -typedef uint32_t fiat_p256_limb_t; -typedef uint32_t fiat_p256_felem[FIAT_P256_NLIMBS]; -static const fiat_p256_felem fiat_p256_one = { - 0x1, 0x0, 0x0, 0xffffffff, 0xffffffff, 0xffffffff, 0xfffffffe, 0x0}; -#endif // 64BIT - - -static void fiat_p256_copy(fiat_p256_limb_t out[FIAT_P256_NLIMBS], - const fiat_p256_limb_t in1[FIAT_P256_NLIMBS]) { - for (size_t i = 0; i < FIAT_P256_NLIMBS; i++) { - out[i] = in1[i]; +static void fiat_p256_opp_conditional(fiat_p256_felem x, crypto_word_t c) { + alignas(32) fiat_p256_felem n; + fiat_p256_opp(n, x); + for (size_t i = 0; i < P256_LIMBS; ++i) { + x[i] = br_cmov(c, n[i], x[i]); } } -static void fiat_p256_cmovznz(fiat_p256_limb_t out[FIAT_P256_NLIMBS], - fiat_p256_limb_t t, - const fiat_p256_limb_t z[FIAT_P256_NLIMBS], - const fiat_p256_limb_t nz[FIAT_P256_NLIMBS]) { - fiat_p256_selectznz(out, !!t, z, nz); +// bit returns the `i`th bit in `in`. +static crypto_word_t bit(const uint8_t in[32], int i) { + if (i < 0 || i >= 256) { + return 0; + } + return (in[i >> 3] >> (i & 7)) & 1; } -static void fiat_p256_from_words(fiat_p256_felem out, - const BN_ULONG in[32 / sizeof(BN_ULONG)]) { - // Typically, `BN_ULONG` and `fiat_p256_limb_t` will be the same type, but on - // 64-bit platforms without `uint128_t`, they are different. However, on - // little-endian systems, `uint64_t[4]` and `uint32_t[8]` have the same - // layout. - OPENSSL_memcpy(out, in, 32); +// point multiplication +static void fiat_p256_select_point_16(fiat_p256_felem dst[3], + const fiat_p256_felem src[16][3], + size_t i) { + OPENSSL_memset(dst, 0, 3 * sizeof(fiat_p256_felem)); + OPENSSL_CLANG_PRAGMA("clang loop unroll_count(4)") + for (size_t j = 0; j < 16; j++) { + constant_time_conditional_memxor(dst, &src[j], 3 * sizeof(fiat_p256_felem), + constant_time_eq_w(i, j)); + } } -static void fiat_p256_from_generic(fiat_p256_felem out, const EC_FELEM *in) { - fiat_p256_from_words(out, in->words); +// Precompute multiples of `p`. p_pre_comp[i] is (i+1) * `p`. +static void p256_point_mul(fiat_p256_felem out[3], const fiat_p256_felem p[3], + const uint8_t s[32]) { + alignas(32) fiat_p256_felem p_pre_comp[16][3]; + OPENSSL_memcpy(p_pre_comp[0], p, sizeof(p_pre_comp[0])); + for (size_t j = 2; j <= 16; ++j) { + if (j & 1) { + p256_point_add_vartime_if_doubling((uintptr_t)p_pre_comp[j - 1], + (uintptr_t)p_pre_comp[j - 2], + (uintptr_t)p_pre_comp[0]); + } else { + p256_point_double((uintptr_t)p_pre_comp[j - 1], + (uintptr_t)p_pre_comp[(j - 1) / 2]); + } + } + alignas(32) fiat_p256_felem ret[3]; + bool ret_is_zero = true; + + for (size_t i = 51; i < 52; i--) { + if (!ret_is_zero) { + for (size_t k = 4; k < 5; k--) { + p256_point_double((uintptr_t)ret, (uintptr_t)ret); + } + } + + crypto_word_t bits = 0; + OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") + for (size_t k = 5; k < 6; k--) { + bits |= bit(s, i * 5 - 1 + k) << k; + } + crypto_word_t sign, digit; + ec_GFp_nistp_recode_scalar_bits(&sign, &digit, bits); + + alignas(32) fiat_p256_felem t[3]; + fiat_p256_select_point_16(t, p_pre_comp, digit - 1); + fiat_p256_opp_conditional(t[1], sign); + + if (!ret_is_zero) { + p256_point_add_vartime_if_doubling((uintptr_t)ret, (uintptr_t)ret, + (uintptr_t)t); + } else { + OPENSSL_memcpy(ret, t, sizeof(ret)); + ret_is_zero = false; + } + } + // The first loop iteration will initialize `ret`. + assert(!ret_is_zero); + + OPENSSL_memcpy(out, ret, sizeof(ret)); } -static void fiat_p256_to_generic(EC_FELEM *out, const fiat_p256_felem in) { - // See `fiat_p256_from_words`. - OPENSSL_memcpy(out->words, in, 32); +// point_mul_public +#include "./p256_table.h" + +static void ec_GFp_nistp256_point_mul_public(const EC_GROUP *group, + EC_JACOBIAN *r, + const EC_SCALAR *gs, + const EC_JACOBIAN *p, + const EC_SCALAR *ps) { + const uint8_t *g_scalar = (uint8_t *)gs->words; + int8_t p_wNAF[257] = {0}; + + alignas(32) fiat_p256_felem p_pre_comp[1 << (4 - 1)][3]; + OPENSSL_memcpy(p_pre_comp[0][2], &p->Z, 32); + if (!p256_point_iszero((uintptr_t)p_pre_comp[0])) { + ec_compute_wNAF(group, p_wNAF, ps, /*bits=*/256, /*w=*/4); + // Precompute multiples of `p`. p_pre_comp[i] is (2*i+1) * `p`. + OPENSSL_memcpy(p_pre_comp[0][0], &p->X, 32); + OPENSSL_memcpy(p_pre_comp[0][1], &p->Y, 32); + alignas(32) fiat_p256_felem p2[3]; + p256_point_double((uintptr_t)p2, (uintptr_t)p_pre_comp[0]); + for (size_t i = 1; i < std::size(p_pre_comp); i++) { + p256_point_add_nz_nz_neq((uintptr_t)p_pre_comp[i], + (uintptr_t)p_pre_comp[i - 1], (uintptr_t)p2); + } + } + + alignas(32) fiat_p256_felem ret[3] = {}; + bool ret_is_zero = true; // Save some point operations, avoid 0+Q + for (int i = 256; i >= 0; i--) { + if (!ret_is_zero) { + p256_point_double((uintptr_t)ret, (uintptr_t)ret); + } + + if (i <= 31) { + OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") + for (size_t j = 1; j < 2; j--) { + crypto_word_t bits = 0; + OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") + for (size_t k = 3; k < 4; k--) { + bits |= bit(g_scalar, i + j * 32 + k * 64) << k; + } + if (bits != 0) { + if (!ret_is_zero) { + alignas(32) fiat_p256_felem t[3]; + fiat_p256_set_one(t[2]); + OPENSSL_memcpy(t, fiat_p256_g_pre_comp[j][bits - 1], 64); + p256_point_add_affinenz_conditional_vartime_if_doubling( + (uintptr_t)ret, (uintptr_t)ret, (uintptr_t)t, 1); + ret_is_zero = p256_point_iszero((uintptr_t)ret); + } else { + OPENSSL_memcpy(ret, fiat_p256_g_pre_comp[j][bits - 1], + sizeof(fiat_p256_g_pre_comp[j][bits - 1])); + fiat_p256_set_one(ret[2]); + ret_is_zero = false; + } + } + } + } + + int digit = p_wNAF[i]; + if (digit != 0) { + assert(digit & 1); + size_t idx = (size_t)(digit < 0 ? (-digit) >> 1 : digit >> 1); + fiat_p256_felem t[3]; + OPENSSL_memcpy(t, p_pre_comp[idx], sizeof(t)); + if (digit < 0) { + fiat_p256_opp(t[1], t[1]); + } + if (!ret_is_zero) { + p256_point_add_vartime_if_doubling((uintptr_t)ret, (uintptr_t)ret, + (uintptr_t)t); + ret_is_zero = p256_point_iszero((uintptr_t)ret); + } else { + OPENSSL_memcpy(ret, t, sizeof(ret)); + ret_is_zero = false; + } + } + } + + OPENSSL_memcpy(&r->X, ret[0], 32); + OPENSSL_memcpy(&r->Y, ret[1], 32); + OPENSSL_memcpy(&r->Z, ret[2], 32); } +// p256_point_mul_base (small and full) + +// fiat_p256_select_point_affine selects the `i`th point from a precomputation +// table and copies it to `dst`. If `n<=i`, the output is (0, 0). +static void fiat_p256_select_point_affine(fiat_p256_felem dst[2], + const fiat_p256_felem src[/*n*/][2], + size_t i, size_t n) { + OPENSSL_memset(dst, 0, 2 * sizeof(fiat_p256_felem)); + OPENSSL_CLANG_PRAGMA("clang loop unroll_count(8)") + for (size_t j = 0; j < n; j++) { + constant_time_conditional_memxor(dst, &src[j], 2 * sizeof(fiat_p256_felem), + constant_time_eq_w(i, j)); + } +} + +static void fiat_p256_conditional_zero_or_one(fiat_p256_felem out, + crypto_word_t c) { + fiat_p256_set_one(out); + crypto_word_t mask = value_barrier_w(~constant_time_is_zero_w(c)); + for (size_t i = 0; i < P256_LIMBS; ++i) { + out[i] &= mask; + } +} + +#if defined(OPENSSL_SMALL) + +#ifdef __clang__ +__attribute__((noinline)) // Do not inline into unrolled loop below. +#endif +static void fiat_p256_select_point_affine_15( + fiat_p256_felem dst[2], const fiat_p256_felem src[/*n*/][2], size_t i) { + fiat_p256_select_point_affine(dst, src, i, 15); // Intended to be inlined. +} + +static void p256_point_mul_base(fiat_p256_felem ret[3], const uint8_t s[32]) { + bool ret_is_zero = true; // Save two point operations in the first round. + alignas(32) fiat_p256_felem t[3]; + fiat_p256_set_one(t[2]); + for (size_t i = 31; i < 32; i--) { + if (!ret_is_zero) { + p256_point_double((uintptr_t)ret, (uintptr_t)ret); + } + OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") + for (size_t j = 1; j < 2; j--) { + crypto_word_t bits = 0; + OPENSSL_CLANG_PRAGMA("clang loop unroll(full)") + for (size_t k = 3; k < 4; k--) { + bits |= bit(s, i + 32 * j + 64 * k) << k; + } + fiat_p256_select_point_affine_15(t, fiat_p256_g_pre_comp[j], bits - 1); + + if (!ret_is_zero) { + p256_point_add_affinenz_conditional_vartime_if_doubling( + (uintptr_t)ret, (uintptr_t)ret, (uintptr_t)t, (uintptr_t)bits); + } else { + OPENSSL_memcpy(ret, t, sizeof(t)); + fiat_p256_conditional_zero_or_one(ret[2], bits); + ret_is_zero = false; + } + } + } + + // The first loop iteration will initialize `ret`. + assert(!ret_is_zero); +} +#else // defined(OPENSSL_SMALL) + +/* + * Originally written by Shay Gueron (1, 2), and Vlad Krasnov (1) + * (1) Intel Corporation, Israel Development Center, Haifa, Israel + * (2) University of Haifa, Israel + * + * Reference: + * S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with + * 256 Bit Primes" + */ + +// Precomputed tables for the default generator +typedef fiat_p256_felem PRECOMP256_ROW[64][2]; +#include "p256-nistz-table.h" + +#ifdef __clang__ +__attribute__((noinline)) // Do not inline into unrolled loop below. +#endif +static void fiat_p256_select_point_affine_16(fiat_p256_felem dst[2], + const fiat_p256_felem src[16][2], + size_t i) { + fiat_p256_select_point_affine(dst, src, i, 16); +} + +#ifdef __clang__ +__attribute__((noinline)) // Do not inline into unrolled loop below. +#endif +static void fiat_p256_select_point_affine_64(fiat_p256_felem dst[2], + const fiat_p256_felem src[64][2], + size_t i) { + fiat_p256_select_point_affine(dst, src, i, 64); +} + +// See `ec_GFp_nistp_recode_scalar_bits` in util.c for details +static crypto_word_t booth_recode_w7(crypto_word_t in) { + crypto_word_t s, d; + s = ~((in >> 7) - 1); + d = (1 << 8) - in - 1; + d = (d & s) | (in & ~s); + d = (d >> 1) + (d & 1); + return (d << 1) + (s & 1); +} + +static void p256_point_mul_base(fiat_p256_felem ret[3], const uint8_t s[32]) { + bool ret_is_zero = true; + alignas(32) fiat_p256_felem t[3]; + fiat_p256_set_one(t[2]); + for (size_t i = 36; i < 37; i--) { + // Load 7-bit windows, plus one bit below the window for `booth_recode_w7`, + // i.e. 8 bits from 7 * i - 1 to 7 * (i + 1). + constexpr size_t kMask = (1 << (7 + 1)) - 1; + crypto_word_t wvalue; + if (i == 0) { + wvalue = (s[0] << 1) & kMask; + } else { + size_t first_bit = 7 * i - 1; + size_t idx = first_bit / 8; + // The window may span two bytes. + wvalue = + s[idx] | (static_cast<crypto_word_t>(idx < 31 ? s[idx + 1] : 0) << 8); + wvalue = (wvalue >> (first_bit % 8)) & kMask; + } + wvalue = booth_recode_w7(wvalue); + if (i == 36) { + // The last window has only 4 bits instead of the full 7. + declassify_assert((wvalue >> 1) <= 16); + fiat_p256_select_point_affine_16(t, ecp_nistz256_precomputed[i], + (wvalue >> 1) - 1); + } else { + fiat_p256_select_point_affine_64(t, ecp_nistz256_precomputed[i], + (wvalue >> 1) - 1); + } + fiat_p256_opp_conditional(t[1], wvalue & 1); + + if (!ret_is_zero) { + p256_point_add_affinenz_conditional_vartime_if_doubling( + (uintptr_t)ret, (uintptr_t)ret, (uintptr_t)t, wvalue >> 1); + } else { + OPENSSL_memcpy(ret, t, sizeof(t)); + fiat_p256_conditional_zero_or_one(ret[2], wvalue >> 1); + ret_is_zero = false; + } + } + + // The first loop iteration will initialize `ret`. + assert(!ret_is_zero); +} +#endif // !OPENSSL_SMALL + + +// FIELD-ELEMENT INVERSION + + // fiat_p256_inv_square calculates `out` = `in`^{-2} // // Based on Fermat's Little Theorem: @@ -98,156 +365,182 @@ // This implements the addition chain described in // https://briansmith.org/ecc-inversion-addition-chains-01#p256_field_inversion fiat_p256_felem x2, x3, x6, x12, x15, x30, x32; - fiat_p256_square(x2, in); // 2^2 - 2^1 - fiat_p256_mul(x2, x2, in); // 2^2 - 2^0 + p256_coord_sqr(x2, in); // 2^2 - 2^1 + p256_coord_mul(x2, x2, in); // 2^2 - 2^0 - fiat_p256_square(x3, x2); // 2^3 - 2^1 - fiat_p256_mul(x3, x3, in); // 2^3 - 2^0 + p256_coord_sqr(x3, x2); // 2^3 - 2^1 + p256_coord_mul(x3, x3, in); // 2^3 - 2^0 - fiat_p256_square(x6, x3); + p256_coord_sqr(x6, x3); for (int i = 1; i < 3; i++) { - fiat_p256_square(x6, x6); - } // 2^6 - 2^3 - fiat_p256_mul(x6, x6, x3); // 2^6 - 2^0 + p256_coord_sqr(x6, x6); // 2^6 - 2^3 + } + p256_coord_mul(x6, x6, x3); // 2^6 - 2^0 - fiat_p256_square(x12, x6); + p256_coord_sqr(x12, x6); for (int i = 1; i < 6; i++) { - fiat_p256_square(x12, x12); - } // 2^12 - 2^6 - fiat_p256_mul(x12, x12, x6); // 2^12 - 2^0 + p256_coord_sqr(x12, x12); // 2^12 - 2^6 + } + p256_coord_mul(x12, x12, x6); // 2^12 - 2^0 - fiat_p256_square(x15, x12); + p256_coord_sqr(x15, x12); for (int i = 1; i < 3; i++) { - fiat_p256_square(x15, x15); - } // 2^15 - 2^3 - fiat_p256_mul(x15, x15, x3); // 2^15 - 2^0 + p256_coord_sqr(x15, x15); // 2^15 - 2^3 + } + p256_coord_mul(x15, x15, x3); // 2^15 - 2^0 - fiat_p256_square(x30, x15); + p256_coord_sqr(x30, x15); for (int i = 1; i < 15; i++) { - fiat_p256_square(x30, x30); - } // 2^30 - 2^15 - fiat_p256_mul(x30, x30, x15); // 2^30 - 2^0 + p256_coord_sqr(x30, x30); // 2^30 - 2^15 + } + p256_coord_mul(x30, x30, x15); // 2^30 - 2^0 - fiat_p256_square(x32, x30); - fiat_p256_square(x32, x32); // 2^32 - 2^2 - fiat_p256_mul(x32, x32, x2); // 2^32 - 2^0 + p256_coord_sqr(x32, x30); + p256_coord_sqr(x32, x32); // 2^32 - 2^2 + p256_coord_mul(x32, x32, x2); // 2^32 - 2^0 fiat_p256_felem ret; - fiat_p256_square(ret, x32); + p256_coord_sqr(ret, x32); for (int i = 1; i < 31 + 1; i++) { - fiat_p256_square(ret, ret); - } // 2^64 - 2^32 - fiat_p256_mul(ret, ret, in); // 2^64 - 2^32 + 2^0 + p256_coord_sqr(ret, ret); // 2^64 - 2^32 + } + p256_coord_mul(ret, ret, in); // 2^64 - 2^32 + 2^0 for (int i = 0; i < 96 + 32; i++) { - fiat_p256_square(ret, ret); - } // 2^192 - 2^160 + 2^128 - fiat_p256_mul(ret, ret, x32); // 2^192 - 2^160 + 2^128 + 2^32 - 2^0 + p256_coord_sqr(ret, ret); // 2^192 - 2^160 + 2^128 + } + p256_coord_mul(ret, ret, x32); // 2^192 - 2^160 + 2^128 + 2^32 - 2^0 for (int i = 0; i < 32; i++) { - fiat_p256_square(ret, ret); - } // 2^224 - 2^192 + 2^160 + 2^64 - 2^32 - fiat_p256_mul(ret, ret, x32); // 2^224 - 2^192 + 2^160 + 2^64 - 2^0 + p256_coord_sqr(ret, ret); // 2^224 - 2^192 + 2^160 + 2^64 - 2^32 + } + p256_coord_mul(ret, ret, x32); // 2^224 - 2^192 + 2^160 + 2^64 - 2^0 for (int i = 0; i < 30; i++) { - fiat_p256_square(ret, ret); - } // 2^254 - 2^222 + 2^190 + 2^94 - 2^30 - fiat_p256_mul(ret, ret, x30); // 2^254 - 2^222 + 2^190 + 2^94 - 2^0 - - fiat_p256_square(ret, ret); - fiat_p256_square(out, ret); // 2^256 - 2^224 + 2^192 + 2^96 - 2^2 -} - -// Group operations -// ---------------- -// -// Building on top of the field operations we have the operations on the -// elliptic curve group itself. Points on the curve are represented in Jacobian -// coordinates. - -static void fiat_p256_point_double(fiat_p256_felem x_out, fiat_p256_felem y_out, - fiat_p256_felem z_out, - const fiat_p256_felem x_in, - const fiat_p256_felem y_in, - const fiat_p256_felem z_in) { - uint8_t out[3*32], in[3*32]; - static_assert(sizeof(fiat_p256_felem) == 32); - OPENSSL_memcpy(&in[0], x_in, 32); - OPENSSL_memcpy(&in[32], y_in, 32); - OPENSSL_memcpy(&in[64], z_in, 32); - p256_point_double((br_word_t)out, (br_word_t)in); - OPENSSL_memcpy(x_out, &out[0], 32); - OPENSSL_memcpy(y_out, &out[32], 32); - OPENSSL_memcpy(z_out, &out[64], 32); -} - -static void fiat_p256_point_add(fiat_p256_felem x3, fiat_p256_felem y3, - fiat_p256_felem z3, const fiat_p256_felem x1, - const fiat_p256_felem y1, - const fiat_p256_felem z1, - const fiat_p256_felem x2, - const fiat_p256_felem y2, - const fiat_p256_felem z2) { - uint8_t out[3 * 32], in1[3 * 32], in2[3 * 32]; - static_assert(sizeof(fiat_p256_felem) == 32); - OPENSSL_memcpy(&in1[0], x1, 32); - OPENSSL_memcpy(&in1[32], y1, 32); - OPENSSL_memcpy(&in1[64], z1, 32); - OPENSSL_memcpy(&in2[0], x2, 32); - OPENSSL_memcpy(&in2[32], y2, 32); - OPENSSL_memcpy(&in2[64], z2, 32); - p256_point_add_vartime_if_doubling((br_word_t)out, (br_word_t)in1, - (br_word_t)in2); - OPENSSL_memcpy(x3, &out[0], 32); - OPENSSL_memcpy(y3, &out[32], 32); - OPENSSL_memcpy(z3, &out[64], 32); -} -#include "./p256_table.h" - -// fiat_p256_select_point_affine selects the `idx-1`th point from a -// precomputation table and copies it to out. If `idx` is zero, the output is -// the point at infinity. -static void fiat_p256_select_point_affine( - const fiat_p256_limb_t idx, size_t size, - const fiat_p256_felem pre_comp[/*size*/][2], fiat_p256_felem out[3]) { - OPENSSL_memset(out, 0, sizeof(fiat_p256_felem) * 3); - for (size_t i = 0; i < size; i++) { - fiat_p256_limb_t mismatch = i ^ (idx - 1); - fiat_p256_cmovznz(out[0], mismatch, pre_comp[i][0], out[0]); - fiat_p256_cmovznz(out[1], mismatch, pre_comp[i][1], out[1]); + p256_coord_sqr(ret, ret); // 2^254 - 2^222 + 2^190 + 2^94 - 2^30 } - fiat_p256_cmovznz(out[2], idx, out[2], fiat_p256_one); + p256_coord_mul(ret, ret, x30); // 2^254 - 2^222 + 2^190 + 2^94 - 2^0 + + p256_coord_sqr(ret, ret); + p256_coord_sqr(out, ret); // 2^256 - 2^224 + 2^192 + 2^96 - 2^2 } -// fiat_p256_select_point selects the `idx`th point from a precomputation table -// and copies it to out. -static void fiat_p256_select_point(const fiat_p256_limb_t idx, size_t size, - const fiat_p256_felem pre_comp[/*size*/][3], - fiat_p256_felem out[3]) { - OPENSSL_memset(out, 0, sizeof(fiat_p256_felem) * 3); - for (size_t i = 0; i < size; i++) { - fiat_p256_limb_t mismatch = i ^ idx; - fiat_p256_cmovznz(out[0], mismatch, pre_comp[i][0], out[0]); - fiat_p256_cmovznz(out[1], mismatch, pre_comp[i][1], out[1]); - fiat_p256_cmovznz(out[2], mismatch, pre_comp[i][2], out[2]); + +// Arithmetic modulo curve order + + +static void p256_order_inv0(const EC_GROUP *group, EC_SCALAR *out, + const EC_SCALAR *in) { + // table[i] stores a power of `in` corresponding to the matching enum value. + enum { + // The following indices specify the power in binary. + i_1 = 0, + i_10, + i_11, + i_101, + i_111, + i_1010, + i_1111, + i_10101, + i_101010, + i_101111, + // The following indices specify 2^N-1, or N ones in a row. + i_x6, + i_x8, + i_x16, + i_x32 + }; + BN_ULONG table[15][P256_LIMBS]; + + // https://briansmith.org/ecc-inversion-addition-chains-01#p256_scalar_inversion + // This code uses specialized field arithmetic and saves 12 sqr and 13 mul. + + // Pre-calculate powers. + OPENSSL_memcpy(table[i_1], in->words, P256_LIMBS * sizeof(BN_ULONG)); + + p256_order_sqr(group, table[i_10], table[i_1], 1); + + p256_order_mul(group, table[i_11], table[i_1], table[i_10]); + + p256_order_mul(group, table[i_101], table[i_11], table[i_10]); + + p256_order_mul(group, table[i_111], table[i_101], table[i_10]); + + p256_order_sqr(group, table[i_1010], table[i_101], 1); + + p256_order_mul(group, table[i_1111], table[i_1010], table[i_101]); + + p256_order_sqr(group, table[i_10101], table[i_1010], 1); + p256_order_mul(group, table[i_10101], table[i_10101], table[i_1]); + + p256_order_sqr(group, table[i_101010], table[i_10101], 1); + + p256_order_mul(group, table[i_101111], table[i_101010], table[i_101]); + + p256_order_mul(group, table[i_x6], table[i_101010], table[i_10101]); + + p256_order_sqr(group, table[i_x8], table[i_x6], 2); + p256_order_mul(group, table[i_x8], table[i_x8], table[i_11]); + + p256_order_sqr(group, table[i_x16], table[i_x8], 8); + p256_order_mul(group, table[i_x16], table[i_x16], table[i_x8]); + + p256_order_sqr(group, table[i_x32], table[i_x16], 16); + p256_order_mul(group, table[i_x32], table[i_x32], table[i_x16]); + + // Compute `in` raised to the order-2. + p256_order_sqr(group, out->words, table[i_x32], 64); + p256_order_mul(group, out->words, out->words, table[i_x32]); + static const struct { + uint8_t p, i; + } kChain[27] = {{32, i_x32}, {6, i_101111}, {5, i_111}, {4, i_11}, + {5, i_1111}, {5, i_10101}, {4, i_101}, {3, i_101}, + {3, i_101}, {5, i_111}, {9, i_101111}, {6, i_1111}, + {2, i_1}, {5, i_1}, {6, i_1111}, {5, i_111}, + {4, i_111}, {5, i_111}, {5, i_101}, {3, i_11}, + {10, i_101111}, {2, i_11}, {5, i_11}, {5, i_11}, + {3, i_1}, {7, i_10101}, {6, i_1111}}; + for (const auto &step : kChain) { + p256_order_sqr(group, out->words, out->words, step.p); + p256_order_mul(group, out->words, out->words, table[step.i]); } } -// fiat_p256_get_bit returns the `i`th bit in `in`. -static crypto_word_t fiat_p256_get_bit(const EC_SCALAR *in, int i) { - if (i < 0 || i >= 256) { +static int p256_order_mont_inv_vartime(const EC_GROUP *group, EC_SCALAR *out, + const EC_SCALAR *in) { +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) +#if defined(OPENSSL_X86_64) + if (!CRYPTO_is_AVX_capable()) { + // No AVX support; fallback to generic code. + return ec_simple_scalar_to_montgomery_inv_vartime(group, out, in); + } +#endif + + assert(group->order.N.width == P256_LIMBS); + if (!beeu_mod_inverse_vartime(out->words, in->words, group->order.N.d)) { return 0; } -#if defined(OPENSSL_64_BIT) - static_assert(sizeof(BN_ULONG) == 8, "BN_ULONG was not 64-bit"); - return (in->words[i >> 6] >> (i & 63)) & 1; + + // The result should be returned in the Montgomery domain. + ec_scalar_to_montgomery(group, out, out); + return 1; #else - static_assert(sizeof(BN_ULONG) == 4, "BN_ULONG was not 32-bit"); - return (in->words[i >> 5] >> (i & 31)) & 1; + return ec_simple_scalar_to_montgomery_inv_vartime(group, out, in); #endif } -// OPENSSL EC_METHOD FUNCTIONS + +// OPENSSL EC_METHOD WRAPPERS + + +static void fiat_p256_from_generic(fiat_p256_felem out, const EC_FELEM *in) { + OPENSSL_memcpy(out, in->words, 32); +} + +static void fiat_p256_to_generic(EC_FELEM *out, const fiat_p256_felem in) { + OPENSSL_memcpy(out->words, in, 32); +} // Takes the Jacobian coordinates (X, Y, Z) of a point and returns (X', Y') = // (X/Z^2, Y/Z^3). @@ -267,252 +560,22 @@ if (x_out != nullptr) { fiat_p256_felem x; fiat_p256_from_generic(x, &point->X); - fiat_p256_mul(x, x, z2); + p256_coord_mul(x, x, z2); fiat_p256_to_generic(x_out, x); } if (y_out != nullptr) { fiat_p256_felem y; fiat_p256_from_generic(y, &point->Y); - fiat_p256_square(z2, z2); // z^-4 - fiat_p256_mul(y, y, z1); // y * z - fiat_p256_mul(y, y, z2); // y * z^-3 + p256_coord_sqr(z2, z2); // z^-4 + p256_coord_mul(y, y, z1); // y * z + p256_coord_mul(y, y, z2); // y * z^-3 fiat_p256_to_generic(y_out, y); } return 1; } -static void ec_GFp_nistp256_add(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *a, const EC_JACOBIAN *b) { - fiat_p256_felem x1, y1, z1, x2, y2, z2; - fiat_p256_from_generic(x1, &a->X); - fiat_p256_from_generic(y1, &a->Y); - fiat_p256_from_generic(z1, &a->Z); - fiat_p256_from_generic(x2, &b->X); - fiat_p256_from_generic(y2, &b->Y); - fiat_p256_from_generic(z2, &b->Z); - fiat_p256_point_add(x1, y1, z1, x1, y1, z1, x2, y2, z2); - fiat_p256_to_generic(&r->X, x1); - fiat_p256_to_generic(&r->Y, y1); - fiat_p256_to_generic(&r->Z, z1); -} - -static void ec_GFp_nistp256_dbl(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *a) { - fiat_p256_felem x, y, z; - fiat_p256_from_generic(x, &a->X); - fiat_p256_from_generic(y, &a->Y); - fiat_p256_from_generic(z, &a->Z); - fiat_p256_point_double(x, y, z, x, y, z); - fiat_p256_to_generic(&r->X, x); - fiat_p256_to_generic(&r->Y, y); - fiat_p256_to_generic(&r->Z, z); -} - -static void ec_GFp_nistp256_point_mul(const EC_GROUP *group, EC_JACOBIAN *r, - const EC_JACOBIAN *p, - const EC_SCALAR *scalar) { - fiat_p256_felem p_pre_comp[17][3]; - OPENSSL_memset(&p_pre_comp, 0, sizeof(p_pre_comp)); - // Precompute multiples. - fiat_p256_from_generic(p_pre_comp[1][0], &p->X); - fiat_p256_from_generic(p_pre_comp[1][1], &p->Y); - fiat_p256_from_generic(p_pre_comp[1][2], &p->Z); - for (size_t j = 2; j <= 16; ++j) { - if (j & 1) { - fiat_p256_point_add(p_pre_comp[j][0], p_pre_comp[j][1], p_pre_comp[j][2], - p_pre_comp[1][0], p_pre_comp[1][1], p_pre_comp[1][2], - p_pre_comp[j - 1][0], p_pre_comp[j - 1][1], - p_pre_comp[j - 1][2]); - } else { - fiat_p256_point_double(p_pre_comp[j][0], p_pre_comp[j][1], - p_pre_comp[j][2], p_pre_comp[j / 2][0], - p_pre_comp[j / 2][1], p_pre_comp[j / 2][2]); - } - } - - // Set nq to the point at infinity. - fiat_p256_felem nq[3] = {{0}, {0}, {0}}, ftmp, tmp[3]; - - // Loop over `scalar` msb-to-lsb, incorporating `p_pre_comp` every 5th round. - int skip = 1; // Save two point operations in the first round. - for (size_t i = 255; i < 256; i--) { - // double - if (!skip) { - fiat_p256_point_double(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2]); - } - - // do other additions every 5 doublings - if (i % 5 == 0) { - crypto_word_t bits = fiat_p256_get_bit(scalar, i + 4) << 5; - bits |= fiat_p256_get_bit(scalar, i + 3) << 4; - bits |= fiat_p256_get_bit(scalar, i + 2) << 3; - bits |= fiat_p256_get_bit(scalar, i + 1) << 2; - bits |= fiat_p256_get_bit(scalar, i) << 1; - bits |= fiat_p256_get_bit(scalar, i - 1); - crypto_word_t sign, digit; - ec_GFp_nistp_recode_scalar_bits(&sign, &digit, bits); - - // select the point to add or subtract, in constant time. - fiat_p256_select_point((fiat_p256_limb_t)digit, 17, - (const fiat_p256_felem(*)[3])p_pre_comp, tmp); - fiat_p256_opp(ftmp, tmp[1]); // (X, -Y, Z) is the negative point. - fiat_p256_cmovznz(tmp[1], (fiat_p256_limb_t)sign, tmp[1], ftmp); - - if (!skip) { - fiat_p256_point_add(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2], tmp[0], - tmp[1], tmp[2]); - } else { - fiat_p256_copy(nq[0], tmp[0]); - fiat_p256_copy(nq[1], tmp[1]); - fiat_p256_copy(nq[2], tmp[2]); - skip = 0; - } - } - } - - fiat_p256_to_generic(&r->X, nq[0]); - fiat_p256_to_generic(&r->Y, nq[1]); - fiat_p256_to_generic(&r->Z, nq[2]); -} - -static void ec_GFp_nistp256_point_mul_base(const EC_GROUP *group, - EC_JACOBIAN *r, - const EC_SCALAR *scalar) { - // Set nq to the point at infinity. - fiat_p256_felem nq[3] = {{0}, {0}, {0}}, tmp[3]; - - int skip = 1; // Save two point operations in the first round. - for (size_t i = 31; i < 32; i--) { - if (!skip) { - fiat_p256_point_double(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2]); - } - - // First, look 32 bits upwards. - crypto_word_t bits = fiat_p256_get_bit(scalar, i + 224) << 3; - bits |= fiat_p256_get_bit(scalar, i + 160) << 2; - bits |= fiat_p256_get_bit(scalar, i + 96) << 1; - bits |= fiat_p256_get_bit(scalar, i + 32); - // Select the point to add, in constant time. - fiat_p256_select_point_affine((fiat_p256_limb_t)bits, 15, - fiat_p256_g_pre_comp[1], tmp); - - if (!skip) { - fiat_p256_point_add(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2], tmp[0], - tmp[1], tmp[2]); - } else { - fiat_p256_copy(nq[0], tmp[0]); - fiat_p256_copy(nq[1], tmp[1]); - fiat_p256_copy(nq[2], tmp[2]); - skip = 0; - } - - // Second, look at the current position. - bits = fiat_p256_get_bit(scalar, i + 192) << 3; - bits |= fiat_p256_get_bit(scalar, i + 128) << 2; - bits |= fiat_p256_get_bit(scalar, i + 64) << 1; - bits |= fiat_p256_get_bit(scalar, i); - // Select the point to add, in constant time. - fiat_p256_select_point_affine((fiat_p256_limb_t)bits, 15, - fiat_p256_g_pre_comp[0], tmp); - fiat_p256_point_add(nq[0], nq[1], nq[2], nq[0], nq[1], nq[2], tmp[0], - tmp[1], tmp[2]); - } - - fiat_p256_to_generic(&r->X, nq[0]); - fiat_p256_to_generic(&r->Y, nq[1]); - fiat_p256_to_generic(&r->Z, nq[2]); -} - -static void ec_GFp_nistp256_point_mul_public(const EC_GROUP *group, - EC_JACOBIAN *r, - const EC_SCALAR *g_scalar, - const EC_JACOBIAN *p, - const EC_SCALAR *p_scalar) { -#define P256_WSIZE_PUBLIC 4 - // Precompute multiples of `p`. p_pre_comp[i] is (2*i+1) * `p`. - fiat_p256_felem p_pre_comp[1 << (P256_WSIZE_PUBLIC - 1)][3]; - fiat_p256_from_generic(p_pre_comp[0][0], &p->X); - fiat_p256_from_generic(p_pre_comp[0][1], &p->Y); - fiat_p256_from_generic(p_pre_comp[0][2], &p->Z); - fiat_p256_felem p2[3]; - fiat_p256_point_double(p2[0], p2[1], p2[2], p_pre_comp[0][0], - p_pre_comp[0][1], p_pre_comp[0][2]); - for (size_t i = 1; i < std::size(p_pre_comp); i++) { - fiat_p256_point_add(p_pre_comp[i][0], p_pre_comp[i][1], p_pre_comp[i][2], - p_pre_comp[i - 1][0], p_pre_comp[i - 1][1], - p_pre_comp[i - 1][2], p2[0], p2[1], p2[2]); - } - - // Set up the coefficients for `p_scalar`. - int8_t p_wNAF[257]; - ec_compute_wNAF(group, p_wNAF, p_scalar, 256, P256_WSIZE_PUBLIC); - - // Set `ret` to the point at infinity. - int skip = 1; // Save some point operations. - fiat_p256_felem ret[3] = {{0}, {0}, {0}}; - for (int i = 256; i >= 0; i--) { - if (!skip) { - fiat_p256_point_double(ret[0], ret[1], ret[2], ret[0], ret[1], ret[2]); - } - - // For the `g_scalar`, we use the precomputed table without the - // constant-time lookup. - if (i <= 31) { - // First, look 32 bits upwards. - crypto_word_t bits = fiat_p256_get_bit(g_scalar, i + 224) << 3; - bits |= fiat_p256_get_bit(g_scalar, i + 160) << 2; - bits |= fiat_p256_get_bit(g_scalar, i + 96) << 1; - bits |= fiat_p256_get_bit(g_scalar, i + 32); - if (bits != 0) { - size_t index = (size_t)(bits - 1); - fiat_p256_point_add(ret[0], ret[1], ret[2], ret[0], ret[1], ret[2], - fiat_p256_g_pre_comp[1][index][0], - fiat_p256_g_pre_comp[1][index][1], fiat_p256_one); - skip = 0; - } - - // Second, look at the current position. - bits = fiat_p256_get_bit(g_scalar, i + 192) << 3; - bits |= fiat_p256_get_bit(g_scalar, i + 128) << 2; - bits |= fiat_p256_get_bit(g_scalar, i + 64) << 1; - bits |= fiat_p256_get_bit(g_scalar, i); - if (bits != 0) { - size_t index = (size_t)(bits - 1); - fiat_p256_point_add(ret[0], ret[1], ret[2], ret[0], ret[1], ret[2], - fiat_p256_g_pre_comp[0][index][0], - fiat_p256_g_pre_comp[0][index][1], fiat_p256_one); - skip = 0; - } - } - - int digit = p_wNAF[i]; - if (digit != 0) { - assert(digit & 1); - size_t idx = (size_t)(digit < 0 ? (-digit) >> 1 : digit >> 1); - fiat_p256_felem *y = &p_pre_comp[idx][1], tmp; - if (digit < 0) { - fiat_p256_opp(tmp, p_pre_comp[idx][1]); - y = &tmp; - } - if (!skip) { - fiat_p256_point_add(ret[0], ret[1], ret[2], ret[0], ret[1], ret[2], - p_pre_comp[idx][0], *y, p_pre_comp[idx][2]); - } else { - fiat_p256_copy(ret[0], p_pre_comp[idx][0]); - fiat_p256_copy(ret[1], *y); - fiat_p256_copy(ret[2], p_pre_comp[idx][2]); - skip = 0; - } - } - } - - fiat_p256_to_generic(&r->X, ret[0]); - fiat_p256_to_generic(&r->Y, ret[1]); - fiat_p256_to_generic(&r->Z, ret[2]); -} - static int ec_GFp_nistp256_cmp_x_coordinate(const EC_GROUP *group, const EC_JACOBIAN *p, const EC_SCALAR *r) { @@ -525,11 +588,11 @@ // not. fiat_p256_felem Z2_mont; fiat_p256_from_generic(Z2_mont, &p->Z); - fiat_p256_mul(Z2_mont, Z2_mont, Z2_mont); + p256_coord_mul(Z2_mont, Z2_mont, Z2_mont); fiat_p256_felem r_Z2; - fiat_p256_from_words(r_Z2, r->words); // r < order < p, so this is valid. - fiat_p256_mul(r_Z2, r_Z2, Z2_mont); + OPENSSL_memcpy(r_Z2, r->words, 32); // r < order < p, so this is valid. + p256_coord_mul(r_Z2, r_Z2, Z2_mont); fiat_p256_felem X; fiat_p256_from_generic(X, &p->X); @@ -550,7 +613,7 @@ if (carry == 0 && bn_less_than_words(tmp.words, group->field.N.d, group->field.N.width)) { fiat_p256_from_generic(r_Z2, &tmp); - fiat_p256_mul(r_Z2, r_Z2, Z2_mont); + p256_coord_mul(r_Z2, r_Z2, Z2_mont); if (OPENSSL_memcmp(&r_Z2, &X, sizeof(r_Z2)) == 0) { return 1; } @@ -559,6 +622,56 @@ return 0; } +static void ec_GFp_nistp256_point_mul(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *p, + const EC_SCALAR *scalar) { + fiat_p256_felem t[3]; + fiat_p256_from_generic(t[0], &p->X); + fiat_p256_from_generic(t[1], &p->Y); + fiat_p256_from_generic(t[2], &p->Z); + p256_point_mul(t, t, (uint8_t *)scalar->words); + + fiat_p256_to_generic(&r->X, t[0]); + fiat_p256_to_generic(&r->Y, t[1]); + fiat_p256_to_generic(&r->Z, t[2]); +} + +static void ec_GFp_nistp256_point_mul_base(const EC_GROUP *group, + EC_JACOBIAN *r, const EC_SCALAR *s) { + alignas(32) fiat_p256_felem ret[3]; + p256_point_mul_base(ret, (uint8_t *)s->words); + fiat_p256_to_generic(&r->X, ret[0]); + fiat_p256_to_generic(&r->Y, ret[1]); + fiat_p256_to_generic(&r->Z, ret[2]); +} + +static void ec_GFp_nistp256_add(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *a, const EC_JACOBIAN *b) { + fiat_p256_felem p[3], q[3]; + fiat_p256_from_generic(p[0], &a->X); + fiat_p256_from_generic(p[1], &a->Y); + fiat_p256_from_generic(p[2], &a->Z); + fiat_p256_from_generic(q[0], &b->X); + fiat_p256_from_generic(q[1], &b->Y); + fiat_p256_from_generic(q[2], &b->Z); + p256_point_add_vartime_if_doubling((uintptr_t)p, (uintptr_t)p, (uintptr_t)q); + fiat_p256_to_generic(&r->X, p[0]); + fiat_p256_to_generic(&r->Y, p[1]); + fiat_p256_to_generic(&r->Z, p[2]); +} + +static void ec_GFp_nistp256_dbl(const EC_GROUP *group, EC_JACOBIAN *r, + const EC_JACOBIAN *a) { + fiat_p256_felem p[3]; + fiat_p256_from_generic(p[0], &a->X); + fiat_p256_from_generic(p[1], &a->Y); + fiat_p256_from_generic(p[2], &a->Z); + p256_point_double((uintptr_t)p, (uintptr_t)p); + fiat_p256_to_generic(&r->X, p[0]); + fiat_p256_to_generic(&r->Y, p[1]); + fiat_p256_to_generic(&r->Z, p[2]); +} + BSSL_NAMESPACE_BEGIN DEFINE_METHOD_FUNCTION(EC_METHOD, EC_GFp_nistp256_method) { @@ -569,9 +682,8 @@ out->mul = ec_GFp_nistp256_point_mul; out->mul_base = ec_GFp_nistp256_point_mul_base; out->mul_public = ec_GFp_nistp256_point_mul_public; - out->scalar_inv0_montgomery = ec_simple_scalar_inv0_montgomery; - out->scalar_to_montgomery_inv_vartime = - ec_simple_scalar_to_montgomery_inv_vartime; + out->scalar_inv0_montgomery = p256_order_inv0; + out->scalar_to_montgomery_inv_vartime = p256_order_mont_inv_vartime; out->cmp_x_coordinate = ec_GFp_nistp256_cmp_x_coordinate; }
diff --git a/crypto/fipsmodule/ec/p256_internal.h b/crypto/fipsmodule/ec/p256_internal.h new file mode 100644 index 0000000..a320e0c --- /dev/null +++ b/crypto/fipsmodule/ec/p256_internal.h
@@ -0,0 +1,186 @@ +/* Copyright 2026 The BoringSSL Authors + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ + +#ifndef OPENSSL_HEADER_EC_P256_INTERNAL_H +#define OPENSSL_HEADER_EC_P256_INTERNAL_H + +#include <openssl/base.h> +#include <openssl/bn.h> + +#include "../../internal.h" +#include "../../../third_party/fiat/bedrock_unverified_platform.c.inc" + +#define P256_LIMBS (32 / sizeof(bssl::crypto_word_t)) +typedef bssl::crypto_word_t fiat_p256_felem[P256_LIMBS]; + + +// Choose implementation of arithmetic in the coordinate field. + +#if defined(BORINGSSL_HAS_UINT128) +#include "../../../third_party/fiat/p256_field_64.br.c.inc" + +#include "../../../third_party/fiat/p256_64.h" +#elif defined(OPENSSL_64_BIT) +#include "../../../third_party/fiat/p256_field_64.br.c.inc" + +#include "../../../third_party/fiat/p256_64_msvc.h" +#else +#include "../../../third_party/fiat/p256_field_32.br.c.inc" + +#include "../../../third_party/fiat/p256_32.h" +// Add Bedrock versions of p256_32.h functions for p256_point.br.c.inc to call. +static inline void p256_coord_add(br_word_t out, br_word_t x, br_word_t y) { + fiat_p256_add((uint32_t *)out, (const uint32_t *)x, (const uint32_t *)y); +} +static inline void p256_coord_sub(br_word_t out, br_word_t x, br_word_t y) { + fiat_p256_sub((uint32_t *)out, (const uint32_t *)x, (const uint32_t *)y); +} +#endif + +extern "C" { +#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(OPENSSL_X86_64) +// These functions are only available with gas and SysV ABI, so limit to +// __GNUC__. +void fiat_p256_adx_mul(uint64_t x0[4], const uint64_t x1[4], + const uint64_t x2[4]); +void fiat_p256_adx_sqr(uint64_t x0[4], const uint64_t x1[4]); +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) +void ecp_nistz256_mul_mont(uint64_t pr[4], const uint64_t py[4], uint64_t y0, + uint64_t x0, uint64_t x1, uint64_t x2, uint64_t x3); +void ecp_nistz256_sqr_mont(uint64_t pr[4], const uint64_t py[4], uint64_t y0, + uint64_t x0, uint64_t x1, uint64_t x2, uint64_t x3); +#endif +} + +static inline void p256_coord_mul(fiat_p256_felem out, const fiat_p256_felem x, + const fiat_p256_felem y) { +#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(OPENSSL_X86_64) + if (bssl::CRYPTO_is_BMI1_capable() && bssl::CRYPTO_is_BMI2_capable() && + bssl::CRYPTO_is_ADX_capable()) { + return fiat_p256_adx_mul(out, x, y); + } + fiat_p256_mul(out, x, y); +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) + ecp_nistz256_mul_mont(out, y, y[0], x[0], x[1], x[2], x[3]); +#else + fiat_p256_mul(out, x, y); +#endif +} + +static inline void p256_coord_sqr(fiat_p256_felem out, + const fiat_p256_felem x) { +#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(OPENSSL_X86_64) + if (bssl::CRYPTO_is_BMI1_capable() && bssl::CRYPTO_is_BMI2_capable() && + bssl::CRYPTO_is_ADX_capable()) { + return fiat_p256_adx_sqr(out, x); + } + fiat_p256_square(out, x); +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) + ecp_nistz256_sqr_mont(out, x, x[0], x[0], x[1], x[2], x[3]); +#else + fiat_p256_square(out, x); +#endif +} + +// Add Bedrock versions of these functions for p256_point.br.c.inc to call. +static inline void p256_coord_mul(br_word_t out, br_word_t x, br_word_t y) { + p256_coord_mul((br_word_t *)out, (const br_word_t *)x, (const br_word_t *)y); +} + +static inline void p256_coord_sqr(br_word_t out, br_word_t x) { + p256_coord_sqr((br_word_t *)out, (const br_word_t *)x); +} + + +// Choose implementation of arithmetic in the field modulo curve order. + +extern "C" { + +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) +// beeu_mod_inverse_vartime sets out = a^-1 mod p using a Euclidean algorithm. +// Assumption: 0 < a < p < 2^(256) and p is odd. +int beeu_mod_inverse_vartime(BN_ULONG out[4], const BN_ULONG a[4], + const BN_ULONG p[4]); +#endif + +// P-256 scalar operations. +// +// The following functions compute modulo N, where N is the order of P-256. They +// take fully-reduced inputs and give fully-reduced outputs. + +#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) +// ecp_nistz256_ord_mul_mont sets `res` to `a` * `b` where inputs and outputs +// are in Montgomery form. That is, `res` is `a` * `b` * 2^-256 mod N. +void ecp_nistz256_ord_mul_mont(BN_ULONG res[4], const BN_ULONG a[4], + const BN_ULONG b[4]); + +// ecp_nistz256_ord_sqr_mont sets `res` to `a`^(2*`rep`) where inputs and +// outputs are in Montgomery form. That is, `res` is +// (`a` * 2^-256)^(2*`rep`) * 2^256 mod N. +void ecp_nistz256_ord_sqr_mont(BN_ULONG res[4], const BN_ULONG a[4], + BN_ULONG rep); + +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) +void ecp_nistz256_ord_mul_mont_nohw(BN_ULONG res[4], const BN_ULONG a[4], + const BN_ULONG b[4]); +void ecp_nistz256_ord_mul_mont_adx(BN_ULONG res[4], const BN_ULONG a[4], + const BN_ULONG b[4]); + +void ecp_nistz256_ord_sqr_mont_nohw(BN_ULONG res[4], const BN_ULONG a[4], + BN_ULONG rep); +void ecp_nistz256_ord_sqr_mont_adx(BN_ULONG res[4], const BN_ULONG a[4], + BN_ULONG rep); +#endif + +} // extern C + +static inline void p256_order_mul(const EC_GROUP *group, + BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], + const BN_ULONG b[P256_LIMBS]) { +#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) + return ecp_nistz256_ord_mul_mont(res, a, b); +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) + if (bssl::CRYPTO_is_BMI2_capable() && bssl::CRYPTO_is_ADX_capable()) { + return ecp_nistz256_ord_mul_mont_adx(res, a, b); + } + return ecp_nistz256_ord_mul_mont_nohw(res, a, b); +#else + const BIGNUM *order = &group->order.N; + bssl::bn_mod_mul_montgomery_small(res, a, b, order->width, &group->order); +#endif +} + +static inline void p256_order_sqr(const EC_GROUP *group, + BN_ULONG res[P256_LIMBS], + const BN_ULONG a[P256_LIMBS], BN_ULONG rep) { +#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) + return ecp_nistz256_ord_sqr_mont(res, a, rep); +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) + if (bssl::CRYPTO_is_BMI2_capable() && bssl::CRYPTO_is_ADX_capable()) { + return ecp_nistz256_ord_sqr_mont_adx(res, a, rep); + } + return ecp_nistz256_ord_sqr_mont_nohw(res, a, rep); +#else + bssl::OPENSSL_memmove(res, a, sizeof(BN_ULONG) * P256_LIMBS); + for (BN_ULONG i = 0; i < rep; i++) { + p256_order_mul(group, res, res, res); + } +#endif +} + +#include "../../../third_party/fiat/p256_point.br.c.inc" + +#endif // OPENSSL_HEADER_EC_P256_INTERNAL_H
diff --git a/crypto/fipsmodule/ec/p256_test.cc b/crypto/fipsmodule/ec/p256_test.cc index a8039c3..3439d1b 100644 --- a/crypto/fipsmodule/ec/p256_test.cc +++ b/crypto/fipsmodule/ec/p256_test.cc
@@ -12,21 +12,30 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include <gtest/gtest.h> +#if !defined(BORINGSSL_SHARED_LIBRARY) // These tests use p256_internal.h + +#include <openssl/base.h> + +#include <openssl/bn.h> +#include <openssl/ec.h> +#include <openssl/mem.h> +#include <openssl/nid.h> + +#include "internal.h" + #include "../../internal.h" #include "../../test/abi_test.h" +#include "../../test/file_test.h" +#include "p256_internal.h" BSSL_NAMESPACE_BEGIN namespace { -#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(__x86_64__) && \ - defined(SUPPORTS_ABI_TEST) -extern "C" { -#include "../../../third_party/fiat/p256_field.c.inc" -} +#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && \ + defined(OPENSSL_X86_64) && defined(SUPPORTS_ABI_TEST) -TEST(P256Test, AdxMulABI) { +TEST(P256Test, CoordMulAdxABI) { static const uint64_t in1[4] = {0}, in2[4] = {0}; uint64_t out[4]; if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() && @@ -37,8 +46,7 @@ } } -#include <assert.h> -TEST(P256Test, AdxSquareABI) { +TEST(P256Test, CoordSqrAdxABI) { static const uint64_t in[4] = {0}; uint64_t out[4]; if (CRYPTO_is_BMI1_capable() && CRYPTO_is_BMI2_capable() && @@ -48,7 +56,506 @@ GTEST_SKIP() << "Can't test ABI of ADX code without ADX"; } } + #endif +#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && \ + defined(SUPPORTS_ABI_TEST) + +TEST(P256Test, CoordMulABI) { + static const uint64_t x[4] = {0}, y[4] = {0}; + uint64_t out[4]; + CHECK_ABI(ecp_nistz256_mul_mont, out, y, y[0], x[0], x[1], x[2], x[3]); +} + +TEST(P256Test, CoordSqrABI) { + static const uint64_t x[4] = {0}; + uint64_t out[4]; + CHECK_ABI(ecp_nistz256_sqr_mont, out, x, x[0], x[0], x[1], x[2], x[3]); +} + +#endif + +#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && \ + defined(SUPPORTS_ABI_TEST) + +TEST(P256Test, OrdMulAdxABI) { + static const uint64_t in1[4] = {0}, in2[4] = {0}; + uint64_t out[4]; + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + CHECK_ABI(ecp_nistz256_ord_mul_mont_adx, out, in1, in2); + } else { + GTEST_SKIP() << "Can't test ABI of ADX code without ADX"; + } +} + +TEST(P256Test, OrdSqrAdxABI) { + static const uint64_t in[4] = {0}; + uint64_t out[4]; + if (CRYPTO_is_BMI2_capable() && CRYPTO_is_ADX_capable()) { + CHECK_ABI(ecp_nistz256_ord_sqr_mont_adx, out, in, 1); + } else { + GTEST_SKIP() << "Can't test ABI of ADX code without ADX"; + } +} + +TEST(P256Test, OrdMulNohwABI) { + static const uint64_t in1[4] = {0}, in2[4] = {0}; + uint64_t out[4]; + CHECK_ABI(ecp_nistz256_ord_mul_mont_nohw, out, in1, in2); +} + +TEST(P256Test, OrdSqrNohwABI) { + static const uint64_t in[4] = {0}; + uint64_t out[4]; + CHECK_ABI(ecp_nistz256_ord_sqr_mont_nohw, out, in, 1); +} + +#endif + +typedef fiat_p256_felem P256_POINT_AFFINE[2]; +typedef fiat_p256_felem P256_POINT[3]; + +static inline void point_add(P256_POINT ret, P256_POINT p, P256_POINT q) { + p256_point_add_vartime_if_doubling((uintptr_t)ret, (uintptr_t)p, + (uintptr_t)q); +} + +static inline void point_add_affine(P256_POINT ret, P256_POINT a, + P256_POINT b_affine) { + uintptr_t bnz = 0; + bnz |= b_affine[0][0] | b_affine[0][1] | b_affine[0][2] | b_affine[0][3]; + bnz |= b_affine[1][0] | b_affine[1][1] | b_affine[1][2] | b_affine[1][3]; + p256_point_add_affinenz_conditional_vartime_if_doubling( + (uintptr_t)ret, (uintptr_t)a, (uintptr_t)b_affine, (uintptr_t)bnz); +} + +static inline void point_double(P256_POINT ret, P256_POINT a) { + p256_point_double((uintptr_t)ret, (uintptr_t)a); +} + + +static bool GetFieldElement(FileTest *t, BN_ULONG out[P256_LIMBS], + const char *name) { + std::vector<uint8_t> bytes; + if (!t->GetBytes(&bytes, name)) { + return false; + } + + if (bytes.size() != BN_BYTES * P256_LIMBS) { + ADD_FAILURE() << "Invalid length: " << name; + return false; + } + + // `byte` contains bytes in big-endian while `out` should contain `BN_ULONG`s + // in little-endian. + OPENSSL_memset(out, 0, P256_LIMBS * sizeof(BN_ULONG)); + for (size_t i = 0; i < bytes.size(); i++) { + out[P256_LIMBS - 1 - (i / BN_BYTES)] <<= 8; + out[P256_LIMBS - 1 - (i / BN_BYTES)] |= bytes[i]; + } + + return true; +} + +static std::string FieldElementToString(const BN_ULONG a[P256_LIMBS]) { + std::string ret; + for (size_t i = P256_LIMBS - 1; i < P256_LIMBS; i--) { + char buf[2 * BN_BYTES + 1]; + snprintf(buf, sizeof(buf), BN_HEX_FMT2, a[i]); + ret += buf; + } + return ret; +} + +static testing::AssertionResult ExpectFieldElementsEqual( + const char *expected_expr, const char *actual_expr, + const BN_ULONG expected[P256_LIMBS], const BN_ULONG actual[P256_LIMBS]) { + if (OPENSSL_memcmp(expected, actual, sizeof(BN_ULONG) * P256_LIMBS) == 0) { + return testing::AssertionSuccess(); + } + + return testing::AssertionFailure() + << "Expected: " << FieldElementToString(expected) << " (" + << expected_expr << ")\n" + << "Actual: " << FieldElementToString(actual) << " (" << actual_expr + << ")"; +} + +#define EXPECT_FIELD_ELEMENTS_EQUAL(a, b) \ + EXPECT_PRED_FORMAT2(ExpectFieldElementsEqual, a, b) + +// P-256 scalars and field elements are the same size. +#define EXPECT_SCALARS_EQUAL(a, b) EXPECT_FIELD_ELEMENTS_EQUAL(a, b) + +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_AARCH64)) +TEST(P256Test, BEEU) { +#if defined(OPENSSL_X86_64) + if (!CRYPTO_is_AVX_capable()) { + // No AVX support; cannot run the BEEU code. + return; + } +#endif + + const EC_GROUP *group = EC_group_p256(); + BN_ULONG order_words[P256_LIMBS]; + ASSERT_TRUE( + bn_copy_words(order_words, P256_LIMBS, EC_GROUP_get0_order(group))); + + BN_ULONG in[P256_LIMBS], out[P256_LIMBS]; + EC_SCALAR in_scalar, out_scalar, result; + OPENSSL_memset(in, 0, sizeof(in)); + + // Trying to find the inverse of zero should fail. + ASSERT_FALSE(beeu_mod_inverse_vartime(out, in, order_words)); + // This is not a constant-time function, so instrument both zero and a few + // inputs below. + ASSERT_FALSE(CHECK_ABI(beeu_mod_inverse_vartime, out, in, order_words)); + + BN_ULONG kOne[P256_LIMBS] = {}; + kOne[0] = 1; + + auto beeu_test = [&] { + EXPECT_TRUE(bn_less_than_words(in, order_words, P256_LIMBS)); + ASSERT_TRUE(beeu_mod_inverse_vartime(out, in, order_words)); + EXPECT_TRUE(bn_less_than_words(out, order_words, P256_LIMBS)); + + // Calculate out*in and confirm that it equals one, modulo the order. We + // only have Montgomery multiplication, but if we convert one input, and not + // the other, this gives the result outside the Montgomery domain. + OPENSSL_memcpy(in_scalar.words, in, sizeof(in)); + OPENSSL_memcpy(out_scalar.words, out, sizeof(out)); + ec_scalar_to_montgomery(group, &in_scalar, &in_scalar); + ec_scalar_mul_montgomery(group, &result, &in_scalar, &out_scalar); + EXPECT_SCALARS_EQUAL(kOne, result.words); + + // Invert the result and expect to get back to the original value. + ASSERT_TRUE(beeu_mod_inverse_vartime(out, out, order_words)); + EXPECT_SCALARS_EQUAL(in, out); + }; + + for (BN_ULONG i = 1; i < 2000; i++) { + SCOPED_TRACE(i); + in[0] = i; + if (i >= 1000) { + in[1] = i << 8; + in[2] = i << 32; + in[3] = i << 48; + } else { + in[1] = in[2] = in[3] = 0; + } + + beeu_test(); + if (i < 5) { + EXPECT_TRUE(CHECK_ABI(beeu_mod_inverse_vartime, out, in, order_words)); + } + } + + for (int i = 0; i < 255; i++) { + SCOPED_TRACE(i); + // Test `in` = 2^i. + OPENSSL_memset(in, 0, sizeof(in)); + in[i / BN_BITS2] = BN_ULONG{1} << (i % BN_BITS2); + beeu_test(); + + // Test `in` = N - 2^i. + bn_sub_words(in, order_words, in, P256_LIMBS); + beeu_test(); + } +} +#endif + +static void TestNegate(FileTest *t) { + fiat_p256_felem a, b; + ASSERT_TRUE(GetFieldElement(t, a, "A")); + ASSERT_TRUE(GetFieldElement(t, b, "B")); + + // Test that -A = B. + BN_ULONG ret[P256_LIMBS]; + fiat_p256_opp(ret, a); + EXPECT_FIELD_ELEMENTS_EQUAL(b, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + fiat_p256_opp(ret, ret /* a */); + EXPECT_FIELD_ELEMENTS_EQUAL(b, ret); + + // Test that -B = A. + fiat_p256_opp(ret, b); + EXPECT_FIELD_ELEMENTS_EQUAL(a, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + fiat_p256_opp(ret, ret /* b */); + EXPECT_FIELD_ELEMENTS_EQUAL(a, ret); +} + +static void TestMulMont(FileTest *t) { + BN_ULONG a[P256_LIMBS], b[P256_LIMBS], result[P256_LIMBS]; + ASSERT_TRUE(GetFieldElement(t, a, "A")); + ASSERT_TRUE(GetFieldElement(t, b, "B")); + ASSERT_TRUE(GetFieldElement(t, result, "Result")); + + BN_ULONG ret[P256_LIMBS]; + p256_coord_mul(ret, a, b); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + p256_coord_mul(ret, b, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + p256_coord_mul(ret, ret /* a */, b); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + p256_coord_mul(ret, b, ret); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + p256_coord_mul(ret, a, ret /* b */); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + p256_coord_mul(ret, ret /* b */, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { + p256_coord_sqr(ret, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + p256_coord_sqr(ret, ret /* a */); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + } +} + +static void TestFromMont(FileTest *t) { + BN_ULONG a[P256_LIMBS], result[P256_LIMBS]; + ASSERT_TRUE(GetFieldElement(t, a, "A")); + ASSERT_TRUE(GetFieldElement(t, result, "Result")); + + BN_ULONG ret[P256_LIMBS]; + fiat_p256_from_montgomery(ret, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + fiat_p256_from_montgomery(ret, ret /* a */); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); +} + +static void TestOrdMulMont(FileTest *t) { + // This test works on scalars rather than field elements, but the + // representation is the same. + BN_ULONG a[P256_LIMBS], b[P256_LIMBS], result[P256_LIMBS]; + ASSERT_TRUE(GetFieldElement(t, a, "A")); + ASSERT_TRUE(GetFieldElement(t, b, "B")); + ASSERT_TRUE(GetFieldElement(t, result, "Result")); + SCOPED_TRACE(FieldElementToString(a)); + SCOPED_TRACE(FieldElementToString(b)); + + const EC_GROUP *group = EC_group_p256(); + BN_ULONG ret[P256_LIMBS]; + p256_order_mul(group, ret, a, b); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + p256_order_mul(group, ret, b, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + p256_order_mul(group, ret, ret /* a */, b); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + p256_order_mul(group, ret, b, ret); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + p256_order_mul(group, ret, a, ret /* b */); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + p256_order_mul(group, ret, ret /* b */, a); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { + p256_order_sqr(group, ret, a, 1); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + p256_order_sqr(group, ret, ret /* a */, 1); + EXPECT_FIELD_ELEMENTS_EQUAL(result, ret); + } +} + +static bool PointToAffine(P256_POINT_AFFINE out, const P256_POINT in) { + static const uint8_t kP[] = { + 0xff, 0xff, 0xff, 0xff, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + }; + + UniquePtr<BIGNUM> x(BN_new()), y(BN_new()), z(BN_new()); + UniquePtr<BIGNUM> p(BN_bin2bn(kP, sizeof(kP), nullptr)); + if (!x || !y || !z || !p || !bn_set_words(x.get(), in[0], P256_LIMBS) || + !bn_set_words(y.get(), in[1], P256_LIMBS) || + !bn_set_words(z.get(), in[2], P256_LIMBS)) { + return false; + } + + // Coordinates must be fully-reduced. + if (BN_cmp(x.get(), p.get()) >= 0 || BN_cmp(y.get(), p.get()) >= 0 || + BN_cmp(z.get(), p.get()) >= 0) { + return false; + } + + if (BN_is_zero(z.get())) { + // The point at infinity is represented as (0, 0). + OPENSSL_memset(out, 0, sizeof(P256_POINT_AFFINE)); + return true; + } + + UniquePtr<BN_CTX> ctx(BN_CTX_new()); + UniquePtr<BN_MONT_CTX> mont(BN_MONT_CTX_new_for_modulus(p.get(), ctx.get())); + if (!ctx || !mont || + // Invert Z. + !BN_from_montgomery(z.get(), z.get(), mont.get(), ctx.get()) || + !BN_mod_inverse(z.get(), z.get(), p.get(), ctx.get()) || + !BN_to_montgomery(z.get(), z.get(), mont.get(), ctx.get()) || + // Convert (X, Y, Z) to (X/Z^2, Y/Z^3). + !BN_mod_mul_montgomery(x.get(), x.get(), z.get(), mont.get(), + ctx.get()) || + !BN_mod_mul_montgomery(x.get(), x.get(), z.get(), mont.get(), + ctx.get()) || + !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), + ctx.get()) || + !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), + ctx.get()) || + !BN_mod_mul_montgomery(y.get(), y.get(), z.get(), mont.get(), + ctx.get()) || + !bn_copy_words(out[0], P256_LIMBS, x.get()) || + !bn_copy_words(out[1], P256_LIMBS, y.get())) { + return false; + } + return true; +} + +static testing::AssertionResult ExpectPointsEqual( + const char *expected_expr, const char *actual_expr, + const P256_POINT_AFFINE expected, const P256_POINT actual) { + // There are multiple representations of the same `P256_POINT`, so convert to + // `P256_POINT_AFFINE` and compare. + P256_POINT_AFFINE affine; + if (!PointToAffine(affine, actual)) { + return testing::AssertionFailure() + << "Could not convert " << actual_expr << " to affine: (" + << FieldElementToString(actual[0]) << ", " + << FieldElementToString(actual[1]) << ", " + << FieldElementToString(actual[2]) << ")"; + } + + if (OPENSSL_memcmp(expected, affine, sizeof(P256_POINT_AFFINE)) != 0) { + return testing::AssertionFailure() + << "Expected: (" << FieldElementToString(expected[0]) << ", " + << FieldElementToString(expected[1]) << ") (" << expected_expr + << "; affine)\n" + << "Actual: (" << FieldElementToString(affine[0]) << ", " + << FieldElementToString(affine[1]) << ") (affine(" << actual_expr + << "))\n" + << "ActualXYZ:(" << FieldElementToString(actual[0]) << ", " + << FieldElementToString(actual[1]) << ", " + << FieldElementToString(actual[2]) << ") (" << actual_expr << ")"; + } + + return testing::AssertionSuccess(); +} + +#define EXPECT_POINTS_EQUAL(a, b) EXPECT_PRED_FORMAT2(ExpectPointsEqual, a, b) + +static void TestPointAdd(FileTest *t) { + P256_POINT a, b; + P256_POINT_AFFINE result; + ASSERT_TRUE(GetFieldElement(t, a[0], "A.X")); + ASSERT_TRUE(GetFieldElement(t, a[1], "A.Y")); + ASSERT_TRUE(GetFieldElement(t, a[2], "A.Z")); + ASSERT_TRUE(GetFieldElement(t, b[0], "B.X")); + ASSERT_TRUE(GetFieldElement(t, b[1], "B.Y")); + ASSERT_TRUE(GetFieldElement(t, b[2], "B.Z")); + ASSERT_TRUE(GetFieldElement(t, result[0], "Result.X")); + ASSERT_TRUE(GetFieldElement(t, result[1], "Result.Y")); + + P256_POINT ret; + point_add(ret, a, b); + EXPECT_POINTS_EQUAL(result, ret); + + point_add(ret, b, a); + EXPECT_POINTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + point_add(ret, ret /* a */, b); + EXPECT_POINTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + point_add(ret, b, ret /* a */); + EXPECT_POINTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + point_add(ret, a, ret /* b */); + EXPECT_POINTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + point_add(ret, ret /* b */, a); + EXPECT_POINTS_EQUAL(result, ret); + + P256_POINT a_affine, b_affine; + ASSERT_TRUE(PointToAffine(a_affine, a)); + ASSERT_TRUE(PointToAffine(b_affine, b)); + static const uint64_t p256_fe_one[4] = {0x1, 0xffffffff00000000, + 0xffffffffffffffff, 0xfffffffe}; + OPENSSL_memcpy(a_affine[2], p256_fe_one, sizeof(a_affine[2])); + OPENSSL_memcpy(b_affine[2], p256_fe_one, sizeof(b_affine[2])); + + point_add_affine(ret, a, b_affine); + EXPECT_POINTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + point_add_affine(ret, ret /* a */, b_affine); + EXPECT_POINTS_EQUAL(result, ret); + + point_add_affine(ret, b, a_affine); + EXPECT_POINTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, b, sizeof(ret)); + point_add_affine(ret, ret /* b */, a_affine); + EXPECT_POINTS_EQUAL(result, ret); + + if (OPENSSL_memcmp(a, b, sizeof(a)) == 0) { + point_double(ret, a); + EXPECT_POINTS_EQUAL(result, ret); + + OPENSSL_memcpy(ret, a, sizeof(ret)); + point_double(ret, ret /* a */); + EXPECT_POINTS_EQUAL(result, ret); + } +} + +TEST(P256Test, TestVectors) { + return FileTestGTest("crypto/fipsmodule/ec/p256-tests.txt", [](FileTest *t) { + if (t->GetParameter() == "Negate") { + TestNegate(t); + } else if (t->GetParameter() == "MulMont") { + TestMulMont(t); + } else if (t->GetParameter() == "FromMont") { + TestFromMont(t); + } else if (t->GetParameter() == "PointAdd") { + TestPointAdd(t); + } else if (t->GetParameter() == "OrdMulMont") { + TestOrdMulMont(t); + } else { + FAIL() << "Unknown test type:" << t->GetParameter(); + } + }); +} + } // namespace BSSL_NAMESPACE_END + +#endif // !defined(BORINGSSL_SHARED_LIBRARY)
diff --git a/crypto/internal.h b/crypto/internal.h index 7c835fa..dcc109a 100644 --- a/crypto/internal.h +++ b/crypto/internal.h
@@ -448,8 +448,10 @@ assert(!buffers_alias(dst, n, src, n)); uint8_t *out = (uint8_t *)dst; const uint8_t *in = (const uint8_t *)src; -#if defined(__GNUC__) && !defined(__clang__) +#if defined(__GNUC__) || defined(__clang__) // gcc 13.2.0 doesn't automatically vectorize this loop regardless of barrier + // clang 16.0.6 vectorizes if alias analysis finds src and dst are disjoint, + // but does not conclude that from the !buffers_alias assert. typedef uint8_t v32u8 __attribute__((vector_size(32), aligned(1), may_alias)); size_t n_vec = n & ~(size_t)31; v32u8 masks = ((uint8_t)mask - (v32u8){}); // broadcast
diff --git a/gen/bcm/p256-armv8-asm-apple.S b/gen/bcm/p256-armv8-asm-apple.S index 283240f..2f18db1 100644 --- a/gen/bcm/p256-armv8-asm-apple.S +++ b/gen/bcm/p256-armv8-asm-apple.S
@@ -4,26 +4,6 @@ #include <openssl/asm_base.h> #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__APPLE__) -.section __TEXT,__const -.align 5 -Lpoly: -.quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 -LRR: // 2^512 mod P precomputed for NIST P256 polynomial -.quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd -Lone_mont: -.quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe -Lone: -.quad 1,0,0,0 -Lord: -.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 -LordK: -.quad 0xccd1c8aaee00bc4f -.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 -.align 2 -.text - -// void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4], -// const BN_ULONG x2[4]); .globl _ecp_nistz256_mul_mont .private_extern _ecp_nistz256_mul_mont @@ -34,15 +14,135 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - ldr x3,[x2] // bp[0] - ldp x4,x5,[x1] - ldp x6,x7,[x1,#16] - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] + mul x13,x3,x2 // a[0]*b[0] + umulh x8,x3,x2 - bl __ecp_nistz256_mul_mont + mul x14,x4,x2 // a[1]*b[0] + umulh x9,x4,x2 + + mul x15,x5,x2 // a[2]*b[0] + umulh x10,x5,x2 + + mul x16,x6,x2 // a[3]*b[0] + umulh x11,x6,x2 + ldr x2,[x1,#8] // b[1] + + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adc x17,xzr,x11 + mov x19,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + ldr x2,[x1,#8*(1+1)] // b[1+1] + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + ldr x2,[x1,#8*(2+1)] // b[2+1] + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + mov w7, #-1 // poly1 = 0x00000000ffffffff + // last reduction + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + adcs x16,x17,x11 + adc x17,x19,xzr + + neg x12,x7 // poly3 = 0xffffffff00000001 + adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus + sbcs x9,x14,x7 + sbcs x10,x15,xzr + sbcs x11,x16,x12 + sbcs xzr,x17,xzr // did it borrow? + + csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus + csel x14,x14,x9,lo + csel x15,x15,x10,lo + stp x13,x14,[x0] + csel x16,x16,x11,lo + stp x15,x16,[x0,#16] ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 @@ -50,7 +150,6 @@ ret -// void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); .globl _ecp_nistz256_sqr_mont .private_extern _ecp_nistz256_sqr_mont @@ -61,305 +160,6 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - ldp x4,x5,[x1] - ldp x6,x7,[x1,#16] - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sqr_mont - - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl _ecp_nistz256_div_by_2 -.private_extern _ecp_nistz256_div_by_2 - -.align 4 -_ecp_nistz256_div_by_2: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_div_by_2 - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl _ecp_nistz256_mul_by_2 -.private_extern _ecp_nistz256_mul_by_2 - -.align 4 -_ecp_nistz256_mul_by_2: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - - bl __ecp_nistz256_add_to // ret = a+a // 2*a - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl _ecp_nistz256_mul_by_3 -.private_extern _ecp_nistz256_mul_by_3 - -.align 4 -_ecp_nistz256_mul_by_3: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - mov x4,x14 - mov x5,x15 - mov x6,x16 - mov x7,x17 - - bl __ecp_nistz256_add_to // ret = a+a // 2*a - - mov x8,x4 - mov x9,x5 - mov x10,x6 - mov x11,x7 - - bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4], -// const BN_ULONG x2[4]); -.globl _ecp_nistz256_sub -.private_extern _ecp_nistz256_sub - -.align 4 -_ecp_nistz256_sub: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sub_from - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl _ecp_nistz256_neg -.private_extern _ecp_nistz256_neg - -.align 4 -_ecp_nistz256_neg: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - mov x2,x1 - mov x14,xzr // a = 0 - mov x15,xzr - mov x16,xzr - mov x17,xzr - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sub_from - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded -// to x4-x7 and b[0] - to x3 - -.align 4 -__ecp_nistz256_mul_mont: - mul x14,x4,x3 // a[0]*b[0] - umulh x8,x4,x3 - - mul x15,x5,x3 // a[1]*b[0] - umulh x9,x5,x3 - - mul x16,x6,x3 // a[2]*b[0] - umulh x10,x6,x3 - - mul x17,x7,x3 // a[3]*b[0] - umulh x11,x7,x3 - ldr x3,[x2,#8] // b[1] - - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adc x19,xzr,x11 - mov x20,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - ldr x3,[x2,#8*(1+1)] // b[1+1] - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - ldr x3,[x2,#8*(2+1)] // b[2+1] - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - // last reduction - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - adcs x17,x19,x11 - adc x19,x20,xzr - - adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x19,xzr // did it borrow? - - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] - - ret - - -// note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded -// to x4-x7 - -.align 4 -__ecp_nistz256_sqr_mont: // | | | | | |a1*a0| | // | | | | |a2*a0| | | // | |a3*a2|a3*a0| | | | @@ -374,834 +174,124 @@ // multiplication result, which can't overflow, because it // can never be all ones. - mul x15,x5,x4 // a[1]*a[0] + mul x14,x4,x3 // a[1]*a[0] + umulh x9,x4,x3 + mul x15,x5,x3 // a[2]*a[0] + umulh x10,x5,x3 + mul x16,x6,x3 // a[3]*a[0] + umulh x17,x6,x3 + + adds x15,x15,x9 // accumulate high parts of multiplication + mul x8,x5,x4 // a[2]*a[1] umulh x9,x5,x4 - mul x16,x6,x4 // a[2]*a[0] - umulh x10,x6,x4 - mul x17,x7,x4 // a[3]*a[0] - umulh x19,x7,x4 + adcs x16,x16,x10 + mul x10,x6,x4 // a[3]*a[1] + umulh x11,x6,x4 + adc x17,x17,xzr // can't overflow - adds x16,x16,x9 // accumulate high parts of multiplication - mul x8,x6,x5 // a[2]*a[1] - umulh x9,x6,x5 - adcs x17,x17,x10 - mul x10,x7,x5 // a[3]*a[1] - umulh x11,x7,x5 - adc x19,x19,xzr // can't overflow - - mul x20,x7,x6 // a[3]*a[2] - umulh x1,x7,x6 + mul x19,x6,x5 // a[3]*a[2] + umulh x20,x6,x5 adds x9,x9,x10 // accumulate high parts of multiplication - mul x14,x4,x4 // a[0]*a[0] + mul x13,x3,x3 // a[0]*a[0] adc x10,x11,xzr // can't overflow - adds x17,x17,x8 // accumulate low parts of multiplication + adds x16,x16,x8 // accumulate low parts of multiplication + umulh x3,x3,x3 + adcs x17,x17,x9 + mul x9,x4,x4 // a[1]*a[1] + adcs x19,x19,x10 umulh x4,x4,x4 - adcs x19,x19,x9 - mul x9,x5,x5 // a[1]*a[1] - adcs x20,x20,x10 - umulh x5,x5,x5 - adc x1,x1,xzr // can't overflow + adc x20,x20,xzr // can't overflow - adds x15,x15,x15 // acc[1-6]*=2 - mul x10,x6,x6 // a[2]*a[2] + adds x14,x14,x14 // acc[1-6]*=2 + mul x10,x5,x5 // a[2]*a[2] + adcs x15,x15,x15 + umulh x5,x5,x5 adcs x16,x16,x16 - umulh x6,x6,x6 + mul x11,x6,x6 // a[3]*a[3] adcs x17,x17,x17 - mul x11,x7,x7 // a[3]*a[3] + umulh x6,x6,x6 adcs x19,x19,x19 - umulh x7,x7,x7 adcs x20,x20,x20 - adcs x1,x1,x1 adc x2,xzr,xzr - adds x15,x15,x4 // +a[i]*a[i] - adcs x16,x16,x9 - adcs x17,x17,x5 - adcs x19,x19,x10 - adcs x20,x20,x6 - lsl x8,x14,#32 - adcs x1,x1,x11 - lsr x9,x14,#32 - adc x2,x2,x7 - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - adc x17,x11,xzr // can't overflow - - adds x14,x14,x19 // accumulate upper half - adcs x15,x15,x20 - adcs x16,x16,x1 - adcs x17,x17,x2 - adc x19,xzr,xzr - - adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x19,xzr // did it borrow? - - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] - - ret - - -// Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to -// x4-x7 and x8-x11. This is done because it's used in multiple -// contexts, e.g. in multiplication by 2 and 3... - -.align 4 -__ecp_nistz256_add_to: - adds x14,x14,x8 // ret = a+b + adds x14,x14,x3 // +a[i]*a[i] adcs x15,x15,x9 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x1,xzr,xzr // zap x1 + adcs x16,x16,x4 + adcs x17,x17,x10 + adcs x19,x19,x5 + lsl x8,x13,#32 + adcs x20,x20,x11 + lsr x9,x13,#32 + adc x2,x2,x6 + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + adc x16,x11,xzr // can't overflow - adds x8,x14,#1 // subs x8,x4,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x1,xzr // did subtraction borrow? + mov w7, #-1 // poly1 = 0x00000000ffffffff + adds x13,x13,x17 // accumulate upper half + adcs x14,x14,x19 + adcs x15,x15,x20 + adcs x16,x16,x2 + adc x17,xzr,xzr - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] + neg x12,x7 // poly3 = 0xffffffff00000001 + adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus + sbcs x9,x14,x7 + sbcs x10,x15,xzr + sbcs x11,x16,x12 + sbcs xzr,x17,xzr // did it borrow? - ret + csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus + csel x14,x14,x9,lo + csel x15,x15,x10,lo + stp x13,x14,[x0] + csel x16,x16,x11,lo + stp x15,x16,[x0,#16] - - -.align 4 -__ecp_nistz256_sub_from: - ldp x8,x9,[x2] - ldp x10,x11,[x2,#16] - subs x14,x14,x8 // ret = a-b - sbcs x15,x15,x9 - sbcs x16,x16,x10 - sbcs x17,x17,x11 - sbc x1,xzr,xzr // zap x1 - - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adc x11,x17,x13 - cmp x1,xzr // did subtraction borrow? - - csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret - csel x15,x15,x9,eq - csel x16,x16,x10,eq - stp x14,x15,[x0] - csel x17,x17,x11,eq - stp x16,x17,[x0,#16] - - ret - - - -.align 4 -__ecp_nistz256_sub_morf: - ldp x8,x9,[x2] - ldp x10,x11,[x2,#16] - subs x14,x8,x14 // ret = b-a - sbcs x15,x9,x15 - sbcs x16,x10,x16 - sbcs x17,x11,x17 - sbc x1,xzr,xzr // zap x1 - - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adc x11,x17,x13 - cmp x1,xzr // did subtraction borrow? - - csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret - csel x15,x15,x9,eq - csel x16,x16,x10,eq - stp x14,x15,[x0] - csel x17,x17,x11,eq - stp x16,x17,[x0,#16] - - ret - - - -.align 4 -__ecp_nistz256_div_by_2: - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = a+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adcs x11,x17,x13 - adc x1,xzr,xzr // zap x1 - tst x14,#1 // is a even? - - csel x14,x14,x8,eq // ret = even ? a : a+modulus - csel x15,x15,x9,eq - csel x16,x16,x10,eq - csel x17,x17,x11,eq - csel x1,xzr,x1,eq - - lsr x14,x14,#1 // ret >>= 1 - orr x14,x14,x15,lsl#63 - lsr x15,x15,#1 - orr x15,x15,x16,lsl#63 - lsr x16,x16,#1 - orr x16,x16,x17,lsl#63 - lsr x17,x17,#1 - stp x14,x15,[x0] - orr x17,x17,x1,lsl#63 - stp x16,x17,[x0,#16] - - ret - -.globl _ecp_nistz256_point_double -.private_extern _ecp_nistz256_point_double - -.align 5 -_ecp_nistz256_point_double: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-96]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - sub sp,sp,#32*4 - -Ldouble_shortcut: - ldp x14,x15,[x1,#32] - mov x21,x0 - ldp x16,x17,[x1,#48] - mov x22,x1 - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - mov x8,x14 - ldr x13,[x13,#24] - mov x9,x15 - ldp x4,x5,[x22,#64] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[x22,#64+16] - add x0,sp,#0 - bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y); - - add x0,sp,#64 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z); - - ldp x8,x9,[x22] - ldp x10,x11,[x22,#16] - mov x4,x14 // put Zsqr aside for p256_sub - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x0,sp,#32 - bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x); - - add x2,x22,#0 - mov x14,x4 // restore Zsqr - mov x15,x5 - ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont - mov x16,x6 - mov x17,x7 - ldp x6,x7,[sp,#0+16] - add x0,sp,#64 - bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr); - - add x0,sp,#0 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S); - - ldr x3,[x22,#32] - ldp x4,x5,[x22,#64] - ldp x6,x7,[x22,#64+16] - add x2,x22,#32 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y); - - mov x8,x14 - mov x9,x15 - ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[sp,#0+16] - add x0,x21,#64 - bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0); - - add x0,sp,#96 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S); - - ldr x3,[sp,#64] // forward load for p256_mul_mont - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x0,x21,#32 - bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0); - - add x2,sp,#64 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr); - - mov x8,x14 // duplicate M - mov x9,x15 - mov x10,x16 - mov x11,x17 - mov x4,x14 // put M aside - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x0,sp,#32 - bl __ecp_nistz256_add_to - mov x8,x4 // restore M - mov x9,x5 - ldr x3,[x22] // forward load for p256_mul_mont - mov x10,x6 - ldp x4,x5,[sp,#0] - mov x11,x7 - ldp x6,x7,[sp,#0+16] - bl __ecp_nistz256_add_to // p256_mul_by_3(M, M); - - add x2,x22,#0 - add x0,sp,#0 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x); - - mov x8,x14 - mov x9,x15 - ldp x4,x5,[sp,#32] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[sp,#32+16] - add x0,sp,#96 - bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S); - - add x0,x21,#0 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M); - - add x2,sp,#96 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0); - - add x2,sp,#0 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x); - - ldr x3,[sp,#32] - mov x4,x14 // copy S - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x2,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M); - - add x2,x21,#32 - add x0,x21,#32 - bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y); - - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x29,x30,[sp],#96 + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 AARCH64_VALIDATE_LINK_REGISTER ret -.globl _ecp_nistz256_point_add -.private_extern _ecp_nistz256_point_add - +.section __TEXT,__const .align 5 -_ecp_nistz256_point_add: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-96]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - sub sp,sp,#32*12 +Lord: +.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 +LordK: +.quad 0xccd1c8aaee00bc4f +.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 +.align 2 +.text - ldp x4,x5,[x2,#64] // in2_z - ldp x6,x7,[x2,#64+16] - mov x21,x0 - mov x22,x1 - mov x23,x2 - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] - orr x8,x4,x5 - orr x10,x6,x7 - orr x25,x8,x10 - cmp x25,#0 - csetm x25,ne // ~in2infty - add x0,sp,#192 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z); - - ldp x4,x5,[x22,#64] // in1_z - ldp x6,x7,[x22,#64+16] - orr x8,x4,x5 - orr x10,x6,x7 - orr x24,x8,x10 - cmp x24,#0 - csetm x24,ne // ~in1infty - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); - - ldr x3,[x23,#64] - ldp x4,x5,[sp,#192] - ldp x6,x7,[sp,#192+16] - add x2,x23,#64 - add x0,sp,#320 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,x22,#64 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); - - ldr x3,[x22,#32] - ldp x4,x5,[sp,#320] - ldp x6,x7,[sp,#320+16] - add x2,x22,#32 - add x0,sp,#320 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y); - - ldr x3,[x23,#32] - ldp x4,x5,[sp,#352] - ldp x6,x7,[sp,#352+16] - add x2,x23,#32 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); - - add x2,sp,#320 - ldr x3,[sp,#192] // forward load for p256_mul_mont - ldp x4,x5,[x22] - ldp x6,x7,[x22,#16] - add x0,sp,#160 - bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1); - - orr x14,x14,x15 // see if result is zero - orr x16,x16,x17 - orr x26,x14,x16 // ~is_equal(S1,S2) - - add x2,sp,#192 - add x0,sp,#256 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr); - - ldr x3,[sp,#128] - ldp x4,x5,[x23] - ldp x6,x7,[x23,#16] - add x2,sp,#128 - add x0,sp,#288 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr); - - add x2,sp,#256 - ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont - ldp x6,x7,[sp,#160+16] - add x0,sp,#96 - bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1); - - orr x14,x14,x15 // see if result is zero - orr x16,x16,x17 - orr x14,x14,x16 // ~is_equal(U1,U2) - - mvn x27,x24 // -1/0 -> 0/-1 - mvn x28,x25 // -1/0 -> 0/-1 - orr x14,x14,x27 - orr x14,x14,x28 - orr x14,x14,x26 - cbnz x14,Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2)) - -Ladd_double: - mov x1,x22 - mov x0,x21 - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames - b Ldouble_shortcut - -.align 4 -Ladd_proceed: - add x0,sp,#192 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#96] - ldp x6,x7,[sp,#96+16] - add x2,x22,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); - - ldp x4,x5,[sp,#96] - ldp x6,x7,[sp,#96+16] - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); - - ldr x3,[x23,#64] - ldp x4,x5,[sp,#64] - ldp x6,x7,[sp,#64+16] - add x2,x23,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z); - - ldr x3,[sp,#96] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,sp,#96 - add x0,sp,#224 - bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); - - ldr x3,[sp,#128] - ldp x4,x5,[sp,#256] - ldp x6,x7,[sp,#256+16] - add x2,sp,#128 - add x0,sp,#288 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr); - - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - add x0,sp,#128 - bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); - - add x2,sp,#192 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); - - add x2,sp,#224 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); - - add x2,sp,#288 - ldr x3,[sp,#224] // forward load for p256_mul_mont - ldp x4,x5,[sp,#320] - ldp x6,x7,[sp,#320+16] - add x0,sp,#32 - bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); - - add x2,sp,#224 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub); - - ldr x3,[sp,#160] - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x2,sp,#160 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); - - add x2,sp,#352 - bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); - - ldp x4,x5,[sp,#0] // res - ldp x6,x7,[sp,#0+16] - ldp x8,x9,[x23] // in2 - ldp x10,x11,[x23,#16] - ldp x14,x15,[x22,#0] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#0+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+0+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+0+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#0+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#0+48] - stp x14,x15,[x21,#0] - stp x16,x17,[x21,#0+16] - ldp x14,x15,[x22,#32] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#32+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+32+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+32+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#32+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#32+48] - stp x14,x15,[x21,#32] - stp x16,x17,[x21,#32+16] - ldp x14,x15,[x22,#64] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#64+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - csel x14,x8,x14,ne - csel x15,x9,x15,ne - csel x16,x10,x16,ne - csel x17,x11,x17,ne - stp x14,x15,[x21,#64] - stp x16,x17,[x21,#64+16] - -Ladd_done: - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl _ecp_nistz256_point_add_affine -.private_extern _ecp_nistz256_point_add_affine - -.align 5 -_ecp_nistz256_point_add_affine: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - sub sp,sp,#32*10 - - mov x21,x0 - mov x22,x1 - mov x23,x2 - adrp x13,Lpoly@PAGE - add x13,x13,Lpoly@PAGEOFF - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - ldp x4,x5,[x1,#64] // in1_z - ldp x6,x7,[x1,#64+16] - orr x8,x4,x5 - orr x10,x6,x7 - orr x24,x8,x10 - cmp x24,#0 - csetm x24,ne // ~in1infty - - ldp x14,x15,[x2] // in2_x - ldp x16,x17,[x2,#16] - ldp x8,x9,[x2,#32] // in2_y - ldp x10,x11,[x2,#48] - orr x14,x14,x15 - orr x16,x16,x17 - orr x8,x8,x9 - orr x10,x10,x11 - orr x14,x14,x16 - orr x8,x8,x10 - orr x25,x14,x8 - cmp x25,#0 - csetm x25,ne // ~in2infty - - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); - - mov x4,x14 - mov x5,x15 - mov x6,x16 - mov x7,x17 - ldr x3,[x23] - add x2,x23,#0 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x); - - add x2,x22,#0 - ldr x3,[x22,#64] // forward load for p256_mul_mont - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x0,sp,#160 - bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x); - - add x2,x22,#64 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#160] - ldp x6,x7,[sp,#160+16] - add x2,x22,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); - - ldr x3,[x23,#32] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,x23,#32 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); - - add x2,x22,#32 - ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont - ldp x6,x7,[sp,#160+16] - add x0,sp,#192 - bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y); - - add x0,sp,#224 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); - - ldp x4,x5,[sp,#192] - ldp x6,x7,[sp,#192+16] - add x0,sp,#288 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); - - ldr x3,[sp,#160] - ldp x4,x5,[sp,#224] - ldp x6,x7,[sp,#224+16] - add x2,sp,#160 - add x0,sp,#256 - bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); - - ldr x3,[x22] - ldp x4,x5,[sp,#224] - ldp x6,x7,[sp,#224+16] - add x2,x22,#0 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr); - - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - add x0,sp,#224 - bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); - - add x2,sp,#288 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); - - add x2,sp,#256 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); - - add x2,sp,#96 - ldr x3,[x22,#32] // forward load for p256_mul_mont - ldp x4,x5,[sp,#256] - ldp x6,x7,[sp,#256+16] - add x0,sp,#32 - bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); - - add x2,x22,#32 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub); - - ldr x3,[sp,#192] - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x2,sp,#192 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); - - add x2,sp,#128 - bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); - - ldp x4,x5,[sp,#0] // res - ldp x6,x7,[sp,#0+16] - ldp x8,x9,[x23] // in2 - ldp x10,x11,[x23,#16] - ldp x14,x15,[x22,#0] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#0+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+0+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+0+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#0+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#0+48] - stp x14,x15,[x21,#0] - stp x16,x17,[x21,#0+16] - adrp x23,Lone_mont@PAGE-64 - add x23,x23,Lone_mont@PAGEOFF-64 - ldp x14,x15,[x22,#32] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#32+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+32+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+32+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#32+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#32+48] - stp x14,x15,[x21,#32] - stp x16,x17,[x21,#32+16] - ldp x14,x15,[x22,#64] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#64+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - csel x14,x8,x14,ne - csel x15,x9,x15,ne - csel x16,x10,x16,ne - csel x17,x11,x17,ne - stp x14,x15,[x21,#64] - stp x16,x17,[x21,#64+16] - - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x29,x30,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret - -//////////////////////////////////////////////////////////////////////// // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4], // uint64_t b[4]); .globl _ecp_nistz256_ord_mul_mont @@ -1225,7 +315,7 @@ ldp x12,x13,[x23,#0] ldp x21,x22,[x23,#16] - ldr x23,[x23,#32] + ldr x23,[x23,#32] // LordK mul x14,x4,x3 // a[0]*b[0] umulh x8,x4,x3 @@ -1606,119 +696,4 @@ ldr x29,[sp],#64 ret -//////////////////////////////////////////////////////////////////////// -// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); -.globl _ecp_nistz256_select_w5 -.private_extern _ecp_nistz256_select_w5 - -.align 4 -_ecp_nistz256_select_w5: - AARCH64_VALID_CALL_TARGET - - // x10 := x0 - // w9 := 0; loop counter and incremented internal index - mov x10, x0 - mov w9, #0 - - // [v16-v21] := 0 - movi v16.16b, #0 - movi v17.16b, #0 - movi v18.16b, #0 - movi v19.16b, #0 - movi v20.16b, #0 - movi v21.16b, #0 - -Lselect_w5_loop: - // Loop 16 times. - - // Increment index (loop counter); tested at the end of the loop - add w9, w9, #1 - - // [v22-v27] := Load a (3*256-bit = 6*128-bit) table entry starting at x1 - // and advance x1 to point to the next entry - ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 - - // x11 := (w9 == w2)? All 1s : All 0s - cmp w9, w2 - csetm x11, eq - - // continue loading ... - ld1 {v26.2d, v27.2d}, [x1],#32 - - // duplicate mask_64 into Mask (all 0s or all 1s) - dup v3.2d, x11 - - // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] - // i.e., values in output registers will remain the same if w9 != w2 - bit v16.16b, v22.16b, v3.16b - bit v17.16b, v23.16b, v3.16b - - bit v18.16b, v24.16b, v3.16b - bit v19.16b, v25.16b, v3.16b - - bit v20.16b, v26.16b, v3.16b - bit v21.16b, v27.16b, v3.16b - - // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back - tbz w9, #4, Lselect_w5_loop - - // Write [v16-v21] to memory at the output pointer - st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x10],#64 - st1 {v20.2d, v21.2d}, [x10] - - ret - - - -//////////////////////////////////////////////////////////////////////// -// void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); -.globl _ecp_nistz256_select_w7 -.private_extern _ecp_nistz256_select_w7 - -.align 4 -_ecp_nistz256_select_w7: - AARCH64_VALID_CALL_TARGET - - // w9 := 0; loop counter and incremented internal index - mov w9, #0 - - // [v16-v21] := 0 - movi v16.16b, #0 - movi v17.16b, #0 - movi v18.16b, #0 - movi v19.16b, #0 - -Lselect_w7_loop: - // Loop 64 times. - - // Increment index (loop counter); tested at the end of the loop - add w9, w9, #1 - - // [v22-v25] := Load a (2*256-bit = 4*128-bit) table entry starting at x1 - // and advance x1 to point to the next entry - ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 - - // x11 := (w9 == w2)? All 1s : All 0s - cmp w9, w2 - csetm x11, eq - - // duplicate mask_64 into Mask (all 0s or all 1s) - dup v3.2d, x11 - - // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] - // i.e., values in output registers will remain the same if w9 != w2 - bit v16.16b, v22.16b, v3.16b - bit v17.16b, v23.16b, v3.16b - - bit v18.16b, v24.16b, v3.16b - bit v19.16b, v25.16b, v3.16b - - // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back - tbz w9, #6, Lselect_w7_loop - - // Write [v16-v19] to memory at the output pointer - st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x0] - - ret - #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__APPLE__)
diff --git a/gen/bcm/p256-armv8-asm-linux.S b/gen/bcm/p256-armv8-asm-linux.S index 1d63f02..36295da 100644 --- a/gen/bcm/p256-armv8-asm-linux.S +++ b/gen/bcm/p256-armv8-asm-linux.S
@@ -4,26 +4,6 @@ #include <openssl/asm_base.h> #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(__ELF__) -.section .rodata -.align 5 -.Lpoly: -.quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 -.LRR: // 2^512 mod P precomputed for NIST P256 polynomial -.quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd -.Lone_mont: -.quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe -.Lone: -.quad 1,0,0,0 -.Lord: -.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 -.LordK: -.quad 0xccd1c8aaee00bc4f -.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 -.align 2 -.text - -// void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4], -// const BN_ULONG x2[4]); .globl ecp_nistz256_mul_mont .hidden ecp_nistz256_mul_mont .type ecp_nistz256_mul_mont,%function @@ -34,15 +14,135 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - ldr x3,[x2] // bp[0] - ldp x4,x5,[x1] - ldp x6,x7,[x1,#16] - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] + mul x13,x3,x2 // a[0]*b[0] + umulh x8,x3,x2 - bl __ecp_nistz256_mul_mont + mul x14,x4,x2 // a[1]*b[0] + umulh x9,x4,x2 + + mul x15,x5,x2 // a[2]*b[0] + umulh x10,x5,x2 + + mul x16,x6,x2 // a[3]*b[0] + umulh x11,x6,x2 + ldr x2,[x1,#8] // b[1] + + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adc x17,xzr,x11 + mov x19,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + ldr x2,[x1,#8*(1+1)] // b[1+1] + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + ldr x2,[x1,#8*(2+1)] // b[2+1] + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + mov w7, #-1 // poly1 = 0x00000000ffffffff + // last reduction + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + adcs x16,x17,x11 + adc x17,x19,xzr + + neg x12,x7 // poly3 = 0xffffffff00000001 + adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus + sbcs x9,x14,x7 + sbcs x10,x15,xzr + sbcs x11,x16,x12 + sbcs xzr,x17,xzr // did it borrow? + + csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus + csel x14,x14,x9,lo + csel x15,x15,x10,lo + stp x13,x14,[x0] + csel x16,x16,x11,lo + stp x15,x16,[x0,#16] ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 @@ -50,7 +150,6 @@ ret .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont -// void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); .globl ecp_nistz256_sqr_mont .hidden ecp_nistz256_sqr_mont .type ecp_nistz256_sqr_mont,%function @@ -61,305 +160,6 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - ldp x4,x5,[x1] - ldp x6,x7,[x1,#16] - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sqr_mont - - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont - -// void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_div_by_2 -.hidden ecp_nistz256_div_by_2 -.type ecp_nistz256_div_by_2,%function -.align 4 -ecp_nistz256_div_by_2: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_div_by_2 - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_div_by_2,.-ecp_nistz256_div_by_2 - -// void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_mul_by_2 -.hidden ecp_nistz256_mul_by_2 -.type ecp_nistz256_mul_by_2,%function -.align 4 -ecp_nistz256_mul_by_2: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - - bl __ecp_nistz256_add_to // ret = a+a // 2*a - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2 - -// void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_mul_by_3 -.hidden ecp_nistz256_mul_by_3 -.type ecp_nistz256_mul_by_3,%function -.align 4 -ecp_nistz256_mul_by_3: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - mov x4,x14 - mov x5,x15 - mov x6,x16 - mov x7,x17 - - bl __ecp_nistz256_add_to // ret = a+a // 2*a - - mov x8,x4 - mov x9,x5 - mov x10,x6 - mov x11,x7 - - bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_mul_by_3,.-ecp_nistz256_mul_by_3 - -// void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4], -// const BN_ULONG x2[4]); -.globl ecp_nistz256_sub -.hidden ecp_nistz256_sub -.type ecp_nistz256_sub,%function -.align 4 -ecp_nistz256_sub: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sub_from - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_sub,.-ecp_nistz256_sub - -// void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_neg -.hidden ecp_nistz256_neg -.type ecp_nistz256_neg,%function -.align 4 -ecp_nistz256_neg: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - mov x2,x1 - mov x14,xzr // a = 0 - mov x15,xzr - mov x16,xzr - mov x17,xzr - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sub_from - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_neg,.-ecp_nistz256_neg - -// note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded -// to x4-x7 and b[0] - to x3 -.type __ecp_nistz256_mul_mont,%function -.align 4 -__ecp_nistz256_mul_mont: - mul x14,x4,x3 // a[0]*b[0] - umulh x8,x4,x3 - - mul x15,x5,x3 // a[1]*b[0] - umulh x9,x5,x3 - - mul x16,x6,x3 // a[2]*b[0] - umulh x10,x6,x3 - - mul x17,x7,x3 // a[3]*b[0] - umulh x11,x7,x3 - ldr x3,[x2,#8] // b[1] - - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adc x19,xzr,x11 - mov x20,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - ldr x3,[x2,#8*(1+1)] // b[1+1] - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - ldr x3,[x2,#8*(2+1)] // b[2+1] - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - // last reduction - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - adcs x17,x19,x11 - adc x19,x20,xzr - - adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x19,xzr // did it borrow? - - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] - - ret -.size __ecp_nistz256_mul_mont,.-__ecp_nistz256_mul_mont - -// note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded -// to x4-x7 -.type __ecp_nistz256_sqr_mont,%function -.align 4 -__ecp_nistz256_sqr_mont: // | | | | | |a1*a0| | // | | | | |a2*a0| | | // | |a3*a2|a3*a0| | | | @@ -374,834 +174,124 @@ // multiplication result, which can't overflow, because it // can never be all ones. - mul x15,x5,x4 // a[1]*a[0] + mul x14,x4,x3 // a[1]*a[0] + umulh x9,x4,x3 + mul x15,x5,x3 // a[2]*a[0] + umulh x10,x5,x3 + mul x16,x6,x3 // a[3]*a[0] + umulh x17,x6,x3 + + adds x15,x15,x9 // accumulate high parts of multiplication + mul x8,x5,x4 // a[2]*a[1] umulh x9,x5,x4 - mul x16,x6,x4 // a[2]*a[0] - umulh x10,x6,x4 - mul x17,x7,x4 // a[3]*a[0] - umulh x19,x7,x4 + adcs x16,x16,x10 + mul x10,x6,x4 // a[3]*a[1] + umulh x11,x6,x4 + adc x17,x17,xzr // can't overflow - adds x16,x16,x9 // accumulate high parts of multiplication - mul x8,x6,x5 // a[2]*a[1] - umulh x9,x6,x5 - adcs x17,x17,x10 - mul x10,x7,x5 // a[3]*a[1] - umulh x11,x7,x5 - adc x19,x19,xzr // can't overflow - - mul x20,x7,x6 // a[3]*a[2] - umulh x1,x7,x6 + mul x19,x6,x5 // a[3]*a[2] + umulh x20,x6,x5 adds x9,x9,x10 // accumulate high parts of multiplication - mul x14,x4,x4 // a[0]*a[0] + mul x13,x3,x3 // a[0]*a[0] adc x10,x11,xzr // can't overflow - adds x17,x17,x8 // accumulate low parts of multiplication + adds x16,x16,x8 // accumulate low parts of multiplication + umulh x3,x3,x3 + adcs x17,x17,x9 + mul x9,x4,x4 // a[1]*a[1] + adcs x19,x19,x10 umulh x4,x4,x4 - adcs x19,x19,x9 - mul x9,x5,x5 // a[1]*a[1] - adcs x20,x20,x10 - umulh x5,x5,x5 - adc x1,x1,xzr // can't overflow + adc x20,x20,xzr // can't overflow - adds x15,x15,x15 // acc[1-6]*=2 - mul x10,x6,x6 // a[2]*a[2] + adds x14,x14,x14 // acc[1-6]*=2 + mul x10,x5,x5 // a[2]*a[2] + adcs x15,x15,x15 + umulh x5,x5,x5 adcs x16,x16,x16 - umulh x6,x6,x6 + mul x11,x6,x6 // a[3]*a[3] adcs x17,x17,x17 - mul x11,x7,x7 // a[3]*a[3] + umulh x6,x6,x6 adcs x19,x19,x19 - umulh x7,x7,x7 adcs x20,x20,x20 - adcs x1,x1,x1 adc x2,xzr,xzr - adds x15,x15,x4 // +a[i]*a[i] - adcs x16,x16,x9 - adcs x17,x17,x5 - adcs x19,x19,x10 - adcs x20,x20,x6 - lsl x8,x14,#32 - adcs x1,x1,x11 - lsr x9,x14,#32 - adc x2,x2,x7 - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - adc x17,x11,xzr // can't overflow - - adds x14,x14,x19 // accumulate upper half - adcs x15,x15,x20 - adcs x16,x16,x1 - adcs x17,x17,x2 - adc x19,xzr,xzr - - adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x19,xzr // did it borrow? - - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] - - ret -.size __ecp_nistz256_sqr_mont,.-__ecp_nistz256_sqr_mont - -// Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to -// x4-x7 and x8-x11. This is done because it's used in multiple -// contexts, e.g. in multiplication by 2 and 3... -.type __ecp_nistz256_add_to,%function -.align 4 -__ecp_nistz256_add_to: - adds x14,x14,x8 // ret = a+b + adds x14,x14,x3 // +a[i]*a[i] adcs x15,x15,x9 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x1,xzr,xzr // zap x1 + adcs x16,x16,x4 + adcs x17,x17,x10 + adcs x19,x19,x5 + lsl x8,x13,#32 + adcs x20,x20,x11 + lsr x9,x13,#32 + adc x2,x2,x6 + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + adc x16,x11,xzr // can't overflow - adds x8,x14,#1 // subs x8,x4,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x1,xzr // did subtraction borrow? + mov w7, #-1 // poly1 = 0x00000000ffffffff + adds x13,x13,x17 // accumulate upper half + adcs x14,x14,x19 + adcs x15,x15,x20 + adcs x16,x16,x2 + adc x17,xzr,xzr - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] + neg x12,x7 // poly3 = 0xffffffff00000001 + adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus + sbcs x9,x14,x7 + sbcs x10,x15,xzr + sbcs x11,x16,x12 + sbcs xzr,x17,xzr // did it borrow? - ret -.size __ecp_nistz256_add_to,.-__ecp_nistz256_add_to + csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus + csel x14,x14,x9,lo + csel x15,x15,x10,lo + stp x13,x14,[x0] + csel x16,x16,x11,lo + stp x15,x16,[x0,#16] -.type __ecp_nistz256_sub_from,%function -.align 4 -__ecp_nistz256_sub_from: - ldp x8,x9,[x2] - ldp x10,x11,[x2,#16] - subs x14,x14,x8 // ret = a-b - sbcs x15,x15,x9 - sbcs x16,x16,x10 - sbcs x17,x17,x11 - sbc x1,xzr,xzr // zap x1 - - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adc x11,x17,x13 - cmp x1,xzr // did subtraction borrow? - - csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret - csel x15,x15,x9,eq - csel x16,x16,x10,eq - stp x14,x15,[x0] - csel x17,x17,x11,eq - stp x16,x17,[x0,#16] - - ret -.size __ecp_nistz256_sub_from,.-__ecp_nistz256_sub_from - -.type __ecp_nistz256_sub_morf,%function -.align 4 -__ecp_nistz256_sub_morf: - ldp x8,x9,[x2] - ldp x10,x11,[x2,#16] - subs x14,x8,x14 // ret = b-a - sbcs x15,x9,x15 - sbcs x16,x10,x16 - sbcs x17,x11,x17 - sbc x1,xzr,xzr // zap x1 - - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adc x11,x17,x13 - cmp x1,xzr // did subtraction borrow? - - csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret - csel x15,x15,x9,eq - csel x16,x16,x10,eq - stp x14,x15,[x0] - csel x17,x17,x11,eq - stp x16,x17,[x0,#16] - - ret -.size __ecp_nistz256_sub_morf,.-__ecp_nistz256_sub_morf - -.type __ecp_nistz256_div_by_2,%function -.align 4 -__ecp_nistz256_div_by_2: - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = a+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adcs x11,x17,x13 - adc x1,xzr,xzr // zap x1 - tst x14,#1 // is a even? - - csel x14,x14,x8,eq // ret = even ? a : a+modulus - csel x15,x15,x9,eq - csel x16,x16,x10,eq - csel x17,x17,x11,eq - csel x1,xzr,x1,eq - - lsr x14,x14,#1 // ret >>= 1 - orr x14,x14,x15,lsl#63 - lsr x15,x15,#1 - orr x15,x15,x16,lsl#63 - lsr x16,x16,#1 - orr x16,x16,x17,lsl#63 - lsr x17,x17,#1 - stp x14,x15,[x0] - orr x17,x17,x1,lsl#63 - stp x16,x17,[x0,#16] - - ret -.size __ecp_nistz256_div_by_2,.-__ecp_nistz256_div_by_2 -.globl ecp_nistz256_point_double -.hidden ecp_nistz256_point_double -.type ecp_nistz256_point_double,%function -.align 5 -ecp_nistz256_point_double: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-96]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - sub sp,sp,#32*4 - -.Ldouble_shortcut: - ldp x14,x15,[x1,#32] - mov x21,x0 - ldp x16,x17,[x1,#48] - mov x22,x1 - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - mov x8,x14 - ldr x13,[x13,#24] - mov x9,x15 - ldp x4,x5,[x22,#64] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[x22,#64+16] - add x0,sp,#0 - bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y); - - add x0,sp,#64 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z); - - ldp x8,x9,[x22] - ldp x10,x11,[x22,#16] - mov x4,x14 // put Zsqr aside for p256_sub - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x0,sp,#32 - bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x); - - add x2,x22,#0 - mov x14,x4 // restore Zsqr - mov x15,x5 - ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont - mov x16,x6 - mov x17,x7 - ldp x6,x7,[sp,#0+16] - add x0,sp,#64 - bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr); - - add x0,sp,#0 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S); - - ldr x3,[x22,#32] - ldp x4,x5,[x22,#64] - ldp x6,x7,[x22,#64+16] - add x2,x22,#32 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y); - - mov x8,x14 - mov x9,x15 - ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[sp,#0+16] - add x0,x21,#64 - bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0); - - add x0,sp,#96 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S); - - ldr x3,[sp,#64] // forward load for p256_mul_mont - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x0,x21,#32 - bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0); - - add x2,sp,#64 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr); - - mov x8,x14 // duplicate M - mov x9,x15 - mov x10,x16 - mov x11,x17 - mov x4,x14 // put M aside - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x0,sp,#32 - bl __ecp_nistz256_add_to - mov x8,x4 // restore M - mov x9,x5 - ldr x3,[x22] // forward load for p256_mul_mont - mov x10,x6 - ldp x4,x5,[sp,#0] - mov x11,x7 - ldp x6,x7,[sp,#0+16] - bl __ecp_nistz256_add_to // p256_mul_by_3(M, M); - - add x2,x22,#0 - add x0,sp,#0 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x); - - mov x8,x14 - mov x9,x15 - ldp x4,x5,[sp,#32] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[sp,#32+16] - add x0,sp,#96 - bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S); - - add x0,x21,#0 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M); - - add x2,sp,#96 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0); - - add x2,sp,#0 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x); - - ldr x3,[sp,#32] - mov x4,x14 // copy S - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x2,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M); - - add x2,x21,#32 - add x0,x21,#32 - bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y); - - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x29,x30,[sp],#96 + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 AARCH64_VALIDATE_LINK_REGISTER ret -.size ecp_nistz256_point_double,.-ecp_nistz256_point_double -.globl ecp_nistz256_point_add -.hidden ecp_nistz256_point_add -.type ecp_nistz256_point_add,%function +.size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont +.section .rodata .align 5 -ecp_nistz256_point_add: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-96]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - sub sp,sp,#32*12 +.Lord: +.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 +.LordK: +.quad 0xccd1c8aaee00bc4f +.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 +.align 2 +.text - ldp x4,x5,[x2,#64] // in2_z - ldp x6,x7,[x2,#64+16] - mov x21,x0 - mov x22,x1 - mov x23,x2 - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - orr x8,x4,x5 - orr x10,x6,x7 - orr x25,x8,x10 - cmp x25,#0 - csetm x25,ne // ~in2infty - add x0,sp,#192 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z); - - ldp x4,x5,[x22,#64] // in1_z - ldp x6,x7,[x22,#64+16] - orr x8,x4,x5 - orr x10,x6,x7 - orr x24,x8,x10 - cmp x24,#0 - csetm x24,ne // ~in1infty - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); - - ldr x3,[x23,#64] - ldp x4,x5,[sp,#192] - ldp x6,x7,[sp,#192+16] - add x2,x23,#64 - add x0,sp,#320 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,x22,#64 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); - - ldr x3,[x22,#32] - ldp x4,x5,[sp,#320] - ldp x6,x7,[sp,#320+16] - add x2,x22,#32 - add x0,sp,#320 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y); - - ldr x3,[x23,#32] - ldp x4,x5,[sp,#352] - ldp x6,x7,[sp,#352+16] - add x2,x23,#32 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); - - add x2,sp,#320 - ldr x3,[sp,#192] // forward load for p256_mul_mont - ldp x4,x5,[x22] - ldp x6,x7,[x22,#16] - add x0,sp,#160 - bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1); - - orr x14,x14,x15 // see if result is zero - orr x16,x16,x17 - orr x26,x14,x16 // ~is_equal(S1,S2) - - add x2,sp,#192 - add x0,sp,#256 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr); - - ldr x3,[sp,#128] - ldp x4,x5,[x23] - ldp x6,x7,[x23,#16] - add x2,sp,#128 - add x0,sp,#288 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr); - - add x2,sp,#256 - ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont - ldp x6,x7,[sp,#160+16] - add x0,sp,#96 - bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1); - - orr x14,x14,x15 // see if result is zero - orr x16,x16,x17 - orr x14,x14,x16 // ~is_equal(U1,U2) - - mvn x27,x24 // -1/0 -> 0/-1 - mvn x28,x25 // -1/0 -> 0/-1 - orr x14,x14,x27 - orr x14,x14,x28 - orr x14,x14,x26 - cbnz x14,.Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2)) - -.Ladd_double: - mov x1,x22 - mov x0,x21 - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames - b .Ldouble_shortcut - -.align 4 -.Ladd_proceed: - add x0,sp,#192 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#96] - ldp x6,x7,[sp,#96+16] - add x2,x22,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); - - ldp x4,x5,[sp,#96] - ldp x6,x7,[sp,#96+16] - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); - - ldr x3,[x23,#64] - ldp x4,x5,[sp,#64] - ldp x6,x7,[sp,#64+16] - add x2,x23,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z); - - ldr x3,[sp,#96] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,sp,#96 - add x0,sp,#224 - bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); - - ldr x3,[sp,#128] - ldp x4,x5,[sp,#256] - ldp x6,x7,[sp,#256+16] - add x2,sp,#128 - add x0,sp,#288 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr); - - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - add x0,sp,#128 - bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); - - add x2,sp,#192 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); - - add x2,sp,#224 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); - - add x2,sp,#288 - ldr x3,[sp,#224] // forward load for p256_mul_mont - ldp x4,x5,[sp,#320] - ldp x6,x7,[sp,#320+16] - add x0,sp,#32 - bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); - - add x2,sp,#224 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub); - - ldr x3,[sp,#160] - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x2,sp,#160 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); - - add x2,sp,#352 - bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); - - ldp x4,x5,[sp,#0] // res - ldp x6,x7,[sp,#0+16] - ldp x8,x9,[x23] // in2 - ldp x10,x11,[x23,#16] - ldp x14,x15,[x22,#0] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#0+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+0+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+0+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#0+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#0+48] - stp x14,x15,[x21,#0] - stp x16,x17,[x21,#0+16] - ldp x14,x15,[x22,#32] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#32+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+32+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+32+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#32+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#32+48] - stp x14,x15,[x21,#32] - stp x16,x17,[x21,#32+16] - ldp x14,x15,[x22,#64] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#64+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - csel x14,x8,x14,ne - csel x15,x9,x15,ne - csel x16,x10,x16,ne - csel x17,x11,x17,ne - stp x14,x15,[x21,#64] - stp x16,x17,[x21,#64+16] - -.Ladd_done: - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_point_add,.-ecp_nistz256_point_add -.globl ecp_nistz256_point_add_affine -.hidden ecp_nistz256_point_add_affine -.type ecp_nistz256_point_add_affine,%function -.align 5 -ecp_nistz256_point_add_affine: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - sub sp,sp,#32*10 - - mov x21,x0 - mov x22,x1 - mov x23,x2 - adrp x13,.Lpoly - add x13,x13,:lo12:.Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - ldp x4,x5,[x1,#64] // in1_z - ldp x6,x7,[x1,#64+16] - orr x8,x4,x5 - orr x10,x6,x7 - orr x24,x8,x10 - cmp x24,#0 - csetm x24,ne // ~in1infty - - ldp x14,x15,[x2] // in2_x - ldp x16,x17,[x2,#16] - ldp x8,x9,[x2,#32] // in2_y - ldp x10,x11,[x2,#48] - orr x14,x14,x15 - orr x16,x16,x17 - orr x8,x8,x9 - orr x10,x10,x11 - orr x14,x14,x16 - orr x8,x8,x10 - orr x25,x14,x8 - cmp x25,#0 - csetm x25,ne // ~in2infty - - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); - - mov x4,x14 - mov x5,x15 - mov x6,x16 - mov x7,x17 - ldr x3,[x23] - add x2,x23,#0 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x); - - add x2,x22,#0 - ldr x3,[x22,#64] // forward load for p256_mul_mont - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x0,sp,#160 - bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x); - - add x2,x22,#64 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#160] - ldp x6,x7,[sp,#160+16] - add x2,x22,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); - - ldr x3,[x23,#32] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,x23,#32 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); - - add x2,x22,#32 - ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont - ldp x6,x7,[sp,#160+16] - add x0,sp,#192 - bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y); - - add x0,sp,#224 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); - - ldp x4,x5,[sp,#192] - ldp x6,x7,[sp,#192+16] - add x0,sp,#288 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); - - ldr x3,[sp,#160] - ldp x4,x5,[sp,#224] - ldp x6,x7,[sp,#224+16] - add x2,sp,#160 - add x0,sp,#256 - bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); - - ldr x3,[x22] - ldp x4,x5,[sp,#224] - ldp x6,x7,[sp,#224+16] - add x2,x22,#0 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr); - - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - add x0,sp,#224 - bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); - - add x2,sp,#288 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); - - add x2,sp,#256 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); - - add x2,sp,#96 - ldr x3,[x22,#32] // forward load for p256_mul_mont - ldp x4,x5,[sp,#256] - ldp x6,x7,[sp,#256+16] - add x0,sp,#32 - bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); - - add x2,x22,#32 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub); - - ldr x3,[sp,#192] - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x2,sp,#192 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); - - add x2,sp,#128 - bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); - - ldp x4,x5,[sp,#0] // res - ldp x6,x7,[sp,#0+16] - ldp x8,x9,[x23] // in2 - ldp x10,x11,[x23,#16] - ldp x14,x15,[x22,#0] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#0+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+0+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+0+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#0+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#0+48] - stp x14,x15,[x21,#0] - stp x16,x17,[x21,#0+16] - adrp x23,.Lone_mont-64 - add x23,x23,:lo12:.Lone_mont-64 - ldp x14,x15,[x22,#32] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#32+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+32+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+32+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#32+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#32+48] - stp x14,x15,[x21,#32] - stp x16,x17,[x21,#32+16] - ldp x14,x15,[x22,#64] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#64+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - csel x14,x8,x14,ne - csel x15,x9,x15,ne - csel x16,x10,x16,ne - csel x17,x11,x17,ne - stp x14,x15,[x21,#64] - stp x16,x17,[x21,#64+16] - - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x29,x30,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret -.size ecp_nistz256_point_add_affine,.-ecp_nistz256_point_add_affine -//////////////////////////////////////////////////////////////////////// // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4], // uint64_t b[4]); .globl ecp_nistz256_ord_mul_mont @@ -1225,7 +315,7 @@ ldp x12,x13,[x23,#0] ldp x21,x22,[x23,#16] - ldr x23,[x23,#32] + ldr x23,[x23,#32] // .LordK mul x14,x4,x3 // a[0]*b[0] umulh x8,x4,x3 @@ -1606,119 +696,4 @@ ldr x29,[sp],#64 ret .size ecp_nistz256_ord_sqr_mont,.-ecp_nistz256_ord_sqr_mont -//////////////////////////////////////////////////////////////////////// -// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w5 -.hidden ecp_nistz256_select_w5 -.type ecp_nistz256_select_w5,%function -.align 4 -ecp_nistz256_select_w5: - AARCH64_VALID_CALL_TARGET - - // x10 := x0 - // w9 := 0; loop counter and incremented internal index - mov x10, x0 - mov w9, #0 - - // [v16-v21] := 0 - movi v16.16b, #0 - movi v17.16b, #0 - movi v18.16b, #0 - movi v19.16b, #0 - movi v20.16b, #0 - movi v21.16b, #0 - -.Lselect_w5_loop: - // Loop 16 times. - - // Increment index (loop counter); tested at the end of the loop - add w9, w9, #1 - - // [v22-v27] := Load a (3*256-bit = 6*128-bit) table entry starting at x1 - // and advance x1 to point to the next entry - ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 - - // x11 := (w9 == w2)? All 1s : All 0s - cmp w9, w2 - csetm x11, eq - - // continue loading ... - ld1 {v26.2d, v27.2d}, [x1],#32 - - // duplicate mask_64 into Mask (all 0s or all 1s) - dup v3.2d, x11 - - // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] - // i.e., values in output registers will remain the same if w9 != w2 - bit v16.16b, v22.16b, v3.16b - bit v17.16b, v23.16b, v3.16b - - bit v18.16b, v24.16b, v3.16b - bit v19.16b, v25.16b, v3.16b - - bit v20.16b, v26.16b, v3.16b - bit v21.16b, v27.16b, v3.16b - - // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back - tbz w9, #4, .Lselect_w5_loop - - // Write [v16-v21] to memory at the output pointer - st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x10],#64 - st1 {v20.2d, v21.2d}, [x10] - - ret -.size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5 - - -//////////////////////////////////////////////////////////////////////// -// void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w7 -.hidden ecp_nistz256_select_w7 -.type ecp_nistz256_select_w7,%function -.align 4 -ecp_nistz256_select_w7: - AARCH64_VALID_CALL_TARGET - - // w9 := 0; loop counter and incremented internal index - mov w9, #0 - - // [v16-v21] := 0 - movi v16.16b, #0 - movi v17.16b, #0 - movi v18.16b, #0 - movi v19.16b, #0 - -.Lselect_w7_loop: - // Loop 64 times. - - // Increment index (loop counter); tested at the end of the loop - add w9, w9, #1 - - // [v22-v25] := Load a (2*256-bit = 4*128-bit) table entry starting at x1 - // and advance x1 to point to the next entry - ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 - - // x11 := (w9 == w2)? All 1s : All 0s - cmp w9, w2 - csetm x11, eq - - // duplicate mask_64 into Mask (all 0s or all 1s) - dup v3.2d, x11 - - // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] - // i.e., values in output registers will remain the same if w9 != w2 - bit v16.16b, v22.16b, v3.16b - bit v17.16b, v23.16b, v3.16b - - bit v18.16b, v24.16b, v3.16b - bit v19.16b, v25.16b, v3.16b - - // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back - tbz w9, #6, .Lselect_w7_loop - - // Write [v16-v19] to memory at the output pointer - st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x0] - - ret -.size ecp_nistz256_select_w7,.-ecp_nistz256_select_w7 #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(__ELF__)
diff --git a/gen/bcm/p256-armv8-asm-win.S b/gen/bcm/p256-armv8-asm-win.S index f15b8ab..9856780 100644 --- a/gen/bcm/p256-armv8-asm-win.S +++ b/gen/bcm/p256-armv8-asm-win.S
@@ -4,26 +4,6 @@ #include <openssl/asm_base.h> #if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) && defined(_WIN32) -.section .rodata -.align 5 -Lpoly: -.quad 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 -LRR: // 2^512 mod P precomputed for NIST P256 polynomial -.quad 0x0000000000000003,0xfffffffbffffffff,0xfffffffffffffffe,0x00000004fffffffd -Lone_mont: -.quad 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe -Lone: -.quad 1,0,0,0 -Lord: -.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 -LordK: -.quad 0xccd1c8aaee00bc4f -.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 -.align 2 -.text - -// void ecp_nistz256_mul_mont(BN_ULONG x0[4],const BN_ULONG x1[4], -// const BN_ULONG x2[4]); .globl ecp_nistz256_mul_mont .def ecp_nistz256_mul_mont @@ -36,15 +16,135 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - ldr x3,[x2] // bp[0] - ldp x4,x5,[x1] - ldp x6,x7,[x1,#16] - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] + mul x13,x3,x2 // a[0]*b[0] + umulh x8,x3,x2 - bl __ecp_nistz256_mul_mont + mul x14,x4,x2 // a[1]*b[0] + umulh x9,x4,x2 + + mul x15,x5,x2 // a[2]*b[0] + umulh x10,x5,x2 + + mul x16,x6,x2 // a[3]*b[0] + umulh x11,x6,x2 + ldr x2,[x1,#8] // b[1] + + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adc x17,xzr,x11 + mov x19,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + ldr x2,[x1,#8*(1+1)] // b[1+1] + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + ldr x2,[x1,#8*(2+1)] // b[2+1] + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + mul x8,x3,x2 // lo(a[0]*b[i]) + adcs x14,x15,x9 + mul x9,x4,x2 // lo(a[1]*b[i]) + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + mul x10,x5,x2 // lo(a[2]*b[i]) + adcs x16,x17,x11 + mul x11,x6,x2 // lo(a[3]*b[i]) + adc x17,x19,xzr + + adds x13,x13,x8 // accumulate low parts of multiplication + umulh x8,x3,x2 // hi(a[0]*b[i]) + adcs x14,x14,x9 + umulh x9,x4,x2 // hi(a[1]*b[i]) + adcs x15,x15,x10 + umulh x10,x5,x2 // hi(a[2]*b[i]) + adcs x16,x16,x11 + umulh x11,x6,x2 // hi(a[3]*b[i]) + adc x17,x17,xzr + adds x14,x14,x8 // accumulate high parts of multiplication + lsl x8,x13,#32 + adcs x15,x15,x9 + lsr x9,x13,#32 + adcs x16,x16,x10 + adcs x17,x17,x11 + adc x19,xzr,xzr + mov w7, #-1 // poly1 = 0x00000000ffffffff + // last reduction + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + adcs x16,x17,x11 + adc x17,x19,xzr + + neg x12,x7 // poly3 = 0xffffffff00000001 + adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus + sbcs x9,x14,x7 + sbcs x10,x15,xzr + sbcs x11,x16,x12 + sbcs xzr,x17,xzr // did it borrow? + + csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus + csel x14,x14,x9,lo + csel x15,x15,x10,lo + stp x13,x14,[x0] + csel x16,x16,x11,lo + stp x15,x16,[x0,#16] ldp x19,x20,[sp,#16] ldp x29,x30,[sp],#32 @@ -52,7 +152,6 @@ ret -// void ecp_nistz256_sqr_mont(BN_ULONG x0[4],const BN_ULONG x1[4]); .globl ecp_nistz256_sqr_mont .def ecp_nistz256_sqr_mont @@ -65,319 +164,6 @@ add x29,sp,#0 stp x19,x20,[sp,#16] - ldp x4,x5,[x1] - ldp x6,x7,[x1,#16] - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sqr_mont - - ldp x19,x20,[sp,#16] - ldp x29,x30,[sp],#32 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_div_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_div_by_2 - -.def ecp_nistz256_div_by_2 - .type 32 -.endef -.align 4 -ecp_nistz256_div_by_2: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_div_by_2 - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_mul_by_2(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_mul_by_2 - -.def ecp_nistz256_mul_by_2 - .type 32 -.endef -.align 4 -ecp_nistz256_mul_by_2: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - - bl __ecp_nistz256_add_to // ret = a+a // 2*a - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_mul_by_3(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_mul_by_3 - -.def ecp_nistz256_mul_by_3 - .type 32 -.endef -.align 4 -ecp_nistz256_mul_by_3: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - mov x4,x14 - mov x5,x15 - mov x6,x16 - mov x7,x17 - - bl __ecp_nistz256_add_to // ret = a+a // 2*a - - mov x8,x4 - mov x9,x5 - mov x10,x6 - mov x11,x7 - - bl __ecp_nistz256_add_to // ret += a // 2*a+a=3*a - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_sub(BN_ULONG x0[4],const BN_ULONG x1[4], -// const BN_ULONG x2[4]); -.globl ecp_nistz256_sub - -.def ecp_nistz256_sub - .type 32 -.endef -.align 4 -ecp_nistz256_sub: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - ldp x14,x15,[x1] - ldp x16,x17,[x1,#16] - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sub_from - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// void ecp_nistz256_neg(BN_ULONG x0[4],const BN_ULONG x1[4]); -.globl ecp_nistz256_neg - -.def ecp_nistz256_neg - .type 32 -.endef -.align 4 -ecp_nistz256_neg: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-16]! - add x29,sp,#0 - - mov x2,x1 - mov x14,xzr // a = 0 - mov x15,xzr - mov x16,xzr - mov x17,xzr - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - bl __ecp_nistz256_sub_from - - ldp x29,x30,[sp],#16 - AARCH64_VALIDATE_LINK_REGISTER - ret - - -// note that __ecp_nistz256_mul_mont expects a[0-3] input pre-loaded -// to x4-x7 and b[0] - to x3 -.def __ecp_nistz256_mul_mont - .type 32 -.endef -.align 4 -__ecp_nistz256_mul_mont: - mul x14,x4,x3 // a[0]*b[0] - umulh x8,x4,x3 - - mul x15,x5,x3 // a[1]*b[0] - umulh x9,x5,x3 - - mul x16,x6,x3 // a[2]*b[0] - umulh x10,x6,x3 - - mul x17,x7,x3 // a[3]*b[0] - umulh x11,x7,x3 - ldr x3,[x2,#8] // b[1] - - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adc x19,xzr,x11 - mov x20,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - ldr x3,[x2,#8*(1+1)] // b[1+1] - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - ldr x3,[x2,#8*(2+1)] // b[2+1] - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - mul x8,x4,x3 // lo(a[0]*b[i]) - adcs x15,x16,x9 - mul x9,x5,x3 // lo(a[1]*b[i]) - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - mul x10,x6,x3 // lo(a[2]*b[i]) - adcs x17,x19,x11 - mul x11,x7,x3 // lo(a[3]*b[i]) - adc x19,x20,xzr - - adds x14,x14,x8 // accumulate low parts of multiplication - umulh x8,x4,x3 // hi(a[0]*b[i]) - adcs x15,x15,x9 - umulh x9,x5,x3 // hi(a[1]*b[i]) - adcs x16,x16,x10 - umulh x10,x6,x3 // hi(a[2]*b[i]) - adcs x17,x17,x11 - umulh x11,x7,x3 // hi(a[3]*b[i]) - adc x19,x19,xzr - adds x15,x15,x8 // accumulate high parts of multiplication - lsl x8,x14,#32 - adcs x16,x16,x9 - lsr x9,x14,#32 - adcs x17,x17,x10 - adcs x19,x19,x11 - adc x20,xzr,xzr - // last reduction - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - adcs x17,x19,x11 - adc x19,x20,xzr - - adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x19,xzr // did it borrow? - - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] - - ret - - -// note that __ecp_nistz256_sqr_mont expects a[0-3] input pre-loaded -// to x4-x7 -.def __ecp_nistz256_sqr_mont - .type 32 -.endef -.align 4 -__ecp_nistz256_sqr_mont: // | | | | | |a1*a0| | // | | | | |a2*a0| | | // | |a3*a2|a3*a0| | | | @@ -392,848 +178,124 @@ // multiplication result, which can't overflow, because it // can never be all ones. - mul x15,x5,x4 // a[1]*a[0] + mul x14,x4,x3 // a[1]*a[0] + umulh x9,x4,x3 + mul x15,x5,x3 // a[2]*a[0] + umulh x10,x5,x3 + mul x16,x6,x3 // a[3]*a[0] + umulh x17,x6,x3 + + adds x15,x15,x9 // accumulate high parts of multiplication + mul x8,x5,x4 // a[2]*a[1] umulh x9,x5,x4 - mul x16,x6,x4 // a[2]*a[0] - umulh x10,x6,x4 - mul x17,x7,x4 // a[3]*a[0] - umulh x19,x7,x4 + adcs x16,x16,x10 + mul x10,x6,x4 // a[3]*a[1] + umulh x11,x6,x4 + adc x17,x17,xzr // can't overflow - adds x16,x16,x9 // accumulate high parts of multiplication - mul x8,x6,x5 // a[2]*a[1] - umulh x9,x6,x5 - adcs x17,x17,x10 - mul x10,x7,x5 // a[3]*a[1] - umulh x11,x7,x5 - adc x19,x19,xzr // can't overflow - - mul x20,x7,x6 // a[3]*a[2] - umulh x1,x7,x6 + mul x19,x6,x5 // a[3]*a[2] + umulh x20,x6,x5 adds x9,x9,x10 // accumulate high parts of multiplication - mul x14,x4,x4 // a[0]*a[0] + mul x13,x3,x3 // a[0]*a[0] adc x10,x11,xzr // can't overflow - adds x17,x17,x8 // accumulate low parts of multiplication + adds x16,x16,x8 // accumulate low parts of multiplication + umulh x3,x3,x3 + adcs x17,x17,x9 + mul x9,x4,x4 // a[1]*a[1] + adcs x19,x19,x10 umulh x4,x4,x4 - adcs x19,x19,x9 - mul x9,x5,x5 // a[1]*a[1] - adcs x20,x20,x10 - umulh x5,x5,x5 - adc x1,x1,xzr // can't overflow + adc x20,x20,xzr // can't overflow - adds x15,x15,x15 // acc[1-6]*=2 - mul x10,x6,x6 // a[2]*a[2] + adds x14,x14,x14 // acc[1-6]*=2 + mul x10,x5,x5 // a[2]*a[2] + adcs x15,x15,x15 + umulh x5,x5,x5 adcs x16,x16,x16 - umulh x6,x6,x6 + mul x11,x6,x6 // a[3]*a[3] adcs x17,x17,x17 - mul x11,x7,x7 // a[3]*a[3] + umulh x6,x6,x6 adcs x19,x19,x19 - umulh x7,x7,x7 adcs x20,x20,x20 - adcs x1,x1,x1 adc x2,xzr,xzr - adds x15,x15,x4 // +a[i]*a[i] - adcs x16,x16,x9 - adcs x17,x17,x5 - adcs x19,x19,x10 - adcs x20,x20,x6 - lsl x8,x14,#32 - adcs x1,x1,x11 - lsr x9,x14,#32 - adc x2,x2,x7 - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - lsl x8,x14,#32 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - lsr x9,x14,#32 - adc x17,x11,xzr // can't overflow - subs x10,x14,x8 // "*0xffff0001" - sbc x11,x14,x9 - adds x14,x15,x8 // +=acc[0]<<96 and omit acc[0] - adcs x15,x16,x9 - adcs x16,x17,x10 // +=acc[0]*0xffff0001 - adc x17,x11,xzr // can't overflow - - adds x14,x14,x19 // accumulate upper half - adcs x15,x15,x20 - adcs x16,x16,x1 - adcs x17,x17,x2 - adc x19,xzr,xzr - - adds x8,x14,#1 // subs x8,x14,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x19,xzr // did it borrow? - - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] - - ret - - -// Note that __ecp_nistz256_add_to expects both input vectors pre-loaded to -// x4-x7 and x8-x11. This is done because it's used in multiple -// contexts, e.g. in multiplication by 2 and 3... -.def __ecp_nistz256_add_to - .type 32 -.endef -.align 4 -__ecp_nistz256_add_to: - adds x14,x14,x8 // ret = a+b + adds x14,x14,x3 // +a[i]*a[i] adcs x15,x15,x9 - adcs x16,x16,x10 - adcs x17,x17,x11 - adc x1,xzr,xzr // zap x1 + adcs x16,x16,x4 + adcs x17,x17,x10 + adcs x19,x19,x5 + lsl x8,x13,#32 + adcs x20,x20,x11 + lsr x9,x13,#32 + adc x2,x2,x6 + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + lsl x8,x13,#32 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + lsr x9,x13,#32 + adc x16,x11,xzr // can't overflow + subs x10,x13,x8 // "*0xffff0001" + sbc x11,x13,x9 + adds x13,x14,x8 // +=acc[0]<<96 and omit acc[0] + adcs x14,x15,x9 + adcs x15,x16,x10 // +=acc[0]*0xffff0001 + adc x16,x11,xzr // can't overflow - adds x8,x14,#1 // subs x8,x4,#-1 // tmp = ret-modulus - sbcs x9,x15,x12 - sbcs x10,x16,xzr - sbcs x11,x17,x13 - sbcs xzr,x1,xzr // did subtraction borrow? + mov w7, #-1 // poly1 = 0x00000000ffffffff + adds x13,x13,x17 // accumulate upper half + adcs x14,x14,x19 + adcs x15,x15,x20 + adcs x16,x16,x2 + adc x17,xzr,xzr - csel x14,x14,x8,lo // ret = borrow ? ret : ret-modulus - csel x15,x15,x9,lo - csel x16,x16,x10,lo - stp x14,x15,[x0] - csel x17,x17,x11,lo - stp x16,x17,[x0,#16] + neg x12,x7 // poly3 = 0xffffffff00000001 + adds x8,x13,#1 // subs x8,x13,#-1 // tmp = ret-modulus + sbcs x9,x14,x7 + sbcs x10,x15,xzr + sbcs x11,x16,x12 + sbcs xzr,x17,xzr // did it borrow? - ret + csel x13,x13,x8,lo // ret = borrow ? ret : ret-modulus + csel x14,x14,x9,lo + csel x15,x15,x10,lo + stp x13,x14,[x0] + csel x16,x16,x11,lo + stp x15,x16,[x0,#16] - -.def __ecp_nistz256_sub_from - .type 32 -.endef -.align 4 -__ecp_nistz256_sub_from: - ldp x8,x9,[x2] - ldp x10,x11,[x2,#16] - subs x14,x14,x8 // ret = a-b - sbcs x15,x15,x9 - sbcs x16,x16,x10 - sbcs x17,x17,x11 - sbc x1,xzr,xzr // zap x1 - - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adc x11,x17,x13 - cmp x1,xzr // did subtraction borrow? - - csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret - csel x15,x15,x9,eq - csel x16,x16,x10,eq - stp x14,x15,[x0] - csel x17,x17,x11,eq - stp x16,x17,[x0,#16] - - ret - - -.def __ecp_nistz256_sub_morf - .type 32 -.endef -.align 4 -__ecp_nistz256_sub_morf: - ldp x8,x9,[x2] - ldp x10,x11,[x2,#16] - subs x14,x8,x14 // ret = b-a - sbcs x15,x9,x15 - sbcs x16,x10,x16 - sbcs x17,x11,x17 - sbc x1,xzr,xzr // zap x1 - - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = ret+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adc x11,x17,x13 - cmp x1,xzr // did subtraction borrow? - - csel x14,x14,x8,eq // ret = borrow ? ret+modulus : ret - csel x15,x15,x9,eq - csel x16,x16,x10,eq - stp x14,x15,[x0] - csel x17,x17,x11,eq - stp x16,x17,[x0,#16] - - ret - - -.def __ecp_nistz256_div_by_2 - .type 32 -.endef -.align 4 -__ecp_nistz256_div_by_2: - subs x8,x14,#1 // adds x8,x4,#-1 // tmp = a+modulus - adcs x9,x15,x12 - adcs x10,x16,xzr - adcs x11,x17,x13 - adc x1,xzr,xzr // zap x1 - tst x14,#1 // is a even? - - csel x14,x14,x8,eq // ret = even ? a : a+modulus - csel x15,x15,x9,eq - csel x16,x16,x10,eq - csel x17,x17,x11,eq - csel x1,xzr,x1,eq - - lsr x14,x14,#1 // ret >>= 1 - orr x14,x14,x15,lsl#63 - lsr x15,x15,#1 - orr x15,x15,x16,lsl#63 - lsr x16,x16,#1 - orr x16,x16,x17,lsl#63 - lsr x17,x17,#1 - stp x14,x15,[x0] - orr x17,x17,x1,lsl#63 - stp x16,x17,[x0,#16] - - ret - -.globl ecp_nistz256_point_double - -.def ecp_nistz256_point_double - .type 32 -.endef -.align 5 -ecp_nistz256_point_double: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-96]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - sub sp,sp,#32*4 - -Ldouble_shortcut: - ldp x14,x15,[x1,#32] - mov x21,x0 - ldp x16,x17,[x1,#48] - mov x22,x1 - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - mov x8,x14 - ldr x13,[x13,#24] - mov x9,x15 - ldp x4,x5,[x22,#64] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[x22,#64+16] - add x0,sp,#0 - bl __ecp_nistz256_add_to // p256_mul_by_2(S, in_y); - - add x0,sp,#64 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Zsqr, in_z); - - ldp x8,x9,[x22] - ldp x10,x11,[x22,#16] - mov x4,x14 // put Zsqr aside for p256_sub - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x0,sp,#32 - bl __ecp_nistz256_add_to // p256_add(M, Zsqr, in_x); - - add x2,x22,#0 - mov x14,x4 // restore Zsqr - mov x15,x5 - ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont - mov x16,x6 - mov x17,x7 - ldp x6,x7,[sp,#0+16] - add x0,sp,#64 - bl __ecp_nistz256_sub_morf // p256_sub(Zsqr, in_x, Zsqr); - - add x0,sp,#0 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(S, S); - - ldr x3,[x22,#32] - ldp x4,x5,[x22,#64] - ldp x6,x7,[x22,#64+16] - add x2,x22,#32 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(tmp0, in_z, in_y); - - mov x8,x14 - mov x9,x15 - ldp x4,x5,[sp,#0] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[sp,#0+16] - add x0,x21,#64 - bl __ecp_nistz256_add_to // p256_mul_by_2(res_z, tmp0); - - add x0,sp,#96 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(tmp0, S); - - ldr x3,[sp,#64] // forward load for p256_mul_mont - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x0,x21,#32 - bl __ecp_nistz256_div_by_2 // p256_div_by_2(res_y, tmp0); - - add x2,sp,#64 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(M, M, Zsqr); - - mov x8,x14 // duplicate M - mov x9,x15 - mov x10,x16 - mov x11,x17 - mov x4,x14 // put M aside - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x0,sp,#32 - bl __ecp_nistz256_add_to - mov x8,x4 // restore M - mov x9,x5 - ldr x3,[x22] // forward load for p256_mul_mont - mov x10,x6 - ldp x4,x5,[sp,#0] - mov x11,x7 - ldp x6,x7,[sp,#0+16] - bl __ecp_nistz256_add_to // p256_mul_by_3(M, M); - - add x2,x22,#0 - add x0,sp,#0 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, in_x); - - mov x8,x14 - mov x9,x15 - ldp x4,x5,[sp,#32] // forward load for p256_sqr_mont - mov x10,x16 - mov x11,x17 - ldp x6,x7,[sp,#32+16] - add x0,sp,#96 - bl __ecp_nistz256_add_to // p256_mul_by_2(tmp0, S); - - add x0,x21,#0 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(res_x, M); - - add x2,sp,#96 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, tmp0); - - add x2,sp,#0 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(S, S, res_x); - - ldr x3,[sp,#32] - mov x4,x14 // copy S - mov x5,x15 - mov x6,x16 - mov x7,x17 - add x2,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S, S, M); - - add x2,x21,#32 - add x0,x21,#32 - bl __ecp_nistz256_sub_from // p256_sub(res_y, S, res_y); - - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x29,x30,[sp],#96 + ldp x19,x20,[sp,#16] + ldp x29,x30,[sp],#32 AARCH64_VALIDATE_LINK_REGISTER ret -.globl ecp_nistz256_point_add - -.def ecp_nistz256_point_add - .type 32 -.endef +.section .rodata .align 5 -ecp_nistz256_point_add: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-96]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - stp x27,x28,[sp,#80] - sub sp,sp,#32*12 +Lord: +.quad 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 +LordK: +.quad 0xccd1c8aaee00bc4f +.byte 69,67,80,95,78,73,83,84,90,50,53,54,32,102,111,114,32,65,82,77,118,56,44,32,67,82,89,80,84,79,71,65,77,83,32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115,115,108,46,111,114,103,62,0 +.align 2 +.text - ldp x4,x5,[x2,#64] // in2_z - ldp x6,x7,[x2,#64+16] - mov x21,x0 - mov x22,x1 - mov x23,x2 - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - orr x8,x4,x5 - orr x10,x6,x7 - orr x25,x8,x10 - cmp x25,#0 - csetm x25,ne // ~in2infty - add x0,sp,#192 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z2sqr, in2_z); - - ldp x4,x5,[x22,#64] // in1_z - ldp x6,x7,[x22,#64+16] - orr x8,x4,x5 - orr x10,x6,x7 - orr x24,x8,x10 - cmp x24,#0 - csetm x24,ne // ~in1infty - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); - - ldr x3,[x23,#64] - ldp x4,x5,[sp,#192] - ldp x6,x7,[sp,#192+16] - add x2,x23,#64 - add x0,sp,#320 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, Z2sqr, in2_z); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,x22,#64 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); - - ldr x3,[x22,#32] - ldp x4,x5,[sp,#320] - ldp x6,x7,[sp,#320+16] - add x2,x22,#32 - add x0,sp,#320 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S1, S1, in1_y); - - ldr x3,[x23,#32] - ldp x4,x5,[sp,#352] - ldp x6,x7,[sp,#352+16] - add x2,x23,#32 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); - - add x2,sp,#320 - ldr x3,[sp,#192] // forward load for p256_mul_mont - ldp x4,x5,[x22] - ldp x6,x7,[x22,#16] - add x0,sp,#160 - bl __ecp_nistz256_sub_from // p256_sub(R, S2, S1); - - orr x14,x14,x15 // see if result is zero - orr x16,x16,x17 - orr x26,x14,x16 // ~is_equal(S1,S2) - - add x2,sp,#192 - add x0,sp,#256 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U1, in1_x, Z2sqr); - - ldr x3,[sp,#128] - ldp x4,x5,[x23] - ldp x6,x7,[x23,#16] - add x2,sp,#128 - add x0,sp,#288 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in2_x, Z1sqr); - - add x2,sp,#256 - ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont - ldp x6,x7,[sp,#160+16] - add x0,sp,#96 - bl __ecp_nistz256_sub_from // p256_sub(H, U2, U1); - - orr x14,x14,x15 // see if result is zero - orr x16,x16,x17 - orr x14,x14,x16 // ~is_equal(U1,U2) - - mvn x27,x24 // -1/0 -> 0/-1 - mvn x28,x25 // -1/0 -> 0/-1 - orr x14,x14,x27 - orr x14,x14,x28 - orr x14,x14,x26 - cbnz x14,Ladd_proceed // if(~is_equal(U1,U2) | in1infty | in2infty | ~is_equal(S1,S2)) - -Ladd_double: - mov x1,x22 - mov x0,x21 - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - add sp,sp,#256 // #256 is from #32*(12-4). difference in stack frames - b Ldouble_shortcut - -.align 4 -Ladd_proceed: - add x0,sp,#192 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#96] - ldp x6,x7,[sp,#96+16] - add x2,x22,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); - - ldp x4,x5,[sp,#96] - ldp x6,x7,[sp,#96+16] - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); - - ldr x3,[x23,#64] - ldp x4,x5,[sp,#64] - ldp x6,x7,[sp,#64+16] - add x2,x23,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, res_z, in2_z); - - ldr x3,[sp,#96] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,sp,#96 - add x0,sp,#224 - bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); - - ldr x3,[sp,#128] - ldp x4,x5,[sp,#256] - ldp x6,x7,[sp,#256+16] - add x2,sp,#128 - add x0,sp,#288 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, U1, Hsqr); - - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - add x0,sp,#128 - bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); - - add x2,sp,#192 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); - - add x2,sp,#224 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); - - add x2,sp,#288 - ldr x3,[sp,#224] // forward load for p256_mul_mont - ldp x4,x5,[sp,#320] - ldp x6,x7,[sp,#320+16] - add x0,sp,#32 - bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); - - add x2,sp,#224 - add x0,sp,#352 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S1, Hcub); - - ldr x3,[sp,#160] - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x2,sp,#160 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); - - add x2,sp,#352 - bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); - - ldp x4,x5,[sp,#0] // res - ldp x6,x7,[sp,#0+16] - ldp x8,x9,[x23] // in2 - ldp x10,x11,[x23,#16] - ldp x14,x15,[x22,#0] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#0+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+0+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+0+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#0+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#0+48] - stp x14,x15,[x21,#0] - stp x16,x17,[x21,#0+16] - ldp x14,x15,[x22,#32] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#32+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+32+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+32+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#32+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#32+48] - stp x14,x15,[x21,#32] - stp x16,x17,[x21,#32+16] - ldp x14,x15,[x22,#64] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#64+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - csel x14,x8,x14,ne - csel x15,x9,x15,ne - csel x16,x10,x16,ne - csel x17,x11,x17,ne - stp x14,x15,[x21,#64] - stp x16,x17,[x21,#64+16] - -Ladd_done: - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x27,x28,[x29,#80] - ldp x29,x30,[sp],#96 - AARCH64_VALIDATE_LINK_REGISTER - ret - -.globl ecp_nistz256_point_add_affine - -.def ecp_nistz256_point_add_affine - .type 32 -.endef -.align 5 -ecp_nistz256_point_add_affine: - AARCH64_SIGN_LINK_REGISTER - stp x29,x30,[sp,#-80]! - add x29,sp,#0 - stp x19,x20,[sp,#16] - stp x21,x22,[sp,#32] - stp x23,x24,[sp,#48] - stp x25,x26,[sp,#64] - sub sp,sp,#32*10 - - mov x21,x0 - mov x22,x1 - mov x23,x2 - adrp x13,Lpoly - add x13,x13,:lo12:Lpoly - ldr x12,[x13,#8] - ldr x13,[x13,#24] - - ldp x4,x5,[x1,#64] // in1_z - ldp x6,x7,[x1,#64+16] - orr x8,x4,x5 - orr x10,x6,x7 - orr x24,x8,x10 - cmp x24,#0 - csetm x24,ne // ~in1infty - - ldp x14,x15,[x2] // in2_x - ldp x16,x17,[x2,#16] - ldp x8,x9,[x2,#32] // in2_y - ldp x10,x11,[x2,#48] - orr x14,x14,x15 - orr x16,x16,x17 - orr x8,x8,x9 - orr x10,x10,x11 - orr x14,x14,x16 - orr x8,x8,x10 - orr x25,x14,x8 - cmp x25,#0 - csetm x25,ne // ~in2infty - - add x0,sp,#128 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Z1sqr, in1_z); - - mov x4,x14 - mov x5,x15 - mov x6,x16 - mov x7,x17 - ldr x3,[x23] - add x2,x23,#0 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, Z1sqr, in2_x); - - add x2,x22,#0 - ldr x3,[x22,#64] // forward load for p256_mul_mont - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x0,sp,#160 - bl __ecp_nistz256_sub_from // p256_sub(H, U2, in1_x); - - add x2,x22,#64 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, Z1sqr, in1_z); - - ldr x3,[x22,#64] - ldp x4,x5,[sp,#160] - ldp x6,x7,[sp,#160+16] - add x2,x22,#64 - add x0,sp,#64 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_z, H, in1_z); - - ldr x3,[x23,#32] - ldp x4,x5,[sp,#128] - ldp x6,x7,[sp,#128+16] - add x2,x23,#32 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, S2, in2_y); - - add x2,x22,#32 - ldp x4,x5,[sp,#160] // forward load for p256_sqr_mont - ldp x6,x7,[sp,#160+16] - add x0,sp,#192 - bl __ecp_nistz256_sub_from // p256_sub(R, S2, in1_y); - - add x0,sp,#224 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Hsqr, H); - - ldp x4,x5,[sp,#192] - ldp x6,x7,[sp,#192+16] - add x0,sp,#288 - bl __ecp_nistz256_sqr_mont // p256_sqr_mont(Rsqr, R); - - ldr x3,[sp,#160] - ldp x4,x5,[sp,#224] - ldp x6,x7,[sp,#224+16] - add x2,sp,#160 - add x0,sp,#256 - bl __ecp_nistz256_mul_mont // p256_mul_mont(Hcub, Hsqr, H); - - ldr x3,[x22] - ldp x4,x5,[sp,#224] - ldp x6,x7,[sp,#224+16] - add x2,x22,#0 - add x0,sp,#96 - bl __ecp_nistz256_mul_mont // p256_mul_mont(U2, in1_x, Hsqr); - - mov x8,x14 - mov x9,x15 - mov x10,x16 - mov x11,x17 - add x0,sp,#224 - bl __ecp_nistz256_add_to // p256_mul_by_2(Hsqr, U2); - - add x2,sp,#288 - add x0,sp,#0 - bl __ecp_nistz256_sub_morf // p256_sub(res_x, Rsqr, Hsqr); - - add x2,sp,#256 - bl __ecp_nistz256_sub_from // p256_sub(res_x, res_x, Hcub); - - add x2,sp,#96 - ldr x3,[x22,#32] // forward load for p256_mul_mont - ldp x4,x5,[sp,#256] - ldp x6,x7,[sp,#256+16] - add x0,sp,#32 - bl __ecp_nistz256_sub_morf // p256_sub(res_y, U2, res_x); - - add x2,x22,#32 - add x0,sp,#128 - bl __ecp_nistz256_mul_mont // p256_mul_mont(S2, in1_y, Hcub); - - ldr x3,[sp,#192] - ldp x4,x5,[sp,#32] - ldp x6,x7,[sp,#32+16] - add x2,sp,#192 - add x0,sp,#32 - bl __ecp_nistz256_mul_mont // p256_mul_mont(res_y, res_y, R); - - add x2,sp,#128 - bl __ecp_nistz256_sub_from // p256_sub(res_y, res_y, S2); - - ldp x4,x5,[sp,#0] // res - ldp x6,x7,[sp,#0+16] - ldp x8,x9,[x23] // in2 - ldp x10,x11,[x23,#16] - ldp x14,x15,[x22,#0] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#0+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+0+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+0+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#0+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#0+48] - stp x14,x15,[x21,#0] - stp x16,x17,[x21,#0+16] - adrp x23,Lone_mont-64 - add x23,x23,:lo12:Lone_mont-64 - ldp x14,x15,[x22,#32] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#32+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - ldp x4,x5,[sp,#0+32+32] // res - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - ldp x6,x7,[sp,#0+32+48] - csel x14,x8,x14,ne - csel x15,x9,x15,ne - ldp x8,x9,[x23,#32+32] // in2 - csel x16,x10,x16,ne - csel x17,x11,x17,ne - ldp x10,x11,[x23,#32+48] - stp x14,x15,[x21,#32] - stp x16,x17,[x21,#32+16] - ldp x14,x15,[x22,#64] // in1 - cmp x24,#0 // ~, remember? - ldp x16,x17,[x22,#64+16] - csel x8,x4,x8,ne - csel x9,x5,x9,ne - csel x10,x6,x10,ne - csel x11,x7,x11,ne - cmp x25,#0 // ~, remember? - csel x14,x8,x14,ne - csel x15,x9,x15,ne - csel x16,x10,x16,ne - csel x17,x11,x17,ne - stp x14,x15,[x21,#64] - stp x16,x17,[x21,#64+16] - - add sp,x29,#0 // destroy frame - ldp x19,x20,[x29,#16] - ldp x21,x22,[x29,#32] - ldp x23,x24,[x29,#48] - ldp x25,x26,[x29,#64] - ldp x29,x30,[sp],#80 - AARCH64_VALIDATE_LINK_REGISTER - ret - -//////////////////////////////////////////////////////////////////////// // void ecp_nistz256_ord_mul_mont(uint64_t res[4], uint64_t a[4], // uint64_t b[4]); .globl ecp_nistz256_ord_mul_mont @@ -1259,7 +321,7 @@ ldp x12,x13,[x23,#0] ldp x21,x22,[x23,#16] - ldr x23,[x23,#32] + ldr x23,[x23,#32] // LordK mul x14,x4,x3 // a[0]*b[0] umulh x8,x4,x3 @@ -1642,123 +704,4 @@ ldr x29,[sp],#64 ret -//////////////////////////////////////////////////////////////////////// -// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w5 - -.def ecp_nistz256_select_w5 - .type 32 -.endef -.align 4 -ecp_nistz256_select_w5: - AARCH64_VALID_CALL_TARGET - - // x10 := x0 - // w9 := 0; loop counter and incremented internal index - mov x10, x0 - mov w9, #0 - - // [v16-v21] := 0 - movi v16.16b, #0 - movi v17.16b, #0 - movi v18.16b, #0 - movi v19.16b, #0 - movi v20.16b, #0 - movi v21.16b, #0 - -Lselect_w5_loop: - // Loop 16 times. - - // Increment index (loop counter); tested at the end of the loop - add w9, w9, #1 - - // [v22-v27] := Load a (3*256-bit = 6*128-bit) table entry starting at x1 - // and advance x1 to point to the next entry - ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 - - // x11 := (w9 == w2)? All 1s : All 0s - cmp w9, w2 - csetm x11, eq - - // continue loading ... - ld1 {v26.2d, v27.2d}, [x1],#32 - - // duplicate mask_64 into Mask (all 0s or all 1s) - dup v3.2d, x11 - - // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] - // i.e., values in output registers will remain the same if w9 != w2 - bit v16.16b, v22.16b, v3.16b - bit v17.16b, v23.16b, v3.16b - - bit v18.16b, v24.16b, v3.16b - bit v19.16b, v25.16b, v3.16b - - bit v20.16b, v26.16b, v3.16b - bit v21.16b, v27.16b, v3.16b - - // If bit #4 is not 0 (i.e. idx_ctr < 16) loop back - tbz w9, #4, Lselect_w5_loop - - // Write [v16-v21] to memory at the output pointer - st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x10],#64 - st1 {v20.2d, v21.2d}, [x10] - - ret - - - -//////////////////////////////////////////////////////////////////////// -// void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index); -.globl ecp_nistz256_select_w7 - -.def ecp_nistz256_select_w7 - .type 32 -.endef -.align 4 -ecp_nistz256_select_w7: - AARCH64_VALID_CALL_TARGET - - // w9 := 0; loop counter and incremented internal index - mov w9, #0 - - // [v16-v21] := 0 - movi v16.16b, #0 - movi v17.16b, #0 - movi v18.16b, #0 - movi v19.16b, #0 - -Lselect_w7_loop: - // Loop 64 times. - - // Increment index (loop counter); tested at the end of the loop - add w9, w9, #1 - - // [v22-v25] := Load a (2*256-bit = 4*128-bit) table entry starting at x1 - // and advance x1 to point to the next entry - ld1 {v22.2d, v23.2d, v24.2d, v25.2d}, [x1],#64 - - // x11 := (w9 == w2)? All 1s : All 0s - cmp w9, w2 - csetm x11, eq - - // duplicate mask_64 into Mask (all 0s or all 1s) - dup v3.2d, x11 - - // [v16-v19] := (Mask == all 1s)? [v22-v25] : [v16-v19] - // i.e., values in output registers will remain the same if w9 != w2 - bit v16.16b, v22.16b, v3.16b - bit v17.16b, v23.16b, v3.16b - - bit v18.16b, v24.16b, v3.16b - bit v19.16b, v25.16b, v3.16b - - // If bit #6 is not 0 (i.e. idx_ctr < 64) loop back - tbz w9, #6, Lselect_w7_loop - - // Write [v16-v19] to memory at the output pointer - st1 {v16.2d, v17.2d, v18.2d, v19.2d}, [x0] - - ret - #endif // !OPENSSL_NO_ASM && defined(OPENSSL_AARCH64) && defined(_WIN32)
diff --git a/gen/bcm/p256-x86_64-asm-apple.S b/gen/bcm/p256-x86_64-asm-apple.S index 80ffa01..6bde21e 100644 --- a/gen/bcm/p256-x86_64-asm-apple.S +++ b/gen/bcm/p256-x86_64-asm-apple.S
@@ -7,21 +7,6 @@ .text -.section __DATA,__const -.p2align 6 -L$poly: -.quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 - -L$One: -.long 1,1,1,1,1,1,1,1 -L$Two: -.long 2,2,2,2,2,2,2,2 -L$Three: -.long 3,3,3,3,3,3,3,3 -L$ONE_mont: -.quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe - - L$ord: .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 L$ordK: @@ -30,64 +15,6 @@ -.globl _ecp_nistz256_neg -.private_extern _ecp_nistz256_neg - -.p2align 5 -_ecp_nistz256_neg: - -_CET_ENDBR - pushq %r12 - - pushq %r13 - -L$neg_body: - - xorq %r8,%r8 - xorq %r9,%r9 - xorq %r10,%r10 - xorq %r11,%r11 - xorq %r13,%r13 - - subq 0(%rsi),%r8 - sbbq 8(%rsi),%r9 - sbbq 16(%rsi),%r10 - movq %r8,%rax - sbbq 24(%rsi),%r11 - leaq L$poly(%rip),%rsi - movq %r9,%rdx - sbbq $0,%r13 - - addq 0(%rsi),%r8 - movq %r10,%rcx - adcq 8(%rsi),%r9 - adcq 16(%rsi),%r10 - movq %r11,%r12 - adcq 24(%rsi),%r11 - testq %r13,%r13 - - cmovzq %rax,%r8 - cmovzq %rdx,%r9 - movq %r8,0(%rdi) - cmovzq %rcx,%r10 - movq %r9,8(%rdi) - cmovzq %r12,%r11 - movq %r10,16(%rdi) - movq %r11,24(%rdi) - - movq 0(%rsp),%r13 - - movq 8(%rsp),%r12 - - leaq 16(%rsp),%rsp - -L$neg_epilogue: - ret - - - - - @@ -1152,3362 +1079,4 @@ ret - - - - - - -.globl _ecp_nistz256_mul_mont_nohw -.private_extern _ecp_nistz256_mul_mont_nohw - -.p2align 5 -_ecp_nistz256_mul_mont_nohw: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$mul_body: - movq %rdx,%rbx - movq 0(%rdx),%rax - movq 0(%rsi),%r9 - movq 8(%rsi),%r10 - movq 16(%rsi),%r11 - movq 24(%rsi),%r12 - - call __ecp_nistz256_mul_montq - - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%r12 - - movq 32(%rsp),%rbx - - movq 40(%rsp),%rbp - - leaq 48(%rsp),%rsp - -L$mul_epilogue: - ret - - - - -.p2align 5 -__ecp_nistz256_mul_montq: - - - - movq %rax,%rbp - mulq %r9 - movq L$poly+8(%rip),%r14 - movq %rax,%r8 - movq %rbp,%rax - movq %rdx,%r9 - - mulq %r10 - movq L$poly+24(%rip),%r15 - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq %r11 - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq %r12 - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - xorq %r13,%r13 - movq %rdx,%r12 - - - - - - - - - - - movq %r8,%rbp - shlq $32,%r8 - mulq %r15 - shrq $32,%rbp - addq %r8,%r9 - adcq %rbp,%r10 - adcq %rax,%r11 - movq 8(%rbx),%rax - adcq %rdx,%r12 - adcq $0,%r13 - xorq %r8,%r8 - - - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 8(%rsi) - addq %rcx,%r10 - adcq $0,%rdx - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 16(%rsi) - addq %rcx,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 24(%rsi) - addq %rcx,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %r9,%rax - adcq %rdx,%r13 - adcq $0,%r8 - - - - movq %r9,%rbp - shlq $32,%r9 - mulq %r15 - shrq $32,%rbp - addq %r9,%r10 - adcq %rbp,%r11 - adcq %rax,%r12 - movq 16(%rbx),%rax - adcq %rdx,%r13 - adcq $0,%r8 - xorq %r9,%r9 - - - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 8(%rsi) - addq %rcx,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 16(%rsi) - addq %rcx,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 24(%rsi) - addq %rcx,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %r10,%rax - adcq %rdx,%r8 - adcq $0,%r9 - - - - movq %r10,%rbp - shlq $32,%r10 - mulq %r15 - shrq $32,%rbp - addq %r10,%r11 - adcq %rbp,%r12 - adcq %rax,%r13 - movq 24(%rbx),%rax - adcq %rdx,%r8 - adcq $0,%r9 - xorq %r10,%r10 - - - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 8(%rsi) - addq %rcx,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 16(%rsi) - addq %rcx,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 24(%rsi) - addq %rcx,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r11,%rax - adcq %rdx,%r9 - adcq $0,%r10 - - - - movq %r11,%rbp - shlq $32,%r11 - mulq %r15 - shrq $32,%rbp - addq %r11,%r12 - adcq %rbp,%r13 - movq %r12,%rcx - adcq %rax,%r8 - adcq %rdx,%r9 - movq %r13,%rbp - adcq $0,%r10 - - - - subq $-1,%r12 - movq %r8,%rbx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%rdx - sbbq %r15,%r9 - sbbq $0,%r10 - - cmovcq %rcx,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rbx,%r8 - movq %r13,8(%rdi) - cmovcq %rdx,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret - - - - - - - - - - -.globl _ecp_nistz256_sqr_mont_nohw -.private_extern _ecp_nistz256_sqr_mont_nohw - -.p2align 5 -_ecp_nistz256_sqr_mont_nohw: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$sqr_body: - movq 0(%rsi),%rax - movq 8(%rsi),%r14 - movq 16(%rsi),%r15 - movq 24(%rsi),%r8 - - call __ecp_nistz256_sqr_montq - - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%r12 - - movq 32(%rsp),%rbx - - movq 40(%rsp),%rbp - - leaq 48(%rsp),%rsp - -L$sqr_epilogue: - ret - - - - -.p2align 5 -__ecp_nistz256_sqr_montq: - - movq %rax,%r13 - mulq %r14 - movq %rax,%r9 - movq %r15,%rax - movq %rdx,%r10 - - mulq %r13 - addq %rax,%r10 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq %r13 - addq %rax,%r11 - movq %r15,%rax - adcq $0,%rdx - movq %rdx,%r12 - - - mulq %r14 - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%rbp - - mulq %r14 - addq %rax,%r12 - movq %r8,%rax - adcq $0,%rdx - addq %rbp,%r12 - movq %rdx,%r13 - adcq $0,%r13 - - - mulq %r15 - xorq %r15,%r15 - addq %rax,%r13 - movq 0(%rsi),%rax - movq %rdx,%r14 - adcq $0,%r14 - - addq %r9,%r9 - adcq %r10,%r10 - adcq %r11,%r11 - adcq %r12,%r12 - adcq %r13,%r13 - adcq %r14,%r14 - adcq $0,%r15 - - mulq %rax - movq %rax,%r8 - movq 8(%rsi),%rax - movq %rdx,%rcx - - mulq %rax - addq %rcx,%r9 - adcq %rax,%r10 - movq 16(%rsi),%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq %rax - addq %rcx,%r11 - adcq %rax,%r12 - movq 24(%rsi),%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq %rax - addq %rcx,%r13 - adcq %rax,%r14 - movq %r8,%rax - adcq %rdx,%r15 - - movq L$poly+8(%rip),%rsi - movq L$poly+24(%rip),%rbp - - - - - movq %r8,%rcx - shlq $32,%r8 - mulq %rbp - shrq $32,%rcx - addq %r8,%r9 - adcq %rcx,%r10 - adcq %rax,%r11 - movq %r9,%rax - adcq $0,%rdx - - - - movq %r9,%rcx - shlq $32,%r9 - movq %rdx,%r8 - mulq %rbp - shrq $32,%rcx - addq %r9,%r10 - adcq %rcx,%r11 - adcq %rax,%r8 - movq %r10,%rax - adcq $0,%rdx - - - - movq %r10,%rcx - shlq $32,%r10 - movq %rdx,%r9 - mulq %rbp - shrq $32,%rcx - addq %r10,%r11 - adcq %rcx,%r8 - adcq %rax,%r9 - movq %r11,%rax - adcq $0,%rdx - - - - movq %r11,%rcx - shlq $32,%r11 - movq %rdx,%r10 - mulq %rbp - shrq $32,%rcx - addq %r11,%r8 - adcq %rcx,%r9 - adcq %rax,%r10 - adcq $0,%rdx - xorq %r11,%r11 - - - - addq %r8,%r12 - adcq %r9,%r13 - movq %r12,%r8 - adcq %r10,%r14 - adcq %rdx,%r15 - movq %r13,%r9 - adcq $0,%r11 - - subq $-1,%r12 - movq %r14,%r10 - sbbq %rsi,%r13 - sbbq $0,%r14 - movq %r15,%rcx - sbbq %rbp,%r15 - sbbq $0,%r11 - - cmovcq %r8,%r12 - cmovcq %r9,%r13 - movq %r12,0(%rdi) - cmovcq %r10,%r14 - movq %r13,8(%rdi) - cmovcq %rcx,%r15 - movq %r14,16(%rdi) - movq %r15,24(%rdi) - - ret - - -.globl _ecp_nistz256_mul_mont_adx -.private_extern _ecp_nistz256_mul_mont_adx - -.p2align 5 -_ecp_nistz256_mul_mont_adx: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$mulx_body: - movq %rdx,%rbx - movq 0(%rdx),%rdx - movq 0(%rsi),%r9 - movq 8(%rsi),%r10 - movq 16(%rsi),%r11 - movq 24(%rsi),%r12 - leaq -128(%rsi),%rsi - - call __ecp_nistz256_mul_montx - - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%r12 - - movq 32(%rsp),%rbx - - movq 40(%rsp),%rbp - - leaq 48(%rsp),%rsp - -L$mulx_epilogue: - ret - - - - -.p2align 5 -__ecp_nistz256_mul_montx: - - - - mulxq %r9,%r8,%r9 - mulxq %r10,%rcx,%r10 - movq $32,%r14 - xorq %r13,%r13 - mulxq %r11,%rbp,%r11 - movq L$poly+24(%rip),%r15 - adcq %rcx,%r9 - mulxq %r12,%rcx,%r12 - movq %r8,%rdx - adcq %rbp,%r10 - shlxq %r14,%r8,%rbp - adcq %rcx,%r11 - shrxq %r14,%r8,%rcx - adcq $0,%r12 - - - - addq %rbp,%r9 - adcq %rcx,%r10 - - mulxq %r15,%rcx,%rbp - movq 8(%rbx),%rdx - adcq %rcx,%r11 - adcq %rbp,%r12 - adcq $0,%r13 - xorq %r8,%r8 - - - - mulxq 0+128(%rsi),%rcx,%rbp - adcxq %rcx,%r9 - adoxq %rbp,%r10 - - mulxq 8+128(%rsi),%rcx,%rbp - adcxq %rcx,%r10 - adoxq %rbp,%r11 - - mulxq 16+128(%rsi),%rcx,%rbp - adcxq %rcx,%r11 - adoxq %rbp,%r12 - - mulxq 24+128(%rsi),%rcx,%rbp - movq %r9,%rdx - adcxq %rcx,%r12 - shlxq %r14,%r9,%rcx - adoxq %rbp,%r13 - shrxq %r14,%r9,%rbp - - adcxq %r8,%r13 - adoxq %r8,%r8 - adcq $0,%r8 - - - - addq %rcx,%r10 - adcq %rbp,%r11 - - mulxq %r15,%rcx,%rbp - movq 16(%rbx),%rdx - adcq %rcx,%r12 - adcq %rbp,%r13 - adcq $0,%r8 - xorq %r9,%r9 - - - - mulxq 0+128(%rsi),%rcx,%rbp - adcxq %rcx,%r10 - adoxq %rbp,%r11 - - mulxq 8+128(%rsi),%rcx,%rbp - adcxq %rcx,%r11 - adoxq %rbp,%r12 - - mulxq 16+128(%rsi),%rcx,%rbp - adcxq %rcx,%r12 - adoxq %rbp,%r13 - - mulxq 24+128(%rsi),%rcx,%rbp - movq %r10,%rdx - adcxq %rcx,%r13 - shlxq %r14,%r10,%rcx - adoxq %rbp,%r8 - shrxq %r14,%r10,%rbp - - adcxq %r9,%r8 - adoxq %r9,%r9 - adcq $0,%r9 - - - - addq %rcx,%r11 - adcq %rbp,%r12 - - mulxq %r15,%rcx,%rbp - movq 24(%rbx),%rdx - adcq %rcx,%r13 - adcq %rbp,%r8 - adcq $0,%r9 - xorq %r10,%r10 - - - - mulxq 0+128(%rsi),%rcx,%rbp - adcxq %rcx,%r11 - adoxq %rbp,%r12 - - mulxq 8+128(%rsi),%rcx,%rbp - adcxq %rcx,%r12 - adoxq %rbp,%r13 - - mulxq 16+128(%rsi),%rcx,%rbp - adcxq %rcx,%r13 - adoxq %rbp,%r8 - - mulxq 24+128(%rsi),%rcx,%rbp - movq %r11,%rdx - adcxq %rcx,%r8 - shlxq %r14,%r11,%rcx - adoxq %rbp,%r9 - shrxq %r14,%r11,%rbp - - adcxq %r10,%r9 - adoxq %r10,%r10 - adcq $0,%r10 - - - - addq %rcx,%r12 - adcq %rbp,%r13 - - mulxq %r15,%rcx,%rbp - movq %r12,%rbx - movq L$poly+8(%rip),%r14 - adcq %rcx,%r8 - movq %r13,%rdx - adcq %rbp,%r9 - adcq $0,%r10 - - - - xorl %eax,%eax - movq %r8,%rcx - sbbq $-1,%r12 - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%rbp - sbbq %r15,%r9 - sbbq $0,%r10 - - cmovcq %rbx,%r12 - cmovcq %rdx,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %rbp,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret - - - -.globl _ecp_nistz256_sqr_mont_adx -.private_extern _ecp_nistz256_sqr_mont_adx - -.p2align 5 -_ecp_nistz256_sqr_mont_adx: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - -L$sqrx_body: - movq 0(%rsi),%rdx - movq 8(%rsi),%r14 - movq 16(%rsi),%r15 - movq 24(%rsi),%r8 - leaq -128(%rsi),%rsi - - call __ecp_nistz256_sqr_montx - - movq 0(%rsp),%r15 - - movq 8(%rsp),%r14 - - movq 16(%rsp),%r13 - - movq 24(%rsp),%r12 - - movq 32(%rsp),%rbx - - movq 40(%rsp),%rbp - - leaq 48(%rsp),%rsp - -L$sqrx_epilogue: - ret - - - - -.p2align 5 -__ecp_nistz256_sqr_montx: - - mulxq %r14,%r9,%r10 - mulxq %r15,%rcx,%r11 - xorl %eax,%eax - adcq %rcx,%r10 - mulxq %r8,%rbp,%r12 - movq %r14,%rdx - adcq %rbp,%r11 - adcq $0,%r12 - xorq %r13,%r13 - - - mulxq %r15,%rcx,%rbp - adcxq %rcx,%r11 - adoxq %rbp,%r12 - - mulxq %r8,%rcx,%rbp - movq %r15,%rdx - adcxq %rcx,%r12 - adoxq %rbp,%r13 - adcq $0,%r13 - - - mulxq %r8,%rcx,%r14 - movq 0+128(%rsi),%rdx - xorq %r15,%r15 - adcxq %r9,%r9 - adoxq %rcx,%r13 - adcxq %r10,%r10 - adoxq %r15,%r14 - - mulxq %rdx,%r8,%rbp - movq 8+128(%rsi),%rdx - adcxq %r11,%r11 - adoxq %rbp,%r9 - adcxq %r12,%r12 - mulxq %rdx,%rcx,%rax - movq 16+128(%rsi),%rdx - adcxq %r13,%r13 - adoxq %rcx,%r10 - adcxq %r14,%r14 -.byte 0x67 - mulxq %rdx,%rcx,%rbp - movq 24+128(%rsi),%rdx - adoxq %rax,%r11 - adcxq %r15,%r15 - adoxq %rcx,%r12 - movq $32,%rsi - adoxq %rbp,%r13 -.byte 0x67,0x67 - mulxq %rdx,%rcx,%rax - movq L$poly+24(%rip),%rdx - adoxq %rcx,%r14 - shlxq %rsi,%r8,%rcx - adoxq %rax,%r15 - shrxq %rsi,%r8,%rax - movq %rdx,%rbp - - - addq %rcx,%r9 - adcq %rax,%r10 - - mulxq %r8,%rcx,%r8 - adcq %rcx,%r11 - shlxq %rsi,%r9,%rcx - adcq $0,%r8 - shrxq %rsi,%r9,%rax - - - addq %rcx,%r10 - adcq %rax,%r11 - - mulxq %r9,%rcx,%r9 - adcq %rcx,%r8 - shlxq %rsi,%r10,%rcx - adcq $0,%r9 - shrxq %rsi,%r10,%rax - - - addq %rcx,%r11 - adcq %rax,%r8 - - mulxq %r10,%rcx,%r10 - adcq %rcx,%r9 - shlxq %rsi,%r11,%rcx - adcq $0,%r10 - shrxq %rsi,%r11,%rax - - - addq %rcx,%r8 - adcq %rax,%r9 - - mulxq %r11,%rcx,%r11 - adcq %rcx,%r10 - adcq $0,%r11 - - xorq %rdx,%rdx - addq %r8,%r12 - movq L$poly+8(%rip),%rsi - adcq %r9,%r13 - movq %r12,%r8 - adcq %r10,%r14 - adcq %r11,%r15 - movq %r13,%r9 - adcq $0,%rdx - - subq $-1,%r12 - movq %r14,%r10 - sbbq %rsi,%r13 - sbbq $0,%r14 - movq %r15,%r11 - sbbq %rbp,%r15 - sbbq $0,%rdx - - cmovcq %r8,%r12 - cmovcq %r9,%r13 - movq %r12,0(%rdi) - cmovcq %r10,%r14 - movq %r13,8(%rdi) - cmovcq %r11,%r15 - movq %r14,16(%rdi) - movq %r15,24(%rdi) - - ret - - - - -.globl _ecp_nistz256_select_w5_nohw -.private_extern _ecp_nistz256_select_w5_nohw - -.p2align 5 -_ecp_nistz256_select_w5_nohw: - -_CET_ENDBR - movdqa L$One(%rip),%xmm0 - movd %edx,%xmm1 - - pxor %xmm2,%xmm2 - pxor %xmm3,%xmm3 - pxor %xmm4,%xmm4 - pxor %xmm5,%xmm5 - pxor %xmm6,%xmm6 - pxor %xmm7,%xmm7 - - movdqa %xmm0,%xmm8 - pshufd $0,%xmm1,%xmm1 - - movq $16,%rax -L$select_loop_sse_w5: - - movdqa %xmm8,%xmm15 - paddd %xmm0,%xmm8 - pcmpeqd %xmm1,%xmm15 - - movdqa 0(%rsi),%xmm9 - movdqa 16(%rsi),%xmm10 - movdqa 32(%rsi),%xmm11 - movdqa 48(%rsi),%xmm12 - movdqa 64(%rsi),%xmm13 - movdqa 80(%rsi),%xmm14 - leaq 96(%rsi),%rsi - - pand %xmm15,%xmm9 - pand %xmm15,%xmm10 - por %xmm9,%xmm2 - pand %xmm15,%xmm11 - por %xmm10,%xmm3 - pand %xmm15,%xmm12 - por %xmm11,%xmm4 - pand %xmm15,%xmm13 - por %xmm12,%xmm5 - pand %xmm15,%xmm14 - por %xmm13,%xmm6 - por %xmm14,%xmm7 - - decq %rax - jnz L$select_loop_sse_w5 - - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - movdqu %xmm4,32(%rdi) - movdqu %xmm5,48(%rdi) - movdqu %xmm6,64(%rdi) - movdqu %xmm7,80(%rdi) - ret - -L$SEH_end_ecp_nistz256_select_w5_nohw: - - - - -.globl _ecp_nistz256_select_w7_nohw -.private_extern _ecp_nistz256_select_w7_nohw - -.p2align 5 -_ecp_nistz256_select_w7_nohw: - -_CET_ENDBR - movdqa L$One(%rip),%xmm8 - movd %edx,%xmm1 - - pxor %xmm2,%xmm2 - pxor %xmm3,%xmm3 - pxor %xmm4,%xmm4 - pxor %xmm5,%xmm5 - - movdqa %xmm8,%xmm0 - pshufd $0,%xmm1,%xmm1 - movq $64,%rax - -L$select_loop_sse_w7: - movdqa %xmm8,%xmm15 - paddd %xmm0,%xmm8 - movdqa 0(%rsi),%xmm9 - movdqa 16(%rsi),%xmm10 - pcmpeqd %xmm1,%xmm15 - movdqa 32(%rsi),%xmm11 - movdqa 48(%rsi),%xmm12 - leaq 64(%rsi),%rsi - - pand %xmm15,%xmm9 - pand %xmm15,%xmm10 - por %xmm9,%xmm2 - pand %xmm15,%xmm11 - por %xmm10,%xmm3 - pand %xmm15,%xmm12 - por %xmm11,%xmm4 - prefetcht0 255(%rsi) - por %xmm12,%xmm5 - - decq %rax - jnz L$select_loop_sse_w7 - - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - movdqu %xmm4,32(%rdi) - movdqu %xmm5,48(%rdi) - ret - -L$SEH_end_ecp_nistz256_select_w7_nohw: - - - -.globl _ecp_nistz256_select_w5_avx2 -.private_extern _ecp_nistz256_select_w5_avx2 - -.p2align 5 -_ecp_nistz256_select_w5_avx2: - -_CET_ENDBR - vzeroupper - vmovdqa L$Two(%rip),%ymm0 - - vpxor %ymm2,%ymm2,%ymm2 - vpxor %ymm3,%ymm3,%ymm3 - vpxor %ymm4,%ymm4,%ymm4 - - vmovdqa L$One(%rip),%ymm5 - vmovdqa L$Two(%rip),%ymm10 - - vmovd %edx,%xmm1 - vpermd %ymm1,%ymm2,%ymm1 - - movq $8,%rax -L$select_loop_avx2_w5: - - vmovdqa 0(%rsi),%ymm6 - vmovdqa 32(%rsi),%ymm7 - vmovdqa 64(%rsi),%ymm8 - - vmovdqa 96(%rsi),%ymm11 - vmovdqa 128(%rsi),%ymm12 - vmovdqa 160(%rsi),%ymm13 - - vpcmpeqd %ymm1,%ymm5,%ymm9 - vpcmpeqd %ymm1,%ymm10,%ymm14 - - vpaddd %ymm0,%ymm5,%ymm5 - vpaddd %ymm0,%ymm10,%ymm10 - leaq 192(%rsi),%rsi - - vpand %ymm9,%ymm6,%ymm6 - vpand %ymm9,%ymm7,%ymm7 - vpand %ymm9,%ymm8,%ymm8 - vpand %ymm14,%ymm11,%ymm11 - vpand %ymm14,%ymm12,%ymm12 - vpand %ymm14,%ymm13,%ymm13 - - vpxor %ymm6,%ymm2,%ymm2 - vpxor %ymm7,%ymm3,%ymm3 - vpxor %ymm8,%ymm4,%ymm4 - vpxor %ymm11,%ymm2,%ymm2 - vpxor %ymm12,%ymm3,%ymm3 - vpxor %ymm13,%ymm4,%ymm4 - - decq %rax - jnz L$select_loop_avx2_w5 - - vmovdqu %ymm2,0(%rdi) - vmovdqu %ymm3,32(%rdi) - vmovdqu %ymm4,64(%rdi) - vzeroupper - ret - -L$SEH_end_ecp_nistz256_select_w5_avx2: - - - - -.globl _ecp_nistz256_select_w7_avx2 -.private_extern _ecp_nistz256_select_w7_avx2 - -.p2align 5 -_ecp_nistz256_select_w7_avx2: - -_CET_ENDBR - vzeroupper - vmovdqa L$Three(%rip),%ymm0 - - vpxor %ymm2,%ymm2,%ymm2 - vpxor %ymm3,%ymm3,%ymm3 - - vmovdqa L$One(%rip),%ymm4 - vmovdqa L$Two(%rip),%ymm8 - vmovdqa L$Three(%rip),%ymm12 - - vmovd %edx,%xmm1 - vpermd %ymm1,%ymm2,%ymm1 - - - movq $21,%rax -L$select_loop_avx2_w7: - - vmovdqa 0(%rsi),%ymm5 - vmovdqa 32(%rsi),%ymm6 - - vmovdqa 64(%rsi),%ymm9 - vmovdqa 96(%rsi),%ymm10 - - vmovdqa 128(%rsi),%ymm13 - vmovdqa 160(%rsi),%ymm14 - - vpcmpeqd %ymm1,%ymm4,%ymm7 - vpcmpeqd %ymm1,%ymm8,%ymm11 - vpcmpeqd %ymm1,%ymm12,%ymm15 - - vpaddd %ymm0,%ymm4,%ymm4 - vpaddd %ymm0,%ymm8,%ymm8 - vpaddd %ymm0,%ymm12,%ymm12 - leaq 192(%rsi),%rsi - - vpand %ymm7,%ymm5,%ymm5 - vpand %ymm7,%ymm6,%ymm6 - vpand %ymm11,%ymm9,%ymm9 - vpand %ymm11,%ymm10,%ymm10 - vpand %ymm15,%ymm13,%ymm13 - vpand %ymm15,%ymm14,%ymm14 - - vpxor %ymm5,%ymm2,%ymm2 - vpxor %ymm6,%ymm3,%ymm3 - vpxor %ymm9,%ymm2,%ymm2 - vpxor %ymm10,%ymm3,%ymm3 - vpxor %ymm13,%ymm2,%ymm2 - vpxor %ymm14,%ymm3,%ymm3 - - decq %rax - jnz L$select_loop_avx2_w7 - - - vmovdqa 0(%rsi),%ymm5 - vmovdqa 32(%rsi),%ymm6 - - vpcmpeqd %ymm1,%ymm4,%ymm7 - - vpand %ymm7,%ymm5,%ymm5 - vpand %ymm7,%ymm6,%ymm6 - - vpxor %ymm5,%ymm2,%ymm2 - vpxor %ymm6,%ymm3,%ymm3 - - vmovdqu %ymm2,0(%rdi) - vmovdqu %ymm3,32(%rdi) - vzeroupper - ret - -L$SEH_end_ecp_nistz256_select_w7_avx2: - - -.p2align 5 -__ecp_nistz256_add_toq: - - xorq %r11,%r11 - addq 0(%rbx),%r12 - adcq 8(%rbx),%r13 - movq %r12,%rax - adcq 16(%rbx),%r8 - adcq 24(%rbx),%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret - - - - -.p2align 5 -__ecp_nistz256_sub_fromq: - - subq 0(%rbx),%r12 - sbbq 8(%rbx),%r13 - movq %r12,%rax - sbbq 16(%rbx),%r8 - sbbq 24(%rbx),%r9 - movq %r13,%rbp - sbbq %r11,%r11 - - addq $-1,%r12 - movq %r8,%rcx - adcq %r14,%r13 - adcq $0,%r8 - movq %r9,%r10 - adcq %r15,%r9 - testq %r11,%r11 - - cmovzq %rax,%r12 - cmovzq %rbp,%r13 - movq %r12,0(%rdi) - cmovzq %rcx,%r8 - movq %r13,8(%rdi) - cmovzq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret - - - - -.p2align 5 -__ecp_nistz256_subq: - - subq %r12,%rax - sbbq %r13,%rbp - movq %rax,%r12 - sbbq %r8,%rcx - sbbq %r9,%r10 - movq %rbp,%r13 - sbbq %r11,%r11 - - addq $-1,%rax - movq %rcx,%r8 - adcq %r14,%rbp - adcq $0,%rcx - movq %r10,%r9 - adcq %r15,%r10 - testq %r11,%r11 - - cmovnzq %rax,%r12 - cmovnzq %rbp,%r13 - cmovnzq %rcx,%r8 - cmovnzq %r10,%r9 - - ret - - - - -.p2align 5 -__ecp_nistz256_mul_by_2q: - - xorq %r11,%r11 - addq %r12,%r12 - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret - - -.globl _ecp_nistz256_point_double_nohw -.private_extern _ecp_nistz256_point_double_nohw - -.p2align 5 -_ecp_nistz256_point_double_nohw: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $160+8,%rsp - -L$point_doubleq_body: - -L$point_double_shortcutq: - movdqu 0(%rsi),%xmm0 - movq %rsi,%rbx - movdqu 16(%rsi),%xmm1 - movq 32+0(%rsi),%r12 - movq 32+8(%rsi),%r13 - movq 32+16(%rsi),%r8 - movq 32+24(%rsi),%r9 - movq L$poly+8(%rip),%r14 - movq L$poly+24(%rip),%r15 - movdqa %xmm0,96(%rsp) - movdqa %xmm1,96+16(%rsp) - leaq 32(%rdi),%r10 - leaq 64(%rdi),%r11 - movq %rdi,%xmm0 - movq %r10,%xmm1 - movq %r11,%xmm2 - - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_by_2q - - movq 64+0(%rsi),%rax - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - leaq 64-0(%rsi),%rsi - leaq 64(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 0+0(%rsp),%rax - movq 8+0(%rsp),%r14 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - leaq 0(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 32(%rbx),%rax - movq 64+0(%rbx),%r9 - movq 64+8(%rbx),%r10 - movq 64+16(%rbx),%r11 - movq 64+24(%rbx),%r12 - leaq 64-0(%rbx),%rsi - leaq 32(%rbx),%rbx - movq %xmm2,%rdi - call __ecp_nistz256_mul_montq - call __ecp_nistz256_mul_by_2q - - movq 96+0(%rsp),%r12 - movq 96+8(%rsp),%r13 - leaq 64(%rsp),%rbx - movq 96+16(%rsp),%r8 - movq 96+24(%rsp),%r9 - leaq 32(%rsp),%rdi - call __ecp_nistz256_add_toq - - movq 96+0(%rsp),%r12 - movq 96+8(%rsp),%r13 - leaq 64(%rsp),%rbx - movq 96+16(%rsp),%r8 - movq 96+24(%rsp),%r9 - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 0+0(%rsp),%rax - movq 8+0(%rsp),%r14 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - movq %xmm1,%rdi - call __ecp_nistz256_sqr_montq - xorq %r9,%r9 - movq %r12,%rax - addq $-1,%r12 - movq %r13,%r10 - adcq %rsi,%r13 - movq %r14,%rcx - adcq $0,%r14 - movq %r15,%r8 - adcq %rbp,%r15 - adcq $0,%r9 - xorq %rsi,%rsi - testq $1,%rax - - cmovzq %rax,%r12 - cmovzq %r10,%r13 - cmovzq %rcx,%r14 - cmovzq %r8,%r15 - cmovzq %rsi,%r9 - - movq %r13,%rax - shrq $1,%r12 - shlq $63,%rax - movq %r14,%r10 - shrq $1,%r13 - orq %rax,%r12 - shlq $63,%r10 - movq %r15,%rcx - shrq $1,%r14 - orq %r10,%r13 - shlq $63,%rcx - movq %r12,0(%rdi) - shrq $1,%r15 - movq %r13,8(%rdi) - shlq $63,%r9 - orq %rcx,%r14 - orq %r9,%r15 - movq %r14,16(%rdi) - movq %r15,24(%rdi) - movq 64(%rsp),%rax - leaq 64(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2q - - leaq 32(%rsp),%rbx - leaq 32(%rsp),%rdi - call __ecp_nistz256_add_toq - - movq 96(%rsp),%rax - leaq 96(%rsp),%rbx - movq 0+0(%rsp),%r9 - movq 8+0(%rsp),%r10 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r11 - movq 24+0(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2q - - movq 0+32(%rsp),%rax - movq 8+32(%rsp),%r14 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r15 - movq 24+32(%rsp),%r8 - movq %xmm0,%rdi - call __ecp_nistz256_sqr_montq - - leaq 128(%rsp),%rbx - movq %r14,%r8 - movq %r15,%r9 - movq %rsi,%r14 - movq %rbp,%r15 - call __ecp_nistz256_sub_fromq - - movq 0+0(%rsp),%rax - movq 0+8(%rsp),%rbp - movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 - leaq 0(%rsp),%rdi - call __ecp_nistz256_subq - - movq 32(%rsp),%rax - leaq 32(%rsp),%rbx - movq %r12,%r14 - xorl %ecx,%ecx - movq %r12,0+0(%rsp) - movq %r13,%r10 - movq %r13,0+8(%rsp) - cmovzq %r8,%r11 - movq %r8,0+16(%rsp) - leaq 0-0(%rsp),%rsi - cmovzq %r9,%r12 - movq %r9,0+24(%rsp) - movq %r14,%r9 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq %xmm1,%rbx - movq %xmm1,%rdi - call __ecp_nistz256_sub_fromq - - leaq 160+56(%rsp),%rsi - - movq -48(%rsi),%r15 - - movq -40(%rsi),%r14 - - movq -32(%rsi),%r13 - - movq -24(%rsi),%r12 - - movq -16(%rsi),%rbx - - movq -8(%rsi),%rbp - - leaq (%rsi),%rsp - -L$point_doubleq_epilogue: - ret - - -.globl _ecp_nistz256_point_add_nohw -.private_extern _ecp_nistz256_point_add_nohw - -.p2align 5 -_ecp_nistz256_point_add_nohw: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $576+8,%rsp - -L$point_addq_body: - - movdqu 0(%rsi),%xmm0 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - movdqu 64(%rsi),%xmm4 - movdqu 80(%rsi),%xmm5 - movq %rsi,%rbx - movq %rdx,%rsi - movdqa %xmm0,384(%rsp) - movdqa %xmm1,384+16(%rsp) - movdqa %xmm2,416(%rsp) - movdqa %xmm3,416+16(%rsp) - movdqa %xmm4,448(%rsp) - movdqa %xmm5,448+16(%rsp) - por %xmm4,%xmm5 - - movdqu 0(%rsi),%xmm0 - pshufd $0xb1,%xmm5,%xmm3 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - por %xmm3,%xmm5 - movdqu 48(%rsi),%xmm3 - movq 64+0(%rsi),%rax - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - movdqa %xmm0,480(%rsp) - pshufd $0x1e,%xmm5,%xmm4 - movdqa %xmm1,480+16(%rsp) - movdqu 64(%rsi),%xmm0 - movdqu 80(%rsi),%xmm1 - movdqa %xmm2,512(%rsp) - movdqa %xmm3,512+16(%rsp) - por %xmm4,%xmm5 - pxor %xmm4,%xmm4 - por %xmm0,%xmm1 - movq %rdi,%xmm0 - - leaq 64-0(%rsi),%rsi - movq %rax,544+0(%rsp) - movq %r14,544+8(%rsp) - movq %r15,544+16(%rsp) - movq %r8,544+24(%rsp) - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - pcmpeqd %xmm4,%xmm5 - pshufd $0xb1,%xmm1,%xmm4 - por %xmm1,%xmm4 - pshufd $0,%xmm5,%xmm5 - pshufd $0x1e,%xmm4,%xmm3 - por %xmm3,%xmm4 - pxor %xmm3,%xmm3 - pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - movq 64+0(%rbx),%rax - movq 64+8(%rbx),%r14 - movq 64+16(%rbx),%r15 - movq 64+24(%rbx),%r8 - movq %rbx,%xmm1 - - leaq 64-0(%rbx),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 544(%rsp),%rax - leaq 544(%rsp),%rbx - movq 0+96(%rsp),%r9 - movq 8+96(%rsp),%r10 - leaq 0+96(%rsp),%rsi - movq 16+96(%rsp),%r11 - movq 24+96(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 448(%rsp),%rax - leaq 448(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 416(%rsp),%rax - leaq 416(%rsp),%rbx - movq 0+224(%rsp),%r9 - movq 8+224(%rsp),%r10 - leaq 0+224(%rsp),%rsi - movq 16+224(%rsp),%r11 - movq 24+224(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 512(%rsp),%rax - leaq 512(%rsp),%rbx - movq 0+256(%rsp),%r9 - movq 8+256(%rsp),%r10 - leaq 0+256(%rsp),%rsi - movq 16+256(%rsp),%r11 - movq 24+256(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 224(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - orq %r13,%r12 - movdqa %xmm4,%xmm2 - orq %r8,%r12 - orq %r9,%r12 - por %xmm5,%xmm2 - movq %r12,%xmm3 - - movq 384(%rsp),%rax - leaq 384(%rsp),%rbx - movq 0+96(%rsp),%r9 - movq 8+96(%rsp),%r10 - leaq 0+96(%rsp),%rsi - movq 16+96(%rsp),%r11 - movq 24+96(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 480(%rsp),%rax - leaq 480(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 160(%rsp),%rbx - leaq 0(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - orq %r13,%r12 - orq %r8,%r12 - orq %r9,%r12 - - movq %xmm2,%r8 - movq %xmm3,%r9 - orq %r8,%r12 -.byte 0x3e - jnz L$add_proceedq - - - - testq %r9,%r9 - jz L$add_doubleq - - - - - - - movq %xmm0,%rdi - pxor %xmm0,%xmm0 - movdqu %xmm0,0(%rdi) - movdqu %xmm0,16(%rdi) - movdqu %xmm0,32(%rdi) - movdqu %xmm0,48(%rdi) - movdqu %xmm0,64(%rdi) - movdqu %xmm0,80(%rdi) - jmp L$add_doneq - -.p2align 5 -L$add_doubleq: - movq %xmm1,%rsi - movq %xmm0,%rdi - addq $416,%rsp - - jmp L$point_double_shortcutq - - -.p2align 5 -L$add_proceedq: - movq 0+64(%rsp),%rax - movq 8+64(%rsp),%r14 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r15 - movq 24+64(%rsp),%r8 - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 448(%rsp),%rax - leaq 448(%rsp),%rbx - movq 0+0(%rsp),%r9 - movq 8+0(%rsp),%r10 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r11 - movq 24+0(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 0+0(%rsp),%rax - movq 8+0(%rsp),%r14 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 544(%rsp),%rax - leaq 544(%rsp),%rbx - movq 0+352(%rsp),%r9 - movq 8+352(%rsp),%r10 - leaq 0+352(%rsp),%rsi - movq 16+352(%rsp),%r11 - movq 24+352(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 0(%rsp),%rax - leaq 0(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 160(%rsp),%rax - leaq 160(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montq - - - - - xorq %r11,%r11 - addq %r12,%r12 - leaq 96(%rsp),%rsi - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - movq 0(%rsi),%rax - cmovcq %rbp,%r13 - movq 8(%rsi),%rbp - cmovcq %rcx,%r8 - movq 16(%rsi),%rcx - cmovcq %r10,%r9 - movq 24(%rsi),%r10 - - call __ecp_nistz256_subq - - leaq 128(%rsp),%rbx - leaq 288(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 192+0(%rsp),%rax - movq 192+8(%rsp),%rbp - movq 192+16(%rsp),%rcx - movq 192+24(%rsp),%r10 - leaq 320(%rsp),%rdi - - call __ecp_nistz256_subq - - movq %r12,0(%rdi) - movq %r13,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq 128(%rsp),%rax - leaq 128(%rsp),%rbx - movq 0+224(%rsp),%r9 - movq 8+224(%rsp),%r10 - leaq 0+224(%rsp),%rsi - movq 16+224(%rsp),%r11 - movq 24+224(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 320(%rsp),%rax - leaq 320(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 320(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 256(%rsp),%rbx - leaq 320(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq %xmm0,%rdi - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 352(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 352+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 544(%rsp),%xmm2 - pand 544+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 448(%rsp),%xmm2 - pand 448+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,64(%rdi) - movdqu %xmm3,80(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 288(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 288+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 480(%rsp),%xmm2 - pand 480+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 384(%rsp),%xmm2 - pand 384+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 320(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 320+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 512(%rsp),%xmm2 - pand 512+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 416(%rsp),%xmm2 - pand 416+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - -L$add_doneq: - leaq 576+56(%rsp),%rsi - - movq -48(%rsi),%r15 - - movq -40(%rsi),%r14 - - movq -32(%rsi),%r13 - - movq -24(%rsi),%r12 - - movq -16(%rsi),%rbx - - movq -8(%rsi),%rbp - - leaq (%rsi),%rsp - -L$point_addq_epilogue: - ret - - -.globl _ecp_nistz256_point_add_affine_nohw -.private_extern _ecp_nistz256_point_add_affine_nohw - -.p2align 5 -_ecp_nistz256_point_add_affine_nohw: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $480+8,%rsp - -L$add_affineq_body: - - movdqu 0(%rsi),%xmm0 - movq %rdx,%rbx - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - movdqu 64(%rsi),%xmm4 - movdqu 80(%rsi),%xmm5 - movq 64+0(%rsi),%rax - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - movdqa %xmm0,320(%rsp) - movdqa %xmm1,320+16(%rsp) - movdqa %xmm2,352(%rsp) - movdqa %xmm3,352+16(%rsp) - movdqa %xmm4,384(%rsp) - movdqa %xmm5,384+16(%rsp) - por %xmm4,%xmm5 - - movdqu 0(%rbx),%xmm0 - pshufd $0xb1,%xmm5,%xmm3 - movdqu 16(%rbx),%xmm1 - movdqu 32(%rbx),%xmm2 - por %xmm3,%xmm5 - movdqu 48(%rbx),%xmm3 - movdqa %xmm0,416(%rsp) - pshufd $0x1e,%xmm5,%xmm4 - movdqa %xmm1,416+16(%rsp) - por %xmm0,%xmm1 - movq %rdi,%xmm0 - movdqa %xmm2,448(%rsp) - movdqa %xmm3,448+16(%rsp) - por %xmm2,%xmm3 - por %xmm4,%xmm5 - pxor %xmm4,%xmm4 - por %xmm1,%xmm3 - - leaq 64-0(%rsi),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - pcmpeqd %xmm4,%xmm5 - pshufd $0xb1,%xmm3,%xmm4 - movq 0(%rbx),%rax - - movq %r12,%r9 - por %xmm3,%xmm4 - pshufd $0,%xmm5,%xmm5 - pshufd $0x1e,%xmm4,%xmm3 - movq %r13,%r10 - por %xmm3,%xmm4 - pxor %xmm3,%xmm3 - movq %r14,%r11 - pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - - leaq 32-0(%rsp),%rsi - movq %r15,%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 320(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 384(%rsp),%rax - leaq 384(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 384(%rsp),%rax - leaq 384(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 288(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 448(%rsp),%rax - leaq 448(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 352(%rsp),%rbx - leaq 96(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 0+64(%rsp),%rax - movq 8+64(%rsp),%r14 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r15 - movq 24+64(%rsp),%r8 - leaq 128(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 0+96(%rsp),%rax - movq 8+96(%rsp),%r14 - leaq 0+96(%rsp),%rsi - movq 16+96(%rsp),%r15 - movq 24+96(%rsp),%r8 - leaq 192(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 128(%rsp),%rax - leaq 128(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 320(%rsp),%rax - leaq 320(%rsp),%rbx - movq 0+128(%rsp),%r9 - movq 8+128(%rsp),%r10 - leaq 0+128(%rsp),%rsi - movq 16+128(%rsp),%r11 - movq 24+128(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - - - - xorq %r11,%r11 - addq %r12,%r12 - leaq 192(%rsp),%rsi - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - movq 0(%rsi),%rax - cmovcq %rbp,%r13 - movq 8(%rsi),%rbp - cmovcq %rcx,%r8 - movq 16(%rsi),%rcx - cmovcq %r10,%r9 - movq 24(%rsi),%r10 - - call __ecp_nistz256_subq - - leaq 160(%rsp),%rbx - leaq 224(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 0+0(%rsp),%rax - movq 0+8(%rsp),%rbp - movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 - leaq 64(%rsp),%rdi - - call __ecp_nistz256_subq - - movq %r12,0(%rdi) - movq %r13,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq 352(%rsp),%rax - leaq 352(%rsp),%rbx - movq 0+160(%rsp),%r9 - movq 8+160(%rsp),%r10 - leaq 0+160(%rsp),%rsi - movq 16+160(%rsp),%r11 - movq 24+160(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 96(%rsp),%rax - leaq 96(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 64(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 32(%rsp),%rbx - leaq 256(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq %xmm0,%rdi - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 288(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 288+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand L$ONE_mont(%rip),%xmm2 - pand L$ONE_mont+16(%rip),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 384(%rsp),%xmm2 - pand 384+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,64(%rdi) - movdqu %xmm3,80(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 224(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 224+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 416(%rsp),%xmm2 - pand 416+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 320(%rsp),%xmm2 - pand 320+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 256(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 256+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 448(%rsp),%xmm2 - pand 448+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 352(%rsp),%xmm2 - pand 352+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - - leaq 480+56(%rsp),%rsi - - movq -48(%rsi),%r15 - - movq -40(%rsi),%r14 - - movq -32(%rsi),%r13 - - movq -24(%rsi),%r12 - - movq -16(%rsi),%rbx - - movq -8(%rsi),%rbp - - leaq (%rsi),%rsp - -L$add_affineq_epilogue: - ret - - - -.p2align 5 -__ecp_nistz256_add_tox: - - xorq %r11,%r11 - adcq 0(%rbx),%r12 - adcq 8(%rbx),%r13 - movq %r12,%rax - adcq 16(%rbx),%r8 - adcq 24(%rbx),%r9 - movq %r13,%rbp - adcq $0,%r11 - - xorq %r10,%r10 - sbbq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret - - - - -.p2align 5 -__ecp_nistz256_sub_fromx: - - xorq %r11,%r11 - sbbq 0(%rbx),%r12 - sbbq 8(%rbx),%r13 - movq %r12,%rax - sbbq 16(%rbx),%r8 - sbbq 24(%rbx),%r9 - movq %r13,%rbp - sbbq $0,%r11 - - xorq %r10,%r10 - adcq $-1,%r12 - movq %r8,%rcx - adcq %r14,%r13 - adcq $0,%r8 - movq %r9,%r10 - adcq %r15,%r9 - - btq $0,%r11 - cmovncq %rax,%r12 - cmovncq %rbp,%r13 - movq %r12,0(%rdi) - cmovncq %rcx,%r8 - movq %r13,8(%rdi) - cmovncq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret - - - - -.p2align 5 -__ecp_nistz256_subx: - - xorq %r11,%r11 - sbbq %r12,%rax - sbbq %r13,%rbp - movq %rax,%r12 - sbbq %r8,%rcx - sbbq %r9,%r10 - movq %rbp,%r13 - sbbq $0,%r11 - - xorq %r9,%r9 - adcq $-1,%rax - movq %rcx,%r8 - adcq %r14,%rbp - adcq $0,%rcx - movq %r10,%r9 - adcq %r15,%r10 - - btq $0,%r11 - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - cmovcq %rcx,%r8 - cmovcq %r10,%r9 - - ret - - - - -.p2align 5 -__ecp_nistz256_mul_by_2x: - - xorq %r11,%r11 - adcq %r12,%r12 - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - xorq %r10,%r10 - sbbq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret - - -.globl _ecp_nistz256_point_double_adx -.private_extern _ecp_nistz256_point_double_adx - -.p2align 5 -_ecp_nistz256_point_double_adx: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $160+8,%rsp - -L$point_doublex_body: - -L$point_double_shortcutx: - movdqu 0(%rsi),%xmm0 - movq %rsi,%rbx - movdqu 16(%rsi),%xmm1 - movq 32+0(%rsi),%r12 - movq 32+8(%rsi),%r13 - movq 32+16(%rsi),%r8 - movq 32+24(%rsi),%r9 - movq L$poly+8(%rip),%r14 - movq L$poly+24(%rip),%r15 - movdqa %xmm0,96(%rsp) - movdqa %xmm1,96+16(%rsp) - leaq 32(%rdi),%r10 - leaq 64(%rdi),%r11 - movq %rdi,%xmm0 - movq %r10,%xmm1 - movq %r11,%xmm2 - - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_by_2x - - movq 64+0(%rsi),%rdx - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - leaq 64-128(%rsi),%rsi - leaq 64(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 0+0(%rsp),%rdx - movq 8+0(%rsp),%r14 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - leaq 0(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 32(%rbx),%rdx - movq 64+0(%rbx),%r9 - movq 64+8(%rbx),%r10 - movq 64+16(%rbx),%r11 - movq 64+24(%rbx),%r12 - leaq 64-128(%rbx),%rsi - leaq 32(%rbx),%rbx - movq %xmm2,%rdi - call __ecp_nistz256_mul_montx - call __ecp_nistz256_mul_by_2x - - movq 96+0(%rsp),%r12 - movq 96+8(%rsp),%r13 - leaq 64(%rsp),%rbx - movq 96+16(%rsp),%r8 - movq 96+24(%rsp),%r9 - leaq 32(%rsp),%rdi - call __ecp_nistz256_add_tox - - movq 96+0(%rsp),%r12 - movq 96+8(%rsp),%r13 - leaq 64(%rsp),%rbx - movq 96+16(%rsp),%r8 - movq 96+24(%rsp),%r9 - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 0+0(%rsp),%rdx - movq 8+0(%rsp),%r14 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - movq %xmm1,%rdi - call __ecp_nistz256_sqr_montx - xorq %r9,%r9 - movq %r12,%rax - addq $-1,%r12 - movq %r13,%r10 - adcq %rsi,%r13 - movq %r14,%rcx - adcq $0,%r14 - movq %r15,%r8 - adcq %rbp,%r15 - adcq $0,%r9 - xorq %rsi,%rsi - testq $1,%rax - - cmovzq %rax,%r12 - cmovzq %r10,%r13 - cmovzq %rcx,%r14 - cmovzq %r8,%r15 - cmovzq %rsi,%r9 - - movq %r13,%rax - shrq $1,%r12 - shlq $63,%rax - movq %r14,%r10 - shrq $1,%r13 - orq %rax,%r12 - shlq $63,%r10 - movq %r15,%rcx - shrq $1,%r14 - orq %r10,%r13 - shlq $63,%rcx - movq %r12,0(%rdi) - shrq $1,%r15 - movq %r13,8(%rdi) - shlq $63,%r9 - orq %rcx,%r14 - orq %r9,%r15 - movq %r14,16(%rdi) - movq %r15,24(%rdi) - movq 64(%rsp),%rdx - leaq 64(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2x - - leaq 32(%rsp),%rbx - leaq 32(%rsp),%rdi - call __ecp_nistz256_add_tox - - movq 96(%rsp),%rdx - leaq 96(%rsp),%rbx - movq 0+0(%rsp),%r9 - movq 8+0(%rsp),%r10 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r11 - movq 24+0(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2x - - movq 0+32(%rsp),%rdx - movq 8+32(%rsp),%r14 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r15 - movq 24+32(%rsp),%r8 - movq %xmm0,%rdi - call __ecp_nistz256_sqr_montx - - leaq 128(%rsp),%rbx - movq %r14,%r8 - movq %r15,%r9 - movq %rsi,%r14 - movq %rbp,%r15 - call __ecp_nistz256_sub_fromx - - movq 0+0(%rsp),%rax - movq 0+8(%rsp),%rbp - movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 - leaq 0(%rsp),%rdi - call __ecp_nistz256_subx - - movq 32(%rsp),%rdx - leaq 32(%rsp),%rbx - movq %r12,%r14 - xorl %ecx,%ecx - movq %r12,0+0(%rsp) - movq %r13,%r10 - movq %r13,0+8(%rsp) - cmovzq %r8,%r11 - movq %r8,0+16(%rsp) - leaq 0-128(%rsp),%rsi - cmovzq %r9,%r12 - movq %r9,0+24(%rsp) - movq %r14,%r9 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq %xmm1,%rbx - movq %xmm1,%rdi - call __ecp_nistz256_sub_fromx - - leaq 160+56(%rsp),%rsi - - movq -48(%rsi),%r15 - - movq -40(%rsi),%r14 - - movq -32(%rsi),%r13 - - movq -24(%rsi),%r12 - - movq -16(%rsi),%rbx - - movq -8(%rsi),%rbp - - leaq (%rsi),%rsp - -L$point_doublex_epilogue: - ret - - -.globl _ecp_nistz256_point_add_adx -.private_extern _ecp_nistz256_point_add_adx - -.p2align 5 -_ecp_nistz256_point_add_adx: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $576+8,%rsp - -L$point_addx_body: - - movdqu 0(%rsi),%xmm0 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - movdqu 64(%rsi),%xmm4 - movdqu 80(%rsi),%xmm5 - movq %rsi,%rbx - movq %rdx,%rsi - movdqa %xmm0,384(%rsp) - movdqa %xmm1,384+16(%rsp) - movdqa %xmm2,416(%rsp) - movdqa %xmm3,416+16(%rsp) - movdqa %xmm4,448(%rsp) - movdqa %xmm5,448+16(%rsp) - por %xmm4,%xmm5 - - movdqu 0(%rsi),%xmm0 - pshufd $0xb1,%xmm5,%xmm3 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - por %xmm3,%xmm5 - movdqu 48(%rsi),%xmm3 - movq 64+0(%rsi),%rdx - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - movdqa %xmm0,480(%rsp) - pshufd $0x1e,%xmm5,%xmm4 - movdqa %xmm1,480+16(%rsp) - movdqu 64(%rsi),%xmm0 - movdqu 80(%rsi),%xmm1 - movdqa %xmm2,512(%rsp) - movdqa %xmm3,512+16(%rsp) - por %xmm4,%xmm5 - pxor %xmm4,%xmm4 - por %xmm0,%xmm1 - movq %rdi,%xmm0 - - leaq 64-128(%rsi),%rsi - movq %rdx,544+0(%rsp) - movq %r14,544+8(%rsp) - movq %r15,544+16(%rsp) - movq %r8,544+24(%rsp) - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - pcmpeqd %xmm4,%xmm5 - pshufd $0xb1,%xmm1,%xmm4 - por %xmm1,%xmm4 - pshufd $0,%xmm5,%xmm5 - pshufd $0x1e,%xmm4,%xmm3 - por %xmm3,%xmm4 - pxor %xmm3,%xmm3 - pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - movq 64+0(%rbx),%rdx - movq 64+8(%rbx),%r14 - movq 64+16(%rbx),%r15 - movq 64+24(%rbx),%r8 - movq %rbx,%xmm1 - - leaq 64-128(%rbx),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 544(%rsp),%rdx - leaq 544(%rsp),%rbx - movq 0+96(%rsp),%r9 - movq 8+96(%rsp),%r10 - leaq -128+96(%rsp),%rsi - movq 16+96(%rsp),%r11 - movq 24+96(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 448(%rsp),%rdx - leaq 448(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 416(%rsp),%rdx - leaq 416(%rsp),%rbx - movq 0+224(%rsp),%r9 - movq 8+224(%rsp),%r10 - leaq -128+224(%rsp),%rsi - movq 16+224(%rsp),%r11 - movq 24+224(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 512(%rsp),%rdx - leaq 512(%rsp),%rbx - movq 0+256(%rsp),%r9 - movq 8+256(%rsp),%r10 - leaq -128+256(%rsp),%rsi - movq 16+256(%rsp),%r11 - movq 24+256(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 224(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - orq %r13,%r12 - movdqa %xmm4,%xmm2 - orq %r8,%r12 - orq %r9,%r12 - por %xmm5,%xmm2 - movq %r12,%xmm3 - - movq 384(%rsp),%rdx - leaq 384(%rsp),%rbx - movq 0+96(%rsp),%r9 - movq 8+96(%rsp),%r10 - leaq -128+96(%rsp),%rsi - movq 16+96(%rsp),%r11 - movq 24+96(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 480(%rsp),%rdx - leaq 480(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 160(%rsp),%rbx - leaq 0(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - orq %r13,%r12 - orq %r8,%r12 - orq %r9,%r12 - - movq %xmm2,%r8 - movq %xmm3,%r9 - orq %r8,%r12 -.byte 0x3e - jnz L$add_proceedx - - - - testq %r9,%r9 - jz L$add_doublex - - - - - - - movq %xmm0,%rdi - pxor %xmm0,%xmm0 - movdqu %xmm0,0(%rdi) - movdqu %xmm0,16(%rdi) - movdqu %xmm0,32(%rdi) - movdqu %xmm0,48(%rdi) - movdqu %xmm0,64(%rdi) - movdqu %xmm0,80(%rdi) - jmp L$add_donex - -.p2align 5 -L$add_doublex: - movq %xmm1,%rsi - movq %xmm0,%rdi - addq $416,%rsp - - jmp L$point_double_shortcutx - - -.p2align 5 -L$add_proceedx: - movq 0+64(%rsp),%rdx - movq 8+64(%rsp),%r14 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r15 - movq 24+64(%rsp),%r8 - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 448(%rsp),%rdx - leaq 448(%rsp),%rbx - movq 0+0(%rsp),%r9 - movq 8+0(%rsp),%r10 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r11 - movq 24+0(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 0+0(%rsp),%rdx - movq 8+0(%rsp),%r14 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 544(%rsp),%rdx - leaq 544(%rsp),%rbx - movq 0+352(%rsp),%r9 - movq 8+352(%rsp),%r10 - leaq -128+352(%rsp),%rsi - movq 16+352(%rsp),%r11 - movq 24+352(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 0(%rsp),%rdx - leaq 0(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 160(%rsp),%rdx - leaq 160(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montx - - - - - xorq %r11,%r11 - addq %r12,%r12 - leaq 96(%rsp),%rsi - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - movq 0(%rsi),%rax - cmovcq %rbp,%r13 - movq 8(%rsi),%rbp - cmovcq %rcx,%r8 - movq 16(%rsi),%rcx - cmovcq %r10,%r9 - movq 24(%rsi),%r10 - - call __ecp_nistz256_subx - - leaq 128(%rsp),%rbx - leaq 288(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 192+0(%rsp),%rax - movq 192+8(%rsp),%rbp - movq 192+16(%rsp),%rcx - movq 192+24(%rsp),%r10 - leaq 320(%rsp),%rdi - - call __ecp_nistz256_subx - - movq %r12,0(%rdi) - movq %r13,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq 128(%rsp),%rdx - leaq 128(%rsp),%rbx - movq 0+224(%rsp),%r9 - movq 8+224(%rsp),%r10 - leaq -128+224(%rsp),%rsi - movq 16+224(%rsp),%r11 - movq 24+224(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 320(%rsp),%rdx - leaq 320(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 320(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 256(%rsp),%rbx - leaq 320(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq %xmm0,%rdi - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 352(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 352+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 544(%rsp),%xmm2 - pand 544+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 448(%rsp),%xmm2 - pand 448+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,64(%rdi) - movdqu %xmm3,80(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 288(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 288+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 480(%rsp),%xmm2 - pand 480+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 384(%rsp),%xmm2 - pand 384+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 320(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 320+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 512(%rsp),%xmm2 - pand 512+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 416(%rsp),%xmm2 - pand 416+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - -L$add_donex: - leaq 576+56(%rsp),%rsi - - movq -48(%rsi),%r15 - - movq -40(%rsi),%r14 - - movq -32(%rsi),%r13 - - movq -24(%rsi),%r12 - - movq -16(%rsi),%rbx - - movq -8(%rsi),%rbp - - leaq (%rsi),%rsp - -L$point_addx_epilogue: - ret - - -.globl _ecp_nistz256_point_add_affine_adx -.private_extern _ecp_nistz256_point_add_affine_adx - -.p2align 5 -_ecp_nistz256_point_add_affine_adx: - -_CET_ENDBR - pushq %rbp - - pushq %rbx - - pushq %r12 - - pushq %r13 - - pushq %r14 - - pushq %r15 - - subq $480+8,%rsp - -L$add_affinex_body: - - movdqu 0(%rsi),%xmm0 - movq %rdx,%rbx - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - movdqu 64(%rsi),%xmm4 - movdqu 80(%rsi),%xmm5 - movq 64+0(%rsi),%rdx - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - movdqa %xmm0,320(%rsp) - movdqa %xmm1,320+16(%rsp) - movdqa %xmm2,352(%rsp) - movdqa %xmm3,352+16(%rsp) - movdqa %xmm4,384(%rsp) - movdqa %xmm5,384+16(%rsp) - por %xmm4,%xmm5 - - movdqu 0(%rbx),%xmm0 - pshufd $0xb1,%xmm5,%xmm3 - movdqu 16(%rbx),%xmm1 - movdqu 32(%rbx),%xmm2 - por %xmm3,%xmm5 - movdqu 48(%rbx),%xmm3 - movdqa %xmm0,416(%rsp) - pshufd $0x1e,%xmm5,%xmm4 - movdqa %xmm1,416+16(%rsp) - por %xmm0,%xmm1 - movq %rdi,%xmm0 - movdqa %xmm2,448(%rsp) - movdqa %xmm3,448+16(%rsp) - por %xmm2,%xmm3 - por %xmm4,%xmm5 - pxor %xmm4,%xmm4 - por %xmm1,%xmm3 - - leaq 64-128(%rsi),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - pcmpeqd %xmm4,%xmm5 - pshufd $0xb1,%xmm3,%xmm4 - movq 0(%rbx),%rdx - - movq %r12,%r9 - por %xmm3,%xmm4 - pshufd $0,%xmm5,%xmm5 - pshufd $0x1e,%xmm4,%xmm3 - movq %r13,%r10 - por %xmm3,%xmm4 - pxor %xmm3,%xmm3 - movq %r14,%r11 - pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - - leaq 32-128(%rsp),%rsi - movq %r15,%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 320(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 384(%rsp),%rdx - leaq 384(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 384(%rsp),%rdx - leaq 384(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 288(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 448(%rsp),%rdx - leaq 448(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 352(%rsp),%rbx - leaq 96(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 0+64(%rsp),%rdx - movq 8+64(%rsp),%r14 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r15 - movq 24+64(%rsp),%r8 - leaq 128(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 0+96(%rsp),%rdx - movq 8+96(%rsp),%r14 - leaq -128+96(%rsp),%rsi - movq 16+96(%rsp),%r15 - movq 24+96(%rsp),%r8 - leaq 192(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 128(%rsp),%rdx - leaq 128(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 320(%rsp),%rdx - leaq 320(%rsp),%rbx - movq 0+128(%rsp),%r9 - movq 8+128(%rsp),%r10 - leaq -128+128(%rsp),%rsi - movq 16+128(%rsp),%r11 - movq 24+128(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - - - - xorq %r11,%r11 - addq %r12,%r12 - leaq 192(%rsp),%rsi - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - movq 0(%rsi),%rax - cmovcq %rbp,%r13 - movq 8(%rsi),%rbp - cmovcq %rcx,%r8 - movq 16(%rsi),%rcx - cmovcq %r10,%r9 - movq 24(%rsi),%r10 - - call __ecp_nistz256_subx - - leaq 160(%rsp),%rbx - leaq 224(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 0+0(%rsp),%rax - movq 0+8(%rsp),%rbp - movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 - leaq 64(%rsp),%rdi - - call __ecp_nistz256_subx - - movq %r12,0(%rdi) - movq %r13,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq 352(%rsp),%rdx - leaq 352(%rsp),%rbx - movq 0+160(%rsp),%r9 - movq 8+160(%rsp),%r10 - leaq -128+160(%rsp),%rsi - movq 16+160(%rsp),%r11 - movq 24+160(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 96(%rsp),%rdx - leaq 96(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 64(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 32(%rsp),%rbx - leaq 256(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq %xmm0,%rdi - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 288(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 288+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand L$ONE_mont(%rip),%xmm2 - pand L$ONE_mont+16(%rip),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 384(%rsp),%xmm2 - pand 384+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,64(%rdi) - movdqu %xmm3,80(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 224(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 224+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 416(%rsp),%xmm2 - pand 416+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 320(%rsp),%xmm2 - pand 320+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 256(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 256+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 448(%rsp),%xmm2 - pand 448+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 352(%rsp),%xmm2 - pand 352+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - - leaq 480+56(%rsp),%rsi - - movq -48(%rsi),%r15 - - movq -40(%rsi),%r14 - - movq -32(%rsi),%r13 - - movq -24(%rsi),%r12 - - movq -16(%rsi),%rbx - - movq -8(%rsi),%rbp - - leaq (%rsi),%rsp - -L$add_affinex_epilogue: - ret - - #endif
diff --git a/gen/bcm/p256-x86_64-asm-linux.S b/gen/bcm/p256-x86_64-asm-linux.S index c54d577..04733ab 100644 --- a/gen/bcm/p256-x86_64-asm-linux.S +++ b/gen/bcm/p256-x86_64-asm-linux.S
@@ -7,21 +7,6 @@ .text -.section .rodata -.align 64 -.Lpoly: -.quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001 - -.LOne: -.long 1,1,1,1,1,1,1,1 -.LTwo: -.long 2,2,2,2,2,2,2,2 -.LThree: -.long 3,3,3,3,3,3,3,3 -.LONE_mont: -.quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe - - .Lord: .quad 0xf3b9cac2fc632551, 0xbce6faada7179e84, 0xffffffffffffffff, 0xffffffff00000000 .LordK: @@ -30,66 +15,6 @@ -.globl ecp_nistz256_neg -.hidden ecp_nistz256_neg -.type ecp_nistz256_neg,@function -.align 32 -ecp_nistz256_neg: -.cfi_startproc -_CET_ENDBR - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-16 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-24 -.Lneg_body: - - xorq %r8,%r8 - xorq %r9,%r9 - xorq %r10,%r10 - xorq %r11,%r11 - xorq %r13,%r13 - - subq 0(%rsi),%r8 - sbbq 8(%rsi),%r9 - sbbq 16(%rsi),%r10 - movq %r8,%rax - sbbq 24(%rsi),%r11 - leaq .Lpoly(%rip),%rsi - movq %r9,%rdx - sbbq $0,%r13 - - addq 0(%rsi),%r8 - movq %r10,%rcx - adcq 8(%rsi),%r9 - adcq 16(%rsi),%r10 - movq %r11,%r12 - adcq 24(%rsi),%r11 - testq %r13,%r13 - - cmovzq %rax,%r8 - cmovzq %rdx,%r9 - movq %r8,0(%rdi) - cmovzq %rcx,%r10 - movq %r9,8(%rdi) - cmovzq %r12,%r11 - movq %r10,16(%rdi) - movq %r11,24(%rdi) - - movq 0(%rsp),%r13 -.cfi_restore %r13 - movq 8(%rsp),%r12 -.cfi_restore %r12 - leaq 16(%rsp),%rsp -.cfi_adjust_cfa_offset -16 -.Lneg_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_neg,.-ecp_nistz256_neg - - - @@ -1178,3422 +1103,4 @@ ret .cfi_endproc .size ecp_nistz256_ord_sqr_mont_adx,.-ecp_nistz256_ord_sqr_mont_adx - - - - - - -.globl ecp_nistz256_mul_mont_nohw -.hidden ecp_nistz256_mul_mont_nohw -.type ecp_nistz256_mul_mont_nohw,@function -.align 32 -ecp_nistz256_mul_mont_nohw: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lmul_body: - movq %rdx,%rbx - movq 0(%rdx),%rax - movq 0(%rsi),%r9 - movq 8(%rsi),%r10 - movq 16(%rsi),%r11 - movq 24(%rsi),%r12 - - call __ecp_nistz256_mul_montq - - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%r12 -.cfi_restore %r12 - movq 32(%rsp),%rbx -.cfi_restore %rbx - movq 40(%rsp),%rbp -.cfi_restore %rbp - leaq 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lmul_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_mul_mont_nohw,.-ecp_nistz256_mul_mont_nohw - -.type __ecp_nistz256_mul_montq,@function -.align 32 -__ecp_nistz256_mul_montq: -.cfi_startproc - - - movq %rax,%rbp - mulq %r9 - movq .Lpoly+8(%rip),%r14 - movq %rax,%r8 - movq %rbp,%rax - movq %rdx,%r9 - - mulq %r10 - movq .Lpoly+24(%rip),%r15 - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r10 - - mulq %r11 - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq %r12 - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - xorq %r13,%r13 - movq %rdx,%r12 - - - - - - - - - - - movq %r8,%rbp - shlq $32,%r8 - mulq %r15 - shrq $32,%rbp - addq %r8,%r9 - adcq %rbp,%r10 - adcq %rax,%r11 - movq 8(%rbx),%rax - adcq %rdx,%r12 - adcq $0,%r13 - xorq %r8,%r8 - - - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r9 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 8(%rsi) - addq %rcx,%r10 - adcq $0,%rdx - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 16(%rsi) - addq %rcx,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 24(%rsi) - addq %rcx,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %r9,%rax - adcq %rdx,%r13 - adcq $0,%r8 - - - - movq %r9,%rbp - shlq $32,%r9 - mulq %r15 - shrq $32,%rbp - addq %r9,%r10 - adcq %rbp,%r11 - adcq %rax,%r12 - movq 16(%rbx),%rax - adcq %rdx,%r13 - adcq $0,%r8 - xorq %r9,%r9 - - - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r10 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 8(%rsi) - addq %rcx,%r11 - adcq $0,%rdx - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 16(%rsi) - addq %rcx,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 24(%rsi) - addq %rcx,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %r10,%rax - adcq %rdx,%r8 - adcq $0,%r9 - - - - movq %r10,%rbp - shlq $32,%r10 - mulq %r15 - shrq $32,%rbp - addq %r10,%r11 - adcq %rbp,%r12 - adcq %rax,%r13 - movq 24(%rbx),%rax - adcq %rdx,%r8 - adcq $0,%r9 - xorq %r10,%r10 - - - - movq %rax,%rbp - mulq 0(%rsi) - addq %rax,%r11 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 8(%rsi) - addq %rcx,%r12 - adcq $0,%rdx - addq %rax,%r12 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 16(%rsi) - addq %rcx,%r13 - adcq $0,%rdx - addq %rax,%r13 - movq %rbp,%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq 24(%rsi) - addq %rcx,%r8 - adcq $0,%rdx - addq %rax,%r8 - movq %r11,%rax - adcq %rdx,%r9 - adcq $0,%r10 - - - - movq %r11,%rbp - shlq $32,%r11 - mulq %r15 - shrq $32,%rbp - addq %r11,%r12 - adcq %rbp,%r13 - movq %r12,%rcx - adcq %rax,%r8 - adcq %rdx,%r9 - movq %r13,%rbp - adcq $0,%r10 - - - - subq $-1,%r12 - movq %r8,%rbx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%rdx - sbbq %r15,%r9 - sbbq $0,%r10 - - cmovcq %rcx,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rbx,%r8 - movq %r13,8(%rdi) - cmovcq %rdx,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq - - - - - - - - -.globl ecp_nistz256_sqr_mont_nohw -.hidden ecp_nistz256_sqr_mont_nohw -.type ecp_nistz256_sqr_mont_nohw,@function -.align 32 -ecp_nistz256_sqr_mont_nohw: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lsqr_body: - movq 0(%rsi),%rax - movq 8(%rsi),%r14 - movq 16(%rsi),%r15 - movq 24(%rsi),%r8 - - call __ecp_nistz256_sqr_montq - - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%r12 -.cfi_restore %r12 - movq 32(%rsp),%rbx -.cfi_restore %rbx - movq 40(%rsp),%rbp -.cfi_restore %rbp - leaq 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lsqr_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_sqr_mont_nohw,.-ecp_nistz256_sqr_mont_nohw - -.type __ecp_nistz256_sqr_montq,@function -.align 32 -__ecp_nistz256_sqr_montq: -.cfi_startproc - movq %rax,%r13 - mulq %r14 - movq %rax,%r9 - movq %r15,%rax - movq %rdx,%r10 - - mulq %r13 - addq %rax,%r10 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%r11 - - mulq %r13 - addq %rax,%r11 - movq %r15,%rax - adcq $0,%rdx - movq %rdx,%r12 - - - mulq %r14 - addq %rax,%r11 - movq %r8,%rax - adcq $0,%rdx - movq %rdx,%rbp - - mulq %r14 - addq %rax,%r12 - movq %r8,%rax - adcq $0,%rdx - addq %rbp,%r12 - movq %rdx,%r13 - adcq $0,%r13 - - - mulq %r15 - xorq %r15,%r15 - addq %rax,%r13 - movq 0(%rsi),%rax - movq %rdx,%r14 - adcq $0,%r14 - - addq %r9,%r9 - adcq %r10,%r10 - adcq %r11,%r11 - adcq %r12,%r12 - adcq %r13,%r13 - adcq %r14,%r14 - adcq $0,%r15 - - mulq %rax - movq %rax,%r8 - movq 8(%rsi),%rax - movq %rdx,%rcx - - mulq %rax - addq %rcx,%r9 - adcq %rax,%r10 - movq 16(%rsi),%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq %rax - addq %rcx,%r11 - adcq %rax,%r12 - movq 24(%rsi),%rax - adcq $0,%rdx - movq %rdx,%rcx - - mulq %rax - addq %rcx,%r13 - adcq %rax,%r14 - movq %r8,%rax - adcq %rdx,%r15 - - movq .Lpoly+8(%rip),%rsi - movq .Lpoly+24(%rip),%rbp - - - - - movq %r8,%rcx - shlq $32,%r8 - mulq %rbp - shrq $32,%rcx - addq %r8,%r9 - adcq %rcx,%r10 - adcq %rax,%r11 - movq %r9,%rax - adcq $0,%rdx - - - - movq %r9,%rcx - shlq $32,%r9 - movq %rdx,%r8 - mulq %rbp - shrq $32,%rcx - addq %r9,%r10 - adcq %rcx,%r11 - adcq %rax,%r8 - movq %r10,%rax - adcq $0,%rdx - - - - movq %r10,%rcx - shlq $32,%r10 - movq %rdx,%r9 - mulq %rbp - shrq $32,%rcx - addq %r10,%r11 - adcq %rcx,%r8 - adcq %rax,%r9 - movq %r11,%rax - adcq $0,%rdx - - - - movq %r11,%rcx - shlq $32,%r11 - movq %rdx,%r10 - mulq %rbp - shrq $32,%rcx - addq %r11,%r8 - adcq %rcx,%r9 - adcq %rax,%r10 - adcq $0,%rdx - xorq %r11,%r11 - - - - addq %r8,%r12 - adcq %r9,%r13 - movq %r12,%r8 - adcq %r10,%r14 - adcq %rdx,%r15 - movq %r13,%r9 - adcq $0,%r11 - - subq $-1,%r12 - movq %r14,%r10 - sbbq %rsi,%r13 - sbbq $0,%r14 - movq %r15,%rcx - sbbq %rbp,%r15 - sbbq $0,%r11 - - cmovcq %r8,%r12 - cmovcq %r9,%r13 - movq %r12,0(%rdi) - cmovcq %r10,%r14 - movq %r13,8(%rdi) - cmovcq %rcx,%r15 - movq %r14,16(%rdi) - movq %r15,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq -.globl ecp_nistz256_mul_mont_adx -.hidden ecp_nistz256_mul_mont_adx -.type ecp_nistz256_mul_mont_adx,@function -.align 32 -ecp_nistz256_mul_mont_adx: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lmulx_body: - movq %rdx,%rbx - movq 0(%rdx),%rdx - movq 0(%rsi),%r9 - movq 8(%rsi),%r10 - movq 16(%rsi),%r11 - movq 24(%rsi),%r12 - leaq -128(%rsi),%rsi - - call __ecp_nistz256_mul_montx - - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%r12 -.cfi_restore %r12 - movq 32(%rsp),%rbx -.cfi_restore %rbx - movq 40(%rsp),%rbp -.cfi_restore %rbp - leaq 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lmulx_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_mul_mont_adx,.-ecp_nistz256_mul_mont_adx - -.type __ecp_nistz256_mul_montx,@function -.align 32 -__ecp_nistz256_mul_montx: -.cfi_startproc - - - mulxq %r9,%r8,%r9 - mulxq %r10,%rcx,%r10 - movq $32,%r14 - xorq %r13,%r13 - mulxq %r11,%rbp,%r11 - movq .Lpoly+24(%rip),%r15 - adcq %rcx,%r9 - mulxq %r12,%rcx,%r12 - movq %r8,%rdx - adcq %rbp,%r10 - shlxq %r14,%r8,%rbp - adcq %rcx,%r11 - shrxq %r14,%r8,%rcx - adcq $0,%r12 - - - - addq %rbp,%r9 - adcq %rcx,%r10 - - mulxq %r15,%rcx,%rbp - movq 8(%rbx),%rdx - adcq %rcx,%r11 - adcq %rbp,%r12 - adcq $0,%r13 - xorq %r8,%r8 - - - - mulxq 0+128(%rsi),%rcx,%rbp - adcxq %rcx,%r9 - adoxq %rbp,%r10 - - mulxq 8+128(%rsi),%rcx,%rbp - adcxq %rcx,%r10 - adoxq %rbp,%r11 - - mulxq 16+128(%rsi),%rcx,%rbp - adcxq %rcx,%r11 - adoxq %rbp,%r12 - - mulxq 24+128(%rsi),%rcx,%rbp - movq %r9,%rdx - adcxq %rcx,%r12 - shlxq %r14,%r9,%rcx - adoxq %rbp,%r13 - shrxq %r14,%r9,%rbp - - adcxq %r8,%r13 - adoxq %r8,%r8 - adcq $0,%r8 - - - - addq %rcx,%r10 - adcq %rbp,%r11 - - mulxq %r15,%rcx,%rbp - movq 16(%rbx),%rdx - adcq %rcx,%r12 - adcq %rbp,%r13 - adcq $0,%r8 - xorq %r9,%r9 - - - - mulxq 0+128(%rsi),%rcx,%rbp - adcxq %rcx,%r10 - adoxq %rbp,%r11 - - mulxq 8+128(%rsi),%rcx,%rbp - adcxq %rcx,%r11 - adoxq %rbp,%r12 - - mulxq 16+128(%rsi),%rcx,%rbp - adcxq %rcx,%r12 - adoxq %rbp,%r13 - - mulxq 24+128(%rsi),%rcx,%rbp - movq %r10,%rdx - adcxq %rcx,%r13 - shlxq %r14,%r10,%rcx - adoxq %rbp,%r8 - shrxq %r14,%r10,%rbp - - adcxq %r9,%r8 - adoxq %r9,%r9 - adcq $0,%r9 - - - - addq %rcx,%r11 - adcq %rbp,%r12 - - mulxq %r15,%rcx,%rbp - movq 24(%rbx),%rdx - adcq %rcx,%r13 - adcq %rbp,%r8 - adcq $0,%r9 - xorq %r10,%r10 - - - - mulxq 0+128(%rsi),%rcx,%rbp - adcxq %rcx,%r11 - adoxq %rbp,%r12 - - mulxq 8+128(%rsi),%rcx,%rbp - adcxq %rcx,%r12 - adoxq %rbp,%r13 - - mulxq 16+128(%rsi),%rcx,%rbp - adcxq %rcx,%r13 - adoxq %rbp,%r8 - - mulxq 24+128(%rsi),%rcx,%rbp - movq %r11,%rdx - adcxq %rcx,%r8 - shlxq %r14,%r11,%rcx - adoxq %rbp,%r9 - shrxq %r14,%r11,%rbp - - adcxq %r10,%r9 - adoxq %r10,%r10 - adcq $0,%r10 - - - - addq %rcx,%r12 - adcq %rbp,%r13 - - mulxq %r15,%rcx,%rbp - movq %r12,%rbx - movq .Lpoly+8(%rip),%r14 - adcq %rcx,%r8 - movq %r13,%rdx - adcq %rbp,%r9 - adcq $0,%r10 - - - - xorl %eax,%eax - movq %r8,%rcx - sbbq $-1,%r12 - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%rbp - sbbq %r15,%r9 - sbbq $0,%r10 - - cmovcq %rbx,%r12 - cmovcq %rdx,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %rbp,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx - -.globl ecp_nistz256_sqr_mont_adx -.hidden ecp_nistz256_sqr_mont_adx -.type ecp_nistz256_sqr_mont_adx,@function -.align 32 -ecp_nistz256_sqr_mont_adx: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 -.Lsqrx_body: - movq 0(%rsi),%rdx - movq 8(%rsi),%r14 - movq 16(%rsi),%r15 - movq 24(%rsi),%r8 - leaq -128(%rsi),%rsi - - call __ecp_nistz256_sqr_montx - - movq 0(%rsp),%r15 -.cfi_restore %r15 - movq 8(%rsp),%r14 -.cfi_restore %r14 - movq 16(%rsp),%r13 -.cfi_restore %r13 - movq 24(%rsp),%r12 -.cfi_restore %r12 - movq 32(%rsp),%rbx -.cfi_restore %rbx - movq 40(%rsp),%rbp -.cfi_restore %rbp - leaq 48(%rsp),%rsp -.cfi_adjust_cfa_offset -48 -.Lsqrx_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_sqr_mont_adx,.-ecp_nistz256_sqr_mont_adx - -.type __ecp_nistz256_sqr_montx,@function -.align 32 -__ecp_nistz256_sqr_montx: -.cfi_startproc - mulxq %r14,%r9,%r10 - mulxq %r15,%rcx,%r11 - xorl %eax,%eax - adcq %rcx,%r10 - mulxq %r8,%rbp,%r12 - movq %r14,%rdx - adcq %rbp,%r11 - adcq $0,%r12 - xorq %r13,%r13 - - - mulxq %r15,%rcx,%rbp - adcxq %rcx,%r11 - adoxq %rbp,%r12 - - mulxq %r8,%rcx,%rbp - movq %r15,%rdx - adcxq %rcx,%r12 - adoxq %rbp,%r13 - adcq $0,%r13 - - - mulxq %r8,%rcx,%r14 - movq 0+128(%rsi),%rdx - xorq %r15,%r15 - adcxq %r9,%r9 - adoxq %rcx,%r13 - adcxq %r10,%r10 - adoxq %r15,%r14 - - mulxq %rdx,%r8,%rbp - movq 8+128(%rsi),%rdx - adcxq %r11,%r11 - adoxq %rbp,%r9 - adcxq %r12,%r12 - mulxq %rdx,%rcx,%rax - movq 16+128(%rsi),%rdx - adcxq %r13,%r13 - adoxq %rcx,%r10 - adcxq %r14,%r14 -.byte 0x67 - mulxq %rdx,%rcx,%rbp - movq 24+128(%rsi),%rdx - adoxq %rax,%r11 - adcxq %r15,%r15 - adoxq %rcx,%r12 - movq $32,%rsi - adoxq %rbp,%r13 -.byte 0x67,0x67 - mulxq %rdx,%rcx,%rax - movq .Lpoly+24(%rip),%rdx - adoxq %rcx,%r14 - shlxq %rsi,%r8,%rcx - adoxq %rax,%r15 - shrxq %rsi,%r8,%rax - movq %rdx,%rbp - - - addq %rcx,%r9 - adcq %rax,%r10 - - mulxq %r8,%rcx,%r8 - adcq %rcx,%r11 - shlxq %rsi,%r9,%rcx - adcq $0,%r8 - shrxq %rsi,%r9,%rax - - - addq %rcx,%r10 - adcq %rax,%r11 - - mulxq %r9,%rcx,%r9 - adcq %rcx,%r8 - shlxq %rsi,%r10,%rcx - adcq $0,%r9 - shrxq %rsi,%r10,%rax - - - addq %rcx,%r11 - adcq %rax,%r8 - - mulxq %r10,%rcx,%r10 - adcq %rcx,%r9 - shlxq %rsi,%r11,%rcx - adcq $0,%r10 - shrxq %rsi,%r11,%rax - - - addq %rcx,%r8 - adcq %rax,%r9 - - mulxq %r11,%rcx,%r11 - adcq %rcx,%r10 - adcq $0,%r11 - - xorq %rdx,%rdx - addq %r8,%r12 - movq .Lpoly+8(%rip),%rsi - adcq %r9,%r13 - movq %r12,%r8 - adcq %r10,%r14 - adcq %r11,%r15 - movq %r13,%r9 - adcq $0,%rdx - - subq $-1,%r12 - movq %r14,%r10 - sbbq %rsi,%r13 - sbbq $0,%r14 - movq %r15,%r11 - sbbq %rbp,%r15 - sbbq $0,%rdx - - cmovcq %r8,%r12 - cmovcq %r9,%r13 - movq %r12,0(%rdi) - cmovcq %r10,%r14 - movq %r13,8(%rdi) - cmovcq %r11,%r15 - movq %r14,16(%rdi) - movq %r15,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx - - -.globl ecp_nistz256_select_w5_nohw -.hidden ecp_nistz256_select_w5_nohw -.type ecp_nistz256_select_w5_nohw,@function -.align 32 -ecp_nistz256_select_w5_nohw: -.cfi_startproc -_CET_ENDBR - movdqa .LOne(%rip),%xmm0 - movd %edx,%xmm1 - - pxor %xmm2,%xmm2 - pxor %xmm3,%xmm3 - pxor %xmm4,%xmm4 - pxor %xmm5,%xmm5 - pxor %xmm6,%xmm6 - pxor %xmm7,%xmm7 - - movdqa %xmm0,%xmm8 - pshufd $0,%xmm1,%xmm1 - - movq $16,%rax -.Lselect_loop_sse_w5: - - movdqa %xmm8,%xmm15 - paddd %xmm0,%xmm8 - pcmpeqd %xmm1,%xmm15 - - movdqa 0(%rsi),%xmm9 - movdqa 16(%rsi),%xmm10 - movdqa 32(%rsi),%xmm11 - movdqa 48(%rsi),%xmm12 - movdqa 64(%rsi),%xmm13 - movdqa 80(%rsi),%xmm14 - leaq 96(%rsi),%rsi - - pand %xmm15,%xmm9 - pand %xmm15,%xmm10 - por %xmm9,%xmm2 - pand %xmm15,%xmm11 - por %xmm10,%xmm3 - pand %xmm15,%xmm12 - por %xmm11,%xmm4 - pand %xmm15,%xmm13 - por %xmm12,%xmm5 - pand %xmm15,%xmm14 - por %xmm13,%xmm6 - por %xmm14,%xmm7 - - decq %rax - jnz .Lselect_loop_sse_w5 - - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - movdqu %xmm4,32(%rdi) - movdqu %xmm5,48(%rdi) - movdqu %xmm6,64(%rdi) - movdqu %xmm7,80(%rdi) - ret -.cfi_endproc -.LSEH_end_ecp_nistz256_select_w5_nohw: -.size ecp_nistz256_select_w5_nohw,.-ecp_nistz256_select_w5_nohw - - - -.globl ecp_nistz256_select_w7_nohw -.hidden ecp_nistz256_select_w7_nohw -.type ecp_nistz256_select_w7_nohw,@function -.align 32 -ecp_nistz256_select_w7_nohw: -.cfi_startproc -_CET_ENDBR - movdqa .LOne(%rip),%xmm8 - movd %edx,%xmm1 - - pxor %xmm2,%xmm2 - pxor %xmm3,%xmm3 - pxor %xmm4,%xmm4 - pxor %xmm5,%xmm5 - - movdqa %xmm8,%xmm0 - pshufd $0,%xmm1,%xmm1 - movq $64,%rax - -.Lselect_loop_sse_w7: - movdqa %xmm8,%xmm15 - paddd %xmm0,%xmm8 - movdqa 0(%rsi),%xmm9 - movdqa 16(%rsi),%xmm10 - pcmpeqd %xmm1,%xmm15 - movdqa 32(%rsi),%xmm11 - movdqa 48(%rsi),%xmm12 - leaq 64(%rsi),%rsi - - pand %xmm15,%xmm9 - pand %xmm15,%xmm10 - por %xmm9,%xmm2 - pand %xmm15,%xmm11 - por %xmm10,%xmm3 - pand %xmm15,%xmm12 - por %xmm11,%xmm4 - prefetcht0 255(%rsi) - por %xmm12,%xmm5 - - decq %rax - jnz .Lselect_loop_sse_w7 - - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - movdqu %xmm4,32(%rdi) - movdqu %xmm5,48(%rdi) - ret -.cfi_endproc -.LSEH_end_ecp_nistz256_select_w7_nohw: -.size ecp_nistz256_select_w7_nohw,.-ecp_nistz256_select_w7_nohw - - -.globl ecp_nistz256_select_w5_avx2 -.hidden ecp_nistz256_select_w5_avx2 -.type ecp_nistz256_select_w5_avx2,@function -.align 32 -ecp_nistz256_select_w5_avx2: -.cfi_startproc -_CET_ENDBR - vzeroupper - vmovdqa .LTwo(%rip),%ymm0 - - vpxor %ymm2,%ymm2,%ymm2 - vpxor %ymm3,%ymm3,%ymm3 - vpxor %ymm4,%ymm4,%ymm4 - - vmovdqa .LOne(%rip),%ymm5 - vmovdqa .LTwo(%rip),%ymm10 - - vmovd %edx,%xmm1 - vpermd %ymm1,%ymm2,%ymm1 - - movq $8,%rax -.Lselect_loop_avx2_w5: - - vmovdqa 0(%rsi),%ymm6 - vmovdqa 32(%rsi),%ymm7 - vmovdqa 64(%rsi),%ymm8 - - vmovdqa 96(%rsi),%ymm11 - vmovdqa 128(%rsi),%ymm12 - vmovdqa 160(%rsi),%ymm13 - - vpcmpeqd %ymm1,%ymm5,%ymm9 - vpcmpeqd %ymm1,%ymm10,%ymm14 - - vpaddd %ymm0,%ymm5,%ymm5 - vpaddd %ymm0,%ymm10,%ymm10 - leaq 192(%rsi),%rsi - - vpand %ymm9,%ymm6,%ymm6 - vpand %ymm9,%ymm7,%ymm7 - vpand %ymm9,%ymm8,%ymm8 - vpand %ymm14,%ymm11,%ymm11 - vpand %ymm14,%ymm12,%ymm12 - vpand %ymm14,%ymm13,%ymm13 - - vpxor %ymm6,%ymm2,%ymm2 - vpxor %ymm7,%ymm3,%ymm3 - vpxor %ymm8,%ymm4,%ymm4 - vpxor %ymm11,%ymm2,%ymm2 - vpxor %ymm12,%ymm3,%ymm3 - vpxor %ymm13,%ymm4,%ymm4 - - decq %rax - jnz .Lselect_loop_avx2_w5 - - vmovdqu %ymm2,0(%rdi) - vmovdqu %ymm3,32(%rdi) - vmovdqu %ymm4,64(%rdi) - vzeroupper - ret -.cfi_endproc -.LSEH_end_ecp_nistz256_select_w5_avx2: -.size ecp_nistz256_select_w5_avx2,.-ecp_nistz256_select_w5_avx2 - - - -.globl ecp_nistz256_select_w7_avx2 -.hidden ecp_nistz256_select_w7_avx2 -.type ecp_nistz256_select_w7_avx2,@function -.align 32 -ecp_nistz256_select_w7_avx2: -.cfi_startproc -_CET_ENDBR - vzeroupper - vmovdqa .LThree(%rip),%ymm0 - - vpxor %ymm2,%ymm2,%ymm2 - vpxor %ymm3,%ymm3,%ymm3 - - vmovdqa .LOne(%rip),%ymm4 - vmovdqa .LTwo(%rip),%ymm8 - vmovdqa .LThree(%rip),%ymm12 - - vmovd %edx,%xmm1 - vpermd %ymm1,%ymm2,%ymm1 - - - movq $21,%rax -.Lselect_loop_avx2_w7: - - vmovdqa 0(%rsi),%ymm5 - vmovdqa 32(%rsi),%ymm6 - - vmovdqa 64(%rsi),%ymm9 - vmovdqa 96(%rsi),%ymm10 - - vmovdqa 128(%rsi),%ymm13 - vmovdqa 160(%rsi),%ymm14 - - vpcmpeqd %ymm1,%ymm4,%ymm7 - vpcmpeqd %ymm1,%ymm8,%ymm11 - vpcmpeqd %ymm1,%ymm12,%ymm15 - - vpaddd %ymm0,%ymm4,%ymm4 - vpaddd %ymm0,%ymm8,%ymm8 - vpaddd %ymm0,%ymm12,%ymm12 - leaq 192(%rsi),%rsi - - vpand %ymm7,%ymm5,%ymm5 - vpand %ymm7,%ymm6,%ymm6 - vpand %ymm11,%ymm9,%ymm9 - vpand %ymm11,%ymm10,%ymm10 - vpand %ymm15,%ymm13,%ymm13 - vpand %ymm15,%ymm14,%ymm14 - - vpxor %ymm5,%ymm2,%ymm2 - vpxor %ymm6,%ymm3,%ymm3 - vpxor %ymm9,%ymm2,%ymm2 - vpxor %ymm10,%ymm3,%ymm3 - vpxor %ymm13,%ymm2,%ymm2 - vpxor %ymm14,%ymm3,%ymm3 - - decq %rax - jnz .Lselect_loop_avx2_w7 - - - vmovdqa 0(%rsi),%ymm5 - vmovdqa 32(%rsi),%ymm6 - - vpcmpeqd %ymm1,%ymm4,%ymm7 - - vpand %ymm7,%ymm5,%ymm5 - vpand %ymm7,%ymm6,%ymm6 - - vpxor %ymm5,%ymm2,%ymm2 - vpxor %ymm6,%ymm3,%ymm3 - - vmovdqu %ymm2,0(%rdi) - vmovdqu %ymm3,32(%rdi) - vzeroupper - ret -.cfi_endproc -.LSEH_end_ecp_nistz256_select_w7_avx2: -.size ecp_nistz256_select_w7_avx2,.-ecp_nistz256_select_w7_avx2 -.type __ecp_nistz256_add_toq,@function -.align 32 -__ecp_nistz256_add_toq: -.cfi_startproc - xorq %r11,%r11 - addq 0(%rbx),%r12 - adcq 8(%rbx),%r13 - movq %r12,%rax - adcq 16(%rbx),%r8 - adcq 24(%rbx),%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq - -.type __ecp_nistz256_sub_fromq,@function -.align 32 -__ecp_nistz256_sub_fromq: -.cfi_startproc - subq 0(%rbx),%r12 - sbbq 8(%rbx),%r13 - movq %r12,%rax - sbbq 16(%rbx),%r8 - sbbq 24(%rbx),%r9 - movq %r13,%rbp - sbbq %r11,%r11 - - addq $-1,%r12 - movq %r8,%rcx - adcq %r14,%r13 - adcq $0,%r8 - movq %r9,%r10 - adcq %r15,%r9 - testq %r11,%r11 - - cmovzq %rax,%r12 - cmovzq %rbp,%r13 - movq %r12,0(%rdi) - cmovzq %rcx,%r8 - movq %r13,8(%rdi) - cmovzq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq - -.type __ecp_nistz256_subq,@function -.align 32 -__ecp_nistz256_subq: -.cfi_startproc - subq %r12,%rax - sbbq %r13,%rbp - movq %rax,%r12 - sbbq %r8,%rcx - sbbq %r9,%r10 - movq %rbp,%r13 - sbbq %r11,%r11 - - addq $-1,%rax - movq %rcx,%r8 - adcq %r14,%rbp - adcq $0,%rcx - movq %r10,%r9 - adcq %r15,%r10 - testq %r11,%r11 - - cmovnzq %rax,%r12 - cmovnzq %rbp,%r13 - cmovnzq %rcx,%r8 - cmovnzq %r10,%r9 - - ret -.cfi_endproc -.size __ecp_nistz256_subq,.-__ecp_nistz256_subq - -.type __ecp_nistz256_mul_by_2q,@function -.align 32 -__ecp_nistz256_mul_by_2q: -.cfi_startproc - xorq %r11,%r11 - addq %r12,%r12 - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q -.globl ecp_nistz256_point_double_nohw -.hidden ecp_nistz256_point_double_nohw -.type ecp_nistz256_point_double_nohw,@function -.align 32 -ecp_nistz256_point_double_nohw: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $160+8,%rsp -.cfi_adjust_cfa_offset 32*5+8 -.Lpoint_doubleq_body: - -.Lpoint_double_shortcutq: - movdqu 0(%rsi),%xmm0 - movq %rsi,%rbx - movdqu 16(%rsi),%xmm1 - movq 32+0(%rsi),%r12 - movq 32+8(%rsi),%r13 - movq 32+16(%rsi),%r8 - movq 32+24(%rsi),%r9 - movq .Lpoly+8(%rip),%r14 - movq .Lpoly+24(%rip),%r15 - movdqa %xmm0,96(%rsp) - movdqa %xmm1,96+16(%rsp) - leaq 32(%rdi),%r10 - leaq 64(%rdi),%r11 - movq %rdi,%xmm0 - movq %r10,%xmm1 - movq %r11,%xmm2 - - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_by_2q - - movq 64+0(%rsi),%rax - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - leaq 64-0(%rsi),%rsi - leaq 64(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 0+0(%rsp),%rax - movq 8+0(%rsp),%r14 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - leaq 0(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 32(%rbx),%rax - movq 64+0(%rbx),%r9 - movq 64+8(%rbx),%r10 - movq 64+16(%rbx),%r11 - movq 64+24(%rbx),%r12 - leaq 64-0(%rbx),%rsi - leaq 32(%rbx),%rbx - movq %xmm2,%rdi - call __ecp_nistz256_mul_montq - call __ecp_nistz256_mul_by_2q - - movq 96+0(%rsp),%r12 - movq 96+8(%rsp),%r13 - leaq 64(%rsp),%rbx - movq 96+16(%rsp),%r8 - movq 96+24(%rsp),%r9 - leaq 32(%rsp),%rdi - call __ecp_nistz256_add_toq - - movq 96+0(%rsp),%r12 - movq 96+8(%rsp),%r13 - leaq 64(%rsp),%rbx - movq 96+16(%rsp),%r8 - movq 96+24(%rsp),%r9 - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 0+0(%rsp),%rax - movq 8+0(%rsp),%r14 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - movq %xmm1,%rdi - call __ecp_nistz256_sqr_montq - xorq %r9,%r9 - movq %r12,%rax - addq $-1,%r12 - movq %r13,%r10 - adcq %rsi,%r13 - movq %r14,%rcx - adcq $0,%r14 - movq %r15,%r8 - adcq %rbp,%r15 - adcq $0,%r9 - xorq %rsi,%rsi - testq $1,%rax - - cmovzq %rax,%r12 - cmovzq %r10,%r13 - cmovzq %rcx,%r14 - cmovzq %r8,%r15 - cmovzq %rsi,%r9 - - movq %r13,%rax - shrq $1,%r12 - shlq $63,%rax - movq %r14,%r10 - shrq $1,%r13 - orq %rax,%r12 - shlq $63,%r10 - movq %r15,%rcx - shrq $1,%r14 - orq %r10,%r13 - shlq $63,%rcx - movq %r12,0(%rdi) - shrq $1,%r15 - movq %r13,8(%rdi) - shlq $63,%r9 - orq %rcx,%r14 - orq %r9,%r15 - movq %r14,16(%rdi) - movq %r15,24(%rdi) - movq 64(%rsp),%rax - leaq 64(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2q - - leaq 32(%rsp),%rbx - leaq 32(%rsp),%rdi - call __ecp_nistz256_add_toq - - movq 96(%rsp),%rax - leaq 96(%rsp),%rbx - movq 0+0(%rsp),%r9 - movq 8+0(%rsp),%r10 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r11 - movq 24+0(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2q - - movq 0+32(%rsp),%rax - movq 8+32(%rsp),%r14 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r15 - movq 24+32(%rsp),%r8 - movq %xmm0,%rdi - call __ecp_nistz256_sqr_montq - - leaq 128(%rsp),%rbx - movq %r14,%r8 - movq %r15,%r9 - movq %rsi,%r14 - movq %rbp,%r15 - call __ecp_nistz256_sub_fromq - - movq 0+0(%rsp),%rax - movq 0+8(%rsp),%rbp - movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 - leaq 0(%rsp),%rdi - call __ecp_nistz256_subq - - movq 32(%rsp),%rax - leaq 32(%rsp),%rbx - movq %r12,%r14 - xorl %ecx,%ecx - movq %r12,0+0(%rsp) - movq %r13,%r10 - movq %r13,0+8(%rsp) - cmovzq %r8,%r11 - movq %r8,0+16(%rsp) - leaq 0-0(%rsp),%rsi - cmovzq %r9,%r12 - movq %r9,0+24(%rsp) - movq %r14,%r9 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq %xmm1,%rbx - movq %xmm1,%rdi - call __ecp_nistz256_sub_fromq - - leaq 160+56(%rsp),%rsi -.cfi_def_cfa %rsi,8 - movq -48(%rsi),%r15 -.cfi_restore %r15 - movq -40(%rsi),%r14 -.cfi_restore %r14 - movq -32(%rsi),%r13 -.cfi_restore %r13 - movq -24(%rsi),%r12 -.cfi_restore %r12 - movq -16(%rsi),%rbx -.cfi_restore %rbx - movq -8(%rsi),%rbp -.cfi_restore %rbp - leaq (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Lpoint_doubleq_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_double_nohw,.-ecp_nistz256_point_double_nohw -.globl ecp_nistz256_point_add_nohw -.hidden ecp_nistz256_point_add_nohw -.type ecp_nistz256_point_add_nohw,@function -.align 32 -ecp_nistz256_point_add_nohw: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $576+8,%rsp -.cfi_adjust_cfa_offset 32*18+8 -.Lpoint_addq_body: - - movdqu 0(%rsi),%xmm0 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - movdqu 64(%rsi),%xmm4 - movdqu 80(%rsi),%xmm5 - movq %rsi,%rbx - movq %rdx,%rsi - movdqa %xmm0,384(%rsp) - movdqa %xmm1,384+16(%rsp) - movdqa %xmm2,416(%rsp) - movdqa %xmm3,416+16(%rsp) - movdqa %xmm4,448(%rsp) - movdqa %xmm5,448+16(%rsp) - por %xmm4,%xmm5 - - movdqu 0(%rsi),%xmm0 - pshufd $0xb1,%xmm5,%xmm3 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - por %xmm3,%xmm5 - movdqu 48(%rsi),%xmm3 - movq 64+0(%rsi),%rax - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - movdqa %xmm0,480(%rsp) - pshufd $0x1e,%xmm5,%xmm4 - movdqa %xmm1,480+16(%rsp) - movdqu 64(%rsi),%xmm0 - movdqu 80(%rsi),%xmm1 - movdqa %xmm2,512(%rsp) - movdqa %xmm3,512+16(%rsp) - por %xmm4,%xmm5 - pxor %xmm4,%xmm4 - por %xmm0,%xmm1 - movq %rdi,%xmm0 - - leaq 64-0(%rsi),%rsi - movq %rax,544+0(%rsp) - movq %r14,544+8(%rsp) - movq %r15,544+16(%rsp) - movq %r8,544+24(%rsp) - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - pcmpeqd %xmm4,%xmm5 - pshufd $0xb1,%xmm1,%xmm4 - por %xmm1,%xmm4 - pshufd $0,%xmm5,%xmm5 - pshufd $0x1e,%xmm4,%xmm3 - por %xmm3,%xmm4 - pxor %xmm3,%xmm3 - pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - movq 64+0(%rbx),%rax - movq 64+8(%rbx),%r14 - movq 64+16(%rbx),%r15 - movq 64+24(%rbx),%r8 - movq %rbx,%xmm1 - - leaq 64-0(%rbx),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 544(%rsp),%rax - leaq 544(%rsp),%rbx - movq 0+96(%rsp),%r9 - movq 8+96(%rsp),%r10 - leaq 0+96(%rsp),%rsi - movq 16+96(%rsp),%r11 - movq 24+96(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 448(%rsp),%rax - leaq 448(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 416(%rsp),%rax - leaq 416(%rsp),%rbx - movq 0+224(%rsp),%r9 - movq 8+224(%rsp),%r10 - leaq 0+224(%rsp),%rsi - movq 16+224(%rsp),%r11 - movq 24+224(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 512(%rsp),%rax - leaq 512(%rsp),%rbx - movq 0+256(%rsp),%r9 - movq 8+256(%rsp),%r10 - leaq 0+256(%rsp),%rsi - movq 16+256(%rsp),%r11 - movq 24+256(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 224(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - orq %r13,%r12 - movdqa %xmm4,%xmm2 - orq %r8,%r12 - orq %r9,%r12 - por %xmm5,%xmm2 - movq %r12,%xmm3 - - movq 384(%rsp),%rax - leaq 384(%rsp),%rbx - movq 0+96(%rsp),%r9 - movq 8+96(%rsp),%r10 - leaq 0+96(%rsp),%rsi - movq 16+96(%rsp),%r11 - movq 24+96(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 480(%rsp),%rax - leaq 480(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 160(%rsp),%rbx - leaq 0(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - orq %r13,%r12 - orq %r8,%r12 - orq %r9,%r12 - - movq %xmm2,%r8 - movq %xmm3,%r9 - orq %r8,%r12 -.byte 0x3e - jnz .Ladd_proceedq - - - - testq %r9,%r9 - jz .Ladd_doubleq - - - - - - - movq %xmm0,%rdi - pxor %xmm0,%xmm0 - movdqu %xmm0,0(%rdi) - movdqu %xmm0,16(%rdi) - movdqu %xmm0,32(%rdi) - movdqu %xmm0,48(%rdi) - movdqu %xmm0,64(%rdi) - movdqu %xmm0,80(%rdi) - jmp .Ladd_doneq - -.align 32 -.Ladd_doubleq: - movq %xmm1,%rsi - movq %xmm0,%rdi - addq $416,%rsp -.cfi_adjust_cfa_offset -416 - jmp .Lpoint_double_shortcutq -.cfi_adjust_cfa_offset 416 - -.align 32 -.Ladd_proceedq: - movq 0+64(%rsp),%rax - movq 8+64(%rsp),%r14 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r15 - movq 24+64(%rsp),%r8 - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 448(%rsp),%rax - leaq 448(%rsp),%rbx - movq 0+0(%rsp),%r9 - movq 8+0(%rsp),%r10 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r11 - movq 24+0(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 0+0(%rsp),%rax - movq 8+0(%rsp),%r14 - leaq 0+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 544(%rsp),%rax - leaq 544(%rsp),%rbx - movq 0+352(%rsp),%r9 - movq 8+352(%rsp),%r10 - leaq 0+352(%rsp),%rsi - movq 16+352(%rsp),%r11 - movq 24+352(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 0(%rsp),%rax - leaq 0(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 160(%rsp),%rax - leaq 160(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montq - - - - - xorq %r11,%r11 - addq %r12,%r12 - leaq 96(%rsp),%rsi - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - movq 0(%rsi),%rax - cmovcq %rbp,%r13 - movq 8(%rsi),%rbp - cmovcq %rcx,%r8 - movq 16(%rsi),%rcx - cmovcq %r10,%r9 - movq 24(%rsi),%r10 - - call __ecp_nistz256_subq - - leaq 128(%rsp),%rbx - leaq 288(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 192+0(%rsp),%rax - movq 192+8(%rsp),%rbp - movq 192+16(%rsp),%rcx - movq 192+24(%rsp),%r10 - leaq 320(%rsp),%rdi - - call __ecp_nistz256_subq - - movq %r12,0(%rdi) - movq %r13,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq 128(%rsp),%rax - leaq 128(%rsp),%rbx - movq 0+224(%rsp),%r9 - movq 8+224(%rsp),%r10 - leaq 0+224(%rsp),%rsi - movq 16+224(%rsp),%r11 - movq 24+224(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 320(%rsp),%rax - leaq 320(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 320(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 256(%rsp),%rbx - leaq 320(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq %xmm0,%rdi - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 352(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 352+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 544(%rsp),%xmm2 - pand 544+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 448(%rsp),%xmm2 - pand 448+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,64(%rdi) - movdqu %xmm3,80(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 288(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 288+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 480(%rsp),%xmm2 - pand 480+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 384(%rsp),%xmm2 - pand 384+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 320(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 320+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 512(%rsp),%xmm2 - pand 512+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 416(%rsp),%xmm2 - pand 416+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - -.Ladd_doneq: - leaq 576+56(%rsp),%rsi -.cfi_def_cfa %rsi,8 - movq -48(%rsi),%r15 -.cfi_restore %r15 - movq -40(%rsi),%r14 -.cfi_restore %r14 - movq -32(%rsi),%r13 -.cfi_restore %r13 - movq -24(%rsi),%r12 -.cfi_restore %r12 - movq -16(%rsi),%rbx -.cfi_restore %rbx - movq -8(%rsi),%rbp -.cfi_restore %rbp - leaq (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Lpoint_addq_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_add_nohw,.-ecp_nistz256_point_add_nohw -.globl ecp_nistz256_point_add_affine_nohw -.hidden ecp_nistz256_point_add_affine_nohw -.type ecp_nistz256_point_add_affine_nohw,@function -.align 32 -ecp_nistz256_point_add_affine_nohw: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $480+8,%rsp -.cfi_adjust_cfa_offset 32*15+8 -.Ladd_affineq_body: - - movdqu 0(%rsi),%xmm0 - movq %rdx,%rbx - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - movdqu 64(%rsi),%xmm4 - movdqu 80(%rsi),%xmm5 - movq 64+0(%rsi),%rax - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - movdqa %xmm0,320(%rsp) - movdqa %xmm1,320+16(%rsp) - movdqa %xmm2,352(%rsp) - movdqa %xmm3,352+16(%rsp) - movdqa %xmm4,384(%rsp) - movdqa %xmm5,384+16(%rsp) - por %xmm4,%xmm5 - - movdqu 0(%rbx),%xmm0 - pshufd $0xb1,%xmm5,%xmm3 - movdqu 16(%rbx),%xmm1 - movdqu 32(%rbx),%xmm2 - por %xmm3,%xmm5 - movdqu 48(%rbx),%xmm3 - movdqa %xmm0,416(%rsp) - pshufd $0x1e,%xmm5,%xmm4 - movdqa %xmm1,416+16(%rsp) - por %xmm0,%xmm1 - movq %rdi,%xmm0 - movdqa %xmm2,448(%rsp) - movdqa %xmm3,448+16(%rsp) - por %xmm2,%xmm3 - por %xmm4,%xmm5 - pxor %xmm4,%xmm4 - por %xmm1,%xmm3 - - leaq 64-0(%rsi),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - pcmpeqd %xmm4,%xmm5 - pshufd $0xb1,%xmm3,%xmm4 - movq 0(%rbx),%rax - - movq %r12,%r9 - por %xmm3,%xmm4 - pshufd $0,%xmm5,%xmm5 - pshufd $0x1e,%xmm4,%xmm3 - movq %r13,%r10 - por %xmm3,%xmm4 - pxor %xmm3,%xmm3 - movq %r14,%r11 - pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - - leaq 32-0(%rsp),%rsi - movq %r15,%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 320(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 384(%rsp),%rax - leaq 384(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 384(%rsp),%rax - leaq 384(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 288(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 448(%rsp),%rax - leaq 448(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq 0+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 352(%rsp),%rbx - leaq 96(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 0+64(%rsp),%rax - movq 8+64(%rsp),%r14 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r15 - movq 24+64(%rsp),%r8 - leaq 128(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 0+96(%rsp),%rax - movq 8+96(%rsp),%r14 - leaq 0+96(%rsp),%rsi - movq 16+96(%rsp),%r15 - movq 24+96(%rsp),%r8 - leaq 192(%rsp),%rdi - call __ecp_nistz256_sqr_montq - - movq 128(%rsp),%rax - leaq 128(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 320(%rsp),%rax - leaq 320(%rsp),%rbx - movq 0+128(%rsp),%r9 - movq 8+128(%rsp),%r10 - leaq 0+128(%rsp),%rsi - movq 16+128(%rsp),%r11 - movq 24+128(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montq - - - - - xorq %r11,%r11 - addq %r12,%r12 - leaq 192(%rsp),%rsi - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - movq 0(%rsi),%rax - cmovcq %rbp,%r13 - movq 8(%rsi),%rbp - cmovcq %rcx,%r8 - movq 16(%rsi),%rcx - cmovcq %r10,%r9 - movq 24(%rsi),%r10 - - call __ecp_nistz256_subq - - leaq 160(%rsp),%rbx - leaq 224(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq 0+0(%rsp),%rax - movq 0+8(%rsp),%rbp - movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 - leaq 64(%rsp),%rdi - - call __ecp_nistz256_subq - - movq %r12,0(%rdi) - movq %r13,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq 352(%rsp),%rax - leaq 352(%rsp),%rbx - movq 0+160(%rsp),%r9 - movq 8+160(%rsp),%r10 - leaq 0+160(%rsp),%rsi - movq 16+160(%rsp),%r11 - movq 24+160(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montq - - movq 96(%rsp),%rax - leaq 96(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq 0+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 64(%rsp),%rdi - call __ecp_nistz256_mul_montq - - leaq 32(%rsp),%rbx - leaq 256(%rsp),%rdi - call __ecp_nistz256_sub_fromq - - movq %xmm0,%rdi - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 288(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 288+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand .LONE_mont(%rip),%xmm2 - pand .LONE_mont+16(%rip),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 384(%rsp),%xmm2 - pand 384+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,64(%rdi) - movdqu %xmm3,80(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 224(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 224+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 416(%rsp),%xmm2 - pand 416+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 320(%rsp),%xmm2 - pand 320+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 256(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 256+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 448(%rsp),%xmm2 - pand 448+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 352(%rsp),%xmm2 - pand 352+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - - leaq 480+56(%rsp),%rsi -.cfi_def_cfa %rsi,8 - movq -48(%rsi),%r15 -.cfi_restore %r15 - movq -40(%rsi),%r14 -.cfi_restore %r14 - movq -32(%rsi),%r13 -.cfi_restore %r13 - movq -24(%rsi),%r12 -.cfi_restore %r12 - movq -16(%rsi),%rbx -.cfi_restore %rbx - movq -8(%rsi),%rbp -.cfi_restore %rbp - leaq (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Ladd_affineq_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_add_affine_nohw,.-ecp_nistz256_point_add_affine_nohw -.type __ecp_nistz256_add_tox,@function -.align 32 -__ecp_nistz256_add_tox: -.cfi_startproc - xorq %r11,%r11 - adcq 0(%rbx),%r12 - adcq 8(%rbx),%r13 - movq %r12,%rax - adcq 16(%rbx),%r8 - adcq 24(%rbx),%r9 - movq %r13,%rbp - adcq $0,%r11 - - xorq %r10,%r10 - sbbq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox - -.type __ecp_nistz256_sub_fromx,@function -.align 32 -__ecp_nistz256_sub_fromx: -.cfi_startproc - xorq %r11,%r11 - sbbq 0(%rbx),%r12 - sbbq 8(%rbx),%r13 - movq %r12,%rax - sbbq 16(%rbx),%r8 - sbbq 24(%rbx),%r9 - movq %r13,%rbp - sbbq $0,%r11 - - xorq %r10,%r10 - adcq $-1,%r12 - movq %r8,%rcx - adcq %r14,%r13 - adcq $0,%r8 - movq %r9,%r10 - adcq %r15,%r9 - - btq $0,%r11 - cmovncq %rax,%r12 - cmovncq %rbp,%r13 - movq %r12,0(%rdi) - cmovncq %rcx,%r8 - movq %r13,8(%rdi) - cmovncq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx - -.type __ecp_nistz256_subx,@function -.align 32 -__ecp_nistz256_subx: -.cfi_startproc - xorq %r11,%r11 - sbbq %r12,%rax - sbbq %r13,%rbp - movq %rax,%r12 - sbbq %r8,%rcx - sbbq %r9,%r10 - movq %rbp,%r13 - sbbq $0,%r11 - - xorq %r9,%r9 - adcq $-1,%rax - movq %rcx,%r8 - adcq %r14,%rbp - adcq $0,%rcx - movq %r10,%r9 - adcq %r15,%r10 - - btq $0,%r11 - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - cmovcq %rcx,%r8 - cmovcq %r10,%r9 - - ret -.cfi_endproc -.size __ecp_nistz256_subx,.-__ecp_nistz256_subx - -.type __ecp_nistz256_mul_by_2x,@function -.align 32 -__ecp_nistz256_mul_by_2x: -.cfi_startproc - xorq %r11,%r11 - adcq %r12,%r12 - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - xorq %r10,%r10 - sbbq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - cmovcq %rbp,%r13 - movq %r12,0(%rdi) - cmovcq %rcx,%r8 - movq %r13,8(%rdi) - cmovcq %r10,%r9 - movq %r8,16(%rdi) - movq %r9,24(%rdi) - - ret -.cfi_endproc -.size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x -.globl ecp_nistz256_point_double_adx -.hidden ecp_nistz256_point_double_adx -.type ecp_nistz256_point_double_adx,@function -.align 32 -ecp_nistz256_point_double_adx: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $160+8,%rsp -.cfi_adjust_cfa_offset 32*5+8 -.Lpoint_doublex_body: - -.Lpoint_double_shortcutx: - movdqu 0(%rsi),%xmm0 - movq %rsi,%rbx - movdqu 16(%rsi),%xmm1 - movq 32+0(%rsi),%r12 - movq 32+8(%rsi),%r13 - movq 32+16(%rsi),%r8 - movq 32+24(%rsi),%r9 - movq .Lpoly+8(%rip),%r14 - movq .Lpoly+24(%rip),%r15 - movdqa %xmm0,96(%rsp) - movdqa %xmm1,96+16(%rsp) - leaq 32(%rdi),%r10 - leaq 64(%rdi),%r11 - movq %rdi,%xmm0 - movq %r10,%xmm1 - movq %r11,%xmm2 - - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_by_2x - - movq 64+0(%rsi),%rdx - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - leaq 64-128(%rsi),%rsi - leaq 64(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 0+0(%rsp),%rdx - movq 8+0(%rsp),%r14 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - leaq 0(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 32(%rbx),%rdx - movq 64+0(%rbx),%r9 - movq 64+8(%rbx),%r10 - movq 64+16(%rbx),%r11 - movq 64+24(%rbx),%r12 - leaq 64-128(%rbx),%rsi - leaq 32(%rbx),%rbx - movq %xmm2,%rdi - call __ecp_nistz256_mul_montx - call __ecp_nistz256_mul_by_2x - - movq 96+0(%rsp),%r12 - movq 96+8(%rsp),%r13 - leaq 64(%rsp),%rbx - movq 96+16(%rsp),%r8 - movq 96+24(%rsp),%r9 - leaq 32(%rsp),%rdi - call __ecp_nistz256_add_tox - - movq 96+0(%rsp),%r12 - movq 96+8(%rsp),%r13 - leaq 64(%rsp),%rbx - movq 96+16(%rsp),%r8 - movq 96+24(%rsp),%r9 - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 0+0(%rsp),%rdx - movq 8+0(%rsp),%r14 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - movq %xmm1,%rdi - call __ecp_nistz256_sqr_montx - xorq %r9,%r9 - movq %r12,%rax - addq $-1,%r12 - movq %r13,%r10 - adcq %rsi,%r13 - movq %r14,%rcx - adcq $0,%r14 - movq %r15,%r8 - adcq %rbp,%r15 - adcq $0,%r9 - xorq %rsi,%rsi - testq $1,%rax - - cmovzq %rax,%r12 - cmovzq %r10,%r13 - cmovzq %rcx,%r14 - cmovzq %r8,%r15 - cmovzq %rsi,%r9 - - movq %r13,%rax - shrq $1,%r12 - shlq $63,%rax - movq %r14,%r10 - shrq $1,%r13 - orq %rax,%r12 - shlq $63,%r10 - movq %r15,%rcx - shrq $1,%r14 - orq %r10,%r13 - shlq $63,%rcx - movq %r12,0(%rdi) - shrq $1,%r15 - movq %r13,8(%rdi) - shlq $63,%r9 - orq %rcx,%r14 - orq %r9,%r15 - movq %r14,16(%rdi) - movq %r15,24(%rdi) - movq 64(%rsp),%rdx - leaq 64(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2x - - leaq 32(%rsp),%rbx - leaq 32(%rsp),%rdi - call __ecp_nistz256_add_tox - - movq 96(%rsp),%rdx - leaq 96(%rsp),%rbx - movq 0+0(%rsp),%r9 - movq 8+0(%rsp),%r10 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r11 - movq 24+0(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_by_2x - - movq 0+32(%rsp),%rdx - movq 8+32(%rsp),%r14 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r15 - movq 24+32(%rsp),%r8 - movq %xmm0,%rdi - call __ecp_nistz256_sqr_montx - - leaq 128(%rsp),%rbx - movq %r14,%r8 - movq %r15,%r9 - movq %rsi,%r14 - movq %rbp,%r15 - call __ecp_nistz256_sub_fromx - - movq 0+0(%rsp),%rax - movq 0+8(%rsp),%rbp - movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 - leaq 0(%rsp),%rdi - call __ecp_nistz256_subx - - movq 32(%rsp),%rdx - leaq 32(%rsp),%rbx - movq %r12,%r14 - xorl %ecx,%ecx - movq %r12,0+0(%rsp) - movq %r13,%r10 - movq %r13,0+8(%rsp) - cmovzq %r8,%r11 - movq %r8,0+16(%rsp) - leaq 0-128(%rsp),%rsi - cmovzq %r9,%r12 - movq %r9,0+24(%rsp) - movq %r14,%r9 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq %xmm1,%rbx - movq %xmm1,%rdi - call __ecp_nistz256_sub_fromx - - leaq 160+56(%rsp),%rsi -.cfi_def_cfa %rsi,8 - movq -48(%rsi),%r15 -.cfi_restore %r15 - movq -40(%rsi),%r14 -.cfi_restore %r14 - movq -32(%rsi),%r13 -.cfi_restore %r13 - movq -24(%rsi),%r12 -.cfi_restore %r12 - movq -16(%rsi),%rbx -.cfi_restore %rbx - movq -8(%rsi),%rbp -.cfi_restore %rbp - leaq (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Lpoint_doublex_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_double_adx,.-ecp_nistz256_point_double_adx -.globl ecp_nistz256_point_add_adx -.hidden ecp_nistz256_point_add_adx -.type ecp_nistz256_point_add_adx,@function -.align 32 -ecp_nistz256_point_add_adx: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $576+8,%rsp -.cfi_adjust_cfa_offset 32*18+8 -.Lpoint_addx_body: - - movdqu 0(%rsi),%xmm0 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - movdqu 64(%rsi),%xmm4 - movdqu 80(%rsi),%xmm5 - movq %rsi,%rbx - movq %rdx,%rsi - movdqa %xmm0,384(%rsp) - movdqa %xmm1,384+16(%rsp) - movdqa %xmm2,416(%rsp) - movdqa %xmm3,416+16(%rsp) - movdqa %xmm4,448(%rsp) - movdqa %xmm5,448+16(%rsp) - por %xmm4,%xmm5 - - movdqu 0(%rsi),%xmm0 - pshufd $0xb1,%xmm5,%xmm3 - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - por %xmm3,%xmm5 - movdqu 48(%rsi),%xmm3 - movq 64+0(%rsi),%rdx - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - movdqa %xmm0,480(%rsp) - pshufd $0x1e,%xmm5,%xmm4 - movdqa %xmm1,480+16(%rsp) - movdqu 64(%rsi),%xmm0 - movdqu 80(%rsi),%xmm1 - movdqa %xmm2,512(%rsp) - movdqa %xmm3,512+16(%rsp) - por %xmm4,%xmm5 - pxor %xmm4,%xmm4 - por %xmm0,%xmm1 - movq %rdi,%xmm0 - - leaq 64-128(%rsi),%rsi - movq %rdx,544+0(%rsp) - movq %r14,544+8(%rsp) - movq %r15,544+16(%rsp) - movq %r8,544+24(%rsp) - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - pcmpeqd %xmm4,%xmm5 - pshufd $0xb1,%xmm1,%xmm4 - por %xmm1,%xmm4 - pshufd $0,%xmm5,%xmm5 - pshufd $0x1e,%xmm4,%xmm3 - por %xmm3,%xmm4 - pxor %xmm3,%xmm3 - pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - movq 64+0(%rbx),%rdx - movq 64+8(%rbx),%r14 - movq 64+16(%rbx),%r15 - movq 64+24(%rbx),%r8 - movq %rbx,%xmm1 - - leaq 64-128(%rbx),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 544(%rsp),%rdx - leaq 544(%rsp),%rbx - movq 0+96(%rsp),%r9 - movq 8+96(%rsp),%r10 - leaq -128+96(%rsp),%rsi - movq 16+96(%rsp),%r11 - movq 24+96(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 448(%rsp),%rdx - leaq 448(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 416(%rsp),%rdx - leaq 416(%rsp),%rbx - movq 0+224(%rsp),%r9 - movq 8+224(%rsp),%r10 - leaq -128+224(%rsp),%rsi - movq 16+224(%rsp),%r11 - movq 24+224(%rsp),%r12 - leaq 224(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 512(%rsp),%rdx - leaq 512(%rsp),%rbx - movq 0+256(%rsp),%r9 - movq 8+256(%rsp),%r10 - leaq -128+256(%rsp),%rsi - movq 16+256(%rsp),%r11 - movq 24+256(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 224(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - orq %r13,%r12 - movdqa %xmm4,%xmm2 - orq %r8,%r12 - orq %r9,%r12 - por %xmm5,%xmm2 - movq %r12,%xmm3 - - movq 384(%rsp),%rdx - leaq 384(%rsp),%rbx - movq 0+96(%rsp),%r9 - movq 8+96(%rsp),%r10 - leaq -128+96(%rsp),%rsi - movq 16+96(%rsp),%r11 - movq 24+96(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 480(%rsp),%rdx - leaq 480(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 160(%rsp),%rbx - leaq 0(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - orq %r13,%r12 - orq %r8,%r12 - orq %r9,%r12 - - movq %xmm2,%r8 - movq %xmm3,%r9 - orq %r8,%r12 -.byte 0x3e - jnz .Ladd_proceedx - - - - testq %r9,%r9 - jz .Ladd_doublex - - - - - - - movq %xmm0,%rdi - pxor %xmm0,%xmm0 - movdqu %xmm0,0(%rdi) - movdqu %xmm0,16(%rdi) - movdqu %xmm0,32(%rdi) - movdqu %xmm0,48(%rdi) - movdqu %xmm0,64(%rdi) - movdqu %xmm0,80(%rdi) - jmp .Ladd_donex - -.align 32 -.Ladd_doublex: - movq %xmm1,%rsi - movq %xmm0,%rdi - addq $416,%rsp -.cfi_adjust_cfa_offset -416 - jmp .Lpoint_double_shortcutx -.cfi_adjust_cfa_offset 416 - -.align 32 -.Ladd_proceedx: - movq 0+64(%rsp),%rdx - movq 8+64(%rsp),%r14 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r15 - movq 24+64(%rsp),%r8 - leaq 96(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 448(%rsp),%rdx - leaq 448(%rsp),%rbx - movq 0+0(%rsp),%r9 - movq 8+0(%rsp),%r10 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r11 - movq 24+0(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 0+0(%rsp),%rdx - movq 8+0(%rsp),%r14 - leaq -128+0(%rsp),%rsi - movq 16+0(%rsp),%r15 - movq 24+0(%rsp),%r8 - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 544(%rsp),%rdx - leaq 544(%rsp),%rbx - movq 0+352(%rsp),%r9 - movq 8+352(%rsp),%r10 - leaq -128+352(%rsp),%rsi - movq 16+352(%rsp),%r11 - movq 24+352(%rsp),%r12 - leaq 352(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 0(%rsp),%rdx - leaq 0(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 128(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 160(%rsp),%rdx - leaq 160(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 192(%rsp),%rdi - call __ecp_nistz256_mul_montx - - - - - xorq %r11,%r11 - addq %r12,%r12 - leaq 96(%rsp),%rsi - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - movq 0(%rsi),%rax - cmovcq %rbp,%r13 - movq 8(%rsi),%rbp - cmovcq %rcx,%r8 - movq 16(%rsi),%rcx - cmovcq %r10,%r9 - movq 24(%rsi),%r10 - - call __ecp_nistz256_subx - - leaq 128(%rsp),%rbx - leaq 288(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 192+0(%rsp),%rax - movq 192+8(%rsp),%rbp - movq 192+16(%rsp),%rcx - movq 192+24(%rsp),%r10 - leaq 320(%rsp),%rdi - - call __ecp_nistz256_subx - - movq %r12,0(%rdi) - movq %r13,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq 128(%rsp),%rdx - leaq 128(%rsp),%rbx - movq 0+224(%rsp),%r9 - movq 8+224(%rsp),%r10 - leaq -128+224(%rsp),%rsi - movq 16+224(%rsp),%r11 - movq 24+224(%rsp),%r12 - leaq 256(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 320(%rsp),%rdx - leaq 320(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 320(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 256(%rsp),%rbx - leaq 320(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq %xmm0,%rdi - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 352(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 352+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 544(%rsp),%xmm2 - pand 544+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 448(%rsp),%xmm2 - pand 448+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,64(%rdi) - movdqu %xmm3,80(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 288(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 288+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 480(%rsp),%xmm2 - pand 480+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 384(%rsp),%xmm2 - pand 384+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 320(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 320+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 512(%rsp),%xmm2 - pand 512+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 416(%rsp),%xmm2 - pand 416+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - -.Ladd_donex: - leaq 576+56(%rsp),%rsi -.cfi_def_cfa %rsi,8 - movq -48(%rsi),%r15 -.cfi_restore %r15 - movq -40(%rsi),%r14 -.cfi_restore %r14 - movq -32(%rsi),%r13 -.cfi_restore %r13 - movq -24(%rsi),%r12 -.cfi_restore %r12 - movq -16(%rsi),%rbx -.cfi_restore %rbx - movq -8(%rsi),%rbp -.cfi_restore %rbp - leaq (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Lpoint_addx_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_add_adx,.-ecp_nistz256_point_add_adx -.globl ecp_nistz256_point_add_affine_adx -.hidden ecp_nistz256_point_add_affine_adx -.type ecp_nistz256_point_add_affine_adx,@function -.align 32 -ecp_nistz256_point_add_affine_adx: -.cfi_startproc -_CET_ENDBR - pushq %rbp -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbp,-16 - pushq %rbx -.cfi_adjust_cfa_offset 8 -.cfi_offset %rbx,-24 - pushq %r12 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r12,-32 - pushq %r13 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r13,-40 - pushq %r14 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r14,-48 - pushq %r15 -.cfi_adjust_cfa_offset 8 -.cfi_offset %r15,-56 - subq $480+8,%rsp -.cfi_adjust_cfa_offset 32*15+8 -.Ladd_affinex_body: - - movdqu 0(%rsi),%xmm0 - movq %rdx,%rbx - movdqu 16(%rsi),%xmm1 - movdqu 32(%rsi),%xmm2 - movdqu 48(%rsi),%xmm3 - movdqu 64(%rsi),%xmm4 - movdqu 80(%rsi),%xmm5 - movq 64+0(%rsi),%rdx - movq 64+8(%rsi),%r14 - movq 64+16(%rsi),%r15 - movq 64+24(%rsi),%r8 - movdqa %xmm0,320(%rsp) - movdqa %xmm1,320+16(%rsp) - movdqa %xmm2,352(%rsp) - movdqa %xmm3,352+16(%rsp) - movdqa %xmm4,384(%rsp) - movdqa %xmm5,384+16(%rsp) - por %xmm4,%xmm5 - - movdqu 0(%rbx),%xmm0 - pshufd $0xb1,%xmm5,%xmm3 - movdqu 16(%rbx),%xmm1 - movdqu 32(%rbx),%xmm2 - por %xmm3,%xmm5 - movdqu 48(%rbx),%xmm3 - movdqa %xmm0,416(%rsp) - pshufd $0x1e,%xmm5,%xmm4 - movdqa %xmm1,416+16(%rsp) - por %xmm0,%xmm1 - movq %rdi,%xmm0 - movdqa %xmm2,448(%rsp) - movdqa %xmm3,448+16(%rsp) - por %xmm2,%xmm3 - por %xmm4,%xmm5 - pxor %xmm4,%xmm4 - por %xmm1,%xmm3 - - leaq 64-128(%rsi),%rsi - leaq 32(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - pcmpeqd %xmm4,%xmm5 - pshufd $0xb1,%xmm3,%xmm4 - movq 0(%rbx),%rdx - - movq %r12,%r9 - por %xmm3,%xmm4 - pshufd $0,%xmm5,%xmm5 - pshufd $0x1e,%xmm4,%xmm3 - movq %r13,%r10 - por %xmm3,%xmm4 - pxor %xmm3,%xmm3 - movq %r14,%r11 - pcmpeqd %xmm3,%xmm4 - pshufd $0,%xmm4,%xmm4 - - leaq 32-128(%rsp),%rsi - movq %r15,%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 320(%rsp),%rbx - leaq 64(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 384(%rsp),%rdx - leaq 384(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 384(%rsp),%rdx - leaq 384(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 288(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 448(%rsp),%rdx - leaq 448(%rsp),%rbx - movq 0+32(%rsp),%r9 - movq 8+32(%rsp),%r10 - leaq -128+32(%rsp),%rsi - movq 16+32(%rsp),%r11 - movq 24+32(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 352(%rsp),%rbx - leaq 96(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 0+64(%rsp),%rdx - movq 8+64(%rsp),%r14 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r15 - movq 24+64(%rsp),%r8 - leaq 128(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 0+96(%rsp),%rdx - movq 8+96(%rsp),%r14 - leaq -128+96(%rsp),%rsi - movq 16+96(%rsp),%r15 - movq 24+96(%rsp),%r8 - leaq 192(%rsp),%rdi - call __ecp_nistz256_sqr_montx - - movq 128(%rsp),%rdx - leaq 128(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 160(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 320(%rsp),%rdx - leaq 320(%rsp),%rbx - movq 0+128(%rsp),%r9 - movq 8+128(%rsp),%r10 - leaq -128+128(%rsp),%rsi - movq 16+128(%rsp),%r11 - movq 24+128(%rsp),%r12 - leaq 0(%rsp),%rdi - call __ecp_nistz256_mul_montx - - - - - xorq %r11,%r11 - addq %r12,%r12 - leaq 192(%rsp),%rsi - adcq %r13,%r13 - movq %r12,%rax - adcq %r8,%r8 - adcq %r9,%r9 - movq %r13,%rbp - adcq $0,%r11 - - subq $-1,%r12 - movq %r8,%rcx - sbbq %r14,%r13 - sbbq $0,%r8 - movq %r9,%r10 - sbbq %r15,%r9 - sbbq $0,%r11 - - cmovcq %rax,%r12 - movq 0(%rsi),%rax - cmovcq %rbp,%r13 - movq 8(%rsi),%rbp - cmovcq %rcx,%r8 - movq 16(%rsi),%rcx - cmovcq %r10,%r9 - movq 24(%rsi),%r10 - - call __ecp_nistz256_subx - - leaq 160(%rsp),%rbx - leaq 224(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq 0+0(%rsp),%rax - movq 0+8(%rsp),%rbp - movq 0+16(%rsp),%rcx - movq 0+24(%rsp),%r10 - leaq 64(%rsp),%rdi - - call __ecp_nistz256_subx - - movq %r12,0(%rdi) - movq %r13,8(%rdi) - movq %r8,16(%rdi) - movq %r9,24(%rdi) - movq 352(%rsp),%rdx - leaq 352(%rsp),%rbx - movq 0+160(%rsp),%r9 - movq 8+160(%rsp),%r10 - leaq -128+160(%rsp),%rsi - movq 16+160(%rsp),%r11 - movq 24+160(%rsp),%r12 - leaq 32(%rsp),%rdi - call __ecp_nistz256_mul_montx - - movq 96(%rsp),%rdx - leaq 96(%rsp),%rbx - movq 0+64(%rsp),%r9 - movq 8+64(%rsp),%r10 - leaq -128+64(%rsp),%rsi - movq 16+64(%rsp),%r11 - movq 24+64(%rsp),%r12 - leaq 64(%rsp),%rdi - call __ecp_nistz256_mul_montx - - leaq 32(%rsp),%rbx - leaq 256(%rsp),%rdi - call __ecp_nistz256_sub_fromx - - movq %xmm0,%rdi - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 288(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 288+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand .LONE_mont(%rip),%xmm2 - pand .LONE_mont+16(%rip),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 384(%rsp),%xmm2 - pand 384+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,64(%rdi) - movdqu %xmm3,80(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 224(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 224+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 416(%rsp),%xmm2 - pand 416+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 320(%rsp),%xmm2 - pand 320+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,0(%rdi) - movdqu %xmm3,16(%rdi) - - movdqa %xmm5,%xmm0 - movdqa %xmm5,%xmm1 - pandn 256(%rsp),%xmm0 - movdqa %xmm5,%xmm2 - pandn 256+16(%rsp),%xmm1 - movdqa %xmm5,%xmm3 - pand 448(%rsp),%xmm2 - pand 448+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - - movdqa %xmm4,%xmm0 - movdqa %xmm4,%xmm1 - pandn %xmm2,%xmm0 - movdqa %xmm4,%xmm2 - pandn %xmm3,%xmm1 - movdqa %xmm4,%xmm3 - pand 352(%rsp),%xmm2 - pand 352+16(%rsp),%xmm3 - por %xmm0,%xmm2 - por %xmm1,%xmm3 - movdqu %xmm2,32(%rdi) - movdqu %xmm3,48(%rdi) - - leaq 480+56(%rsp),%rsi -.cfi_def_cfa %rsi,8 - movq -48(%rsi),%r15 -.cfi_restore %r15 - movq -40(%rsi),%r14 -.cfi_restore %r14 - movq -32(%rsi),%r13 -.cfi_restore %r13 - movq -24(%rsi),%r12 -.cfi_restore %r12 - movq -16(%rsi),%rbx -.cfi_restore %rbx - movq -8(%rsi),%rbp -.cfi_restore %rbp - leaq (%rsi),%rsp -.cfi_def_cfa_register %rsp -.Ladd_affinex_epilogue: - ret -.cfi_endproc -.size ecp_nistz256_point_add_affine_adx,.-ecp_nistz256_point_add_affine_adx #endif
diff --git a/gen/bcm/p256-x86_64-asm-win.asm b/gen/bcm/p256-x86_64-asm-win.asm index 729e8ec..4a54536 100644 --- a/gen/bcm/p256-x86_64-asm-win.asm +++ b/gen/bcm/p256-x86_64-asm-win.asm
@@ -15,21 +15,6 @@ -section .rdata rdata align=8 -ALIGN 64 -$L$poly: - DQ 0xffffffffffffffff,0x00000000ffffffff,0x0000000000000000,0xffffffff00000001 - -$L$One: - DD 1,1,1,1,1,1,1,1 -$L$Two: - DD 2,2,2,2,2,2,2,2 -$L$Three: - DD 3,3,3,3,3,3,3,3 -$L$ONE_mont: - DQ 0x0000000000000001,0xffffffff00000000,0xffffffffffffffff,0x00000000fffffffe - - $L$ord: DQ 0xf3b9cac2fc632551,0xbce6faada7179e84,0xffffffffffffffff,0xffffffff00000000 $L$ordK: @@ -39,73 +24,6 @@ -global ecp_nistz256_neg - -ALIGN 32 -ecp_nistz256_neg: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_neg: - mov rdi,rcx - mov rsi,rdx - - - -_CET_ENDBR - push r12 - - push r13 - -$L$neg_body: - - xor r8,r8 - xor r9,r9 - xor r10,r10 - xor r11,r11 - xor r13,r13 - - sub r8,QWORD[rsi] - sbb r9,QWORD[8+rsi] - sbb r10,QWORD[16+rsi] - mov rax,r8 - sbb r11,QWORD[24+rsi] - lea rsi,[$L$poly] - mov rdx,r9 - sbb r13,0 - - add r8,QWORD[rsi] - mov rcx,r10 - adc r9,QWORD[8+rsi] - adc r10,QWORD[16+rsi] - mov r12,r11 - adc r11,QWORD[24+rsi] - test r13,r13 - - cmovz r8,rax - cmovz r9,rdx - mov QWORD[rdi],r8 - cmovz r10,rcx - mov QWORD[8+rdi],r9 - cmovz r11,r12 - mov QWORD[16+rdi],r10 - mov QWORD[24+rdi],r11 - - mov r13,QWORD[rsp] - - mov r12,QWORD[8+rsp] - - lea rsp,[16+rsp] - -$L$neg_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_neg: - - - @@ -1210,3554 +1128,6 @@ ret $L$SEH_end_ecp_nistz256_ord_sqr_mont_adx: - - - - - - -global ecp_nistz256_mul_mont_nohw - -ALIGN 32 -ecp_nistz256_mul_mont_nohw: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_mul_mont_nohw: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - -$L$mul_body: - mov rbx,rdx - mov rax,QWORD[rdx] - mov r9,QWORD[rsi] - mov r10,QWORD[8+rsi] - mov r11,QWORD[16+rsi] - mov r12,QWORD[24+rsi] - - call __ecp_nistz256_mul_montq - - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov r12,QWORD[24+rsp] - - mov rbx,QWORD[32+rsp] - - mov rbp,QWORD[40+rsp] - - lea rsp,[48+rsp] - -$L$mul_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_mul_mont_nohw: - - -ALIGN 32 -__ecp_nistz256_mul_montq: - - - - mov rbp,rax - mul r9 - mov r14,QWORD[(($L$poly+8))] - mov r8,rax - mov rax,rbp - mov r9,rdx - - mul r10 - mov r15,QWORD[(($L$poly+24))] - add r9,rax - mov rax,rbp - adc rdx,0 - mov r10,rdx - - mul r11 - add r10,rax - mov rax,rbp - adc rdx,0 - mov r11,rdx - - mul r12 - add r11,rax - mov rax,r8 - adc rdx,0 - xor r13,r13 - mov r12,rdx - - - - - - - - - - - mov rbp,r8 - shl r8,32 - mul r15 - shr rbp,32 - add r9,r8 - adc r10,rbp - adc r11,rax - mov rax,QWORD[8+rbx] - adc r12,rdx - adc r13,0 - xor r8,r8 - - - - mov rbp,rax - mul QWORD[rsi] - add r9,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[8+rsi] - add r10,rcx - adc rdx,0 - add r10,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[16+rsi] - add r11,rcx - adc rdx,0 - add r11,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[24+rsi] - add r12,rcx - adc rdx,0 - add r12,rax - mov rax,r9 - adc r13,rdx - adc r8,0 - - - - mov rbp,r9 - shl r9,32 - mul r15 - shr rbp,32 - add r10,r9 - adc r11,rbp - adc r12,rax - mov rax,QWORD[16+rbx] - adc r13,rdx - adc r8,0 - xor r9,r9 - - - - mov rbp,rax - mul QWORD[rsi] - add r10,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[8+rsi] - add r11,rcx - adc rdx,0 - add r11,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[16+rsi] - add r12,rcx - adc rdx,0 - add r12,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[24+rsi] - add r13,rcx - adc rdx,0 - add r13,rax - mov rax,r10 - adc r8,rdx - adc r9,0 - - - - mov rbp,r10 - shl r10,32 - mul r15 - shr rbp,32 - add r11,r10 - adc r12,rbp - adc r13,rax - mov rax,QWORD[24+rbx] - adc r8,rdx - adc r9,0 - xor r10,r10 - - - - mov rbp,rax - mul QWORD[rsi] - add r11,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[8+rsi] - add r12,rcx - adc rdx,0 - add r12,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[16+rsi] - add r13,rcx - adc rdx,0 - add r13,rax - mov rax,rbp - adc rdx,0 - mov rcx,rdx - - mul QWORD[24+rsi] - add r8,rcx - adc rdx,0 - add r8,rax - mov rax,r11 - adc r9,rdx - adc r10,0 - - - - mov rbp,r11 - shl r11,32 - mul r15 - shr rbp,32 - add r12,r11 - adc r13,rbp - mov rcx,r12 - adc r8,rax - adc r9,rdx - mov rbp,r13 - adc r10,0 - - - - sub r12,-1 - mov rbx,r8 - sbb r13,r14 - sbb r8,0 - mov rdx,r9 - sbb r9,r15 - sbb r10,0 - - cmovc r12,rcx - cmovc r13,rbp - mov QWORD[rdi],r12 - cmovc r8,rbx - mov QWORD[8+rdi],r13 - cmovc r9,rdx - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - - ret - - - - - - - - - - -global ecp_nistz256_sqr_mont_nohw - -ALIGN 32 -ecp_nistz256_sqr_mont_nohw: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_sqr_mont_nohw: - mov rdi,rcx - mov rsi,rdx - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - -$L$sqr_body: - mov rax,QWORD[rsi] - mov r14,QWORD[8+rsi] - mov r15,QWORD[16+rsi] - mov r8,QWORD[24+rsi] - - call __ecp_nistz256_sqr_montq - - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov r12,QWORD[24+rsp] - - mov rbx,QWORD[32+rsp] - - mov rbp,QWORD[40+rsp] - - lea rsp,[48+rsp] - -$L$sqr_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_sqr_mont_nohw: - - -ALIGN 32 -__ecp_nistz256_sqr_montq: - - mov r13,rax - mul r14 - mov r9,rax - mov rax,r15 - mov r10,rdx - - mul r13 - add r10,rax - mov rax,r8 - adc rdx,0 - mov r11,rdx - - mul r13 - add r11,rax - mov rax,r15 - adc rdx,0 - mov r12,rdx - - - mul r14 - add r11,rax - mov rax,r8 - adc rdx,0 - mov rbp,rdx - - mul r14 - add r12,rax - mov rax,r8 - adc rdx,0 - add r12,rbp - mov r13,rdx - adc r13,0 - - - mul r15 - xor r15,r15 - add r13,rax - mov rax,QWORD[rsi] - mov r14,rdx - adc r14,0 - - add r9,r9 - adc r10,r10 - adc r11,r11 - adc r12,r12 - adc r13,r13 - adc r14,r14 - adc r15,0 - - mul rax - mov r8,rax - mov rax,QWORD[8+rsi] - mov rcx,rdx - - mul rax - add r9,rcx - adc r10,rax - mov rax,QWORD[16+rsi] - adc rdx,0 - mov rcx,rdx - - mul rax - add r11,rcx - adc r12,rax - mov rax,QWORD[24+rsi] - adc rdx,0 - mov rcx,rdx - - mul rax - add r13,rcx - adc r14,rax - mov rax,r8 - adc r15,rdx - - mov rsi,QWORD[(($L$poly+8))] - mov rbp,QWORD[(($L$poly+24))] - - - - - mov rcx,r8 - shl r8,32 - mul rbp - shr rcx,32 - add r9,r8 - adc r10,rcx - adc r11,rax - mov rax,r9 - adc rdx,0 - - - - mov rcx,r9 - shl r9,32 - mov r8,rdx - mul rbp - shr rcx,32 - add r10,r9 - adc r11,rcx - adc r8,rax - mov rax,r10 - adc rdx,0 - - - - mov rcx,r10 - shl r10,32 - mov r9,rdx - mul rbp - shr rcx,32 - add r11,r10 - adc r8,rcx - adc r9,rax - mov rax,r11 - adc rdx,0 - - - - mov rcx,r11 - shl r11,32 - mov r10,rdx - mul rbp - shr rcx,32 - add r8,r11 - adc r9,rcx - adc r10,rax - adc rdx,0 - xor r11,r11 - - - - add r12,r8 - adc r13,r9 - mov r8,r12 - adc r14,r10 - adc r15,rdx - mov r9,r13 - adc r11,0 - - sub r12,-1 - mov r10,r14 - sbb r13,rsi - sbb r14,0 - mov rcx,r15 - sbb r15,rbp - sbb r11,0 - - cmovc r12,r8 - cmovc r13,r9 - mov QWORD[rdi],r12 - cmovc r14,r10 - mov QWORD[8+rdi],r13 - cmovc r15,rcx - mov QWORD[16+rdi],r14 - mov QWORD[24+rdi],r15 - - ret - - -global ecp_nistz256_mul_mont_adx - -ALIGN 32 -ecp_nistz256_mul_mont_adx: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_mul_mont_adx: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - -$L$mulx_body: - mov rbx,rdx - mov rdx,QWORD[rdx] - mov r9,QWORD[rsi] - mov r10,QWORD[8+rsi] - mov r11,QWORD[16+rsi] - mov r12,QWORD[24+rsi] - lea rsi,[((-128))+rsi] - - call __ecp_nistz256_mul_montx - - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov r12,QWORD[24+rsp] - - mov rbx,QWORD[32+rsp] - - mov rbp,QWORD[40+rsp] - - lea rsp,[48+rsp] - -$L$mulx_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_mul_mont_adx: - - -ALIGN 32 -__ecp_nistz256_mul_montx: - - - - mulx r9,r8,r9 - mulx r10,rcx,r10 - mov r14,32 - xor r13,r13 - mulx r11,rbp,r11 - mov r15,QWORD[(($L$poly+24))] - adc r9,rcx - mulx r12,rcx,r12 - mov rdx,r8 - adc r10,rbp - shlx rbp,r8,r14 - adc r11,rcx - shrx rcx,r8,r14 - adc r12,0 - - - - add r9,rbp - adc r10,rcx - - mulx rbp,rcx,r15 - mov rdx,QWORD[8+rbx] - adc r11,rcx - adc r12,rbp - adc r13,0 - xor r8,r8 - - - - mulx rbp,rcx,QWORD[((0+128))+rsi] - adcx r9,rcx - adox r10,rbp - - mulx rbp,rcx,QWORD[((8+128))+rsi] - adcx r10,rcx - adox r11,rbp - - mulx rbp,rcx,QWORD[((16+128))+rsi] - adcx r11,rcx - adox r12,rbp - - mulx rbp,rcx,QWORD[((24+128))+rsi] - mov rdx,r9 - adcx r12,rcx - shlx rcx,r9,r14 - adox r13,rbp - shrx rbp,r9,r14 - - adcx r13,r8 - adox r8,r8 - adc r8,0 - - - - add r10,rcx - adc r11,rbp - - mulx rbp,rcx,r15 - mov rdx,QWORD[16+rbx] - adc r12,rcx - adc r13,rbp - adc r8,0 - xor r9,r9 - - - - mulx rbp,rcx,QWORD[((0+128))+rsi] - adcx r10,rcx - adox r11,rbp - - mulx rbp,rcx,QWORD[((8+128))+rsi] - adcx r11,rcx - adox r12,rbp - - mulx rbp,rcx,QWORD[((16+128))+rsi] - adcx r12,rcx - adox r13,rbp - - mulx rbp,rcx,QWORD[((24+128))+rsi] - mov rdx,r10 - adcx r13,rcx - shlx rcx,r10,r14 - adox r8,rbp - shrx rbp,r10,r14 - - adcx r8,r9 - adox r9,r9 - adc r9,0 - - - - add r11,rcx - adc r12,rbp - - mulx rbp,rcx,r15 - mov rdx,QWORD[24+rbx] - adc r13,rcx - adc r8,rbp - adc r9,0 - xor r10,r10 - - - - mulx rbp,rcx,QWORD[((0+128))+rsi] - adcx r11,rcx - adox r12,rbp - - mulx rbp,rcx,QWORD[((8+128))+rsi] - adcx r12,rcx - adox r13,rbp - - mulx rbp,rcx,QWORD[((16+128))+rsi] - adcx r13,rcx - adox r8,rbp - - mulx rbp,rcx,QWORD[((24+128))+rsi] - mov rdx,r11 - adcx r8,rcx - shlx rcx,r11,r14 - adox r9,rbp - shrx rbp,r11,r14 - - adcx r9,r10 - adox r10,r10 - adc r10,0 - - - - add r12,rcx - adc r13,rbp - - mulx rbp,rcx,r15 - mov rbx,r12 - mov r14,QWORD[(($L$poly+8))] - adc r8,rcx - mov rdx,r13 - adc r9,rbp - adc r10,0 - - - - xor eax,eax - mov rcx,r8 - sbb r12,-1 - sbb r13,r14 - sbb r8,0 - mov rbp,r9 - sbb r9,r15 - sbb r10,0 - - cmovc r12,rbx - cmovc r13,rdx - mov QWORD[rdi],r12 - cmovc r8,rcx - mov QWORD[8+rdi],r13 - cmovc r9,rbp - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - - ret - - - -global ecp_nistz256_sqr_mont_adx - -ALIGN 32 -ecp_nistz256_sqr_mont_adx: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_sqr_mont_adx: - mov rdi,rcx - mov rsi,rdx - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - -$L$sqrx_body: - mov rdx,QWORD[rsi] - mov r14,QWORD[8+rsi] - mov r15,QWORD[16+rsi] - mov r8,QWORD[24+rsi] - lea rsi,[((-128))+rsi] - - call __ecp_nistz256_sqr_montx - - mov r15,QWORD[rsp] - - mov r14,QWORD[8+rsp] - - mov r13,QWORD[16+rsp] - - mov r12,QWORD[24+rsp] - - mov rbx,QWORD[32+rsp] - - mov rbp,QWORD[40+rsp] - - lea rsp,[48+rsp] - -$L$sqrx_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_sqr_mont_adx: - - -ALIGN 32 -__ecp_nistz256_sqr_montx: - - mulx r10,r9,r14 - mulx r11,rcx,r15 - xor eax,eax - adc r10,rcx - mulx r12,rbp,r8 - mov rdx,r14 - adc r11,rbp - adc r12,0 - xor r13,r13 - - - mulx rbp,rcx,r15 - adcx r11,rcx - adox r12,rbp - - mulx rbp,rcx,r8 - mov rdx,r15 - adcx r12,rcx - adox r13,rbp - adc r13,0 - - - mulx r14,rcx,r8 - mov rdx,QWORD[((0+128))+rsi] - xor r15,r15 - adcx r9,r9 - adox r13,rcx - adcx r10,r10 - adox r14,r15 - - mulx rbp,r8,rdx - mov rdx,QWORD[((8+128))+rsi] - adcx r11,r11 - adox r9,rbp - adcx r12,r12 - mulx rax,rcx,rdx - mov rdx,QWORD[((16+128))+rsi] - adcx r13,r13 - adox r10,rcx - adcx r14,r14 - DB 0x67 - mulx rbp,rcx,rdx - mov rdx,QWORD[((24+128))+rsi] - adox r11,rax - adcx r15,r15 - adox r12,rcx - mov rsi,32 - adox r13,rbp - DB 0x67,0x67 - mulx rax,rcx,rdx - mov rdx,QWORD[(($L$poly+24))] - adox r14,rcx - shlx rcx,r8,rsi - adox r15,rax - shrx rax,r8,rsi - mov rbp,rdx - - - add r9,rcx - adc r10,rax - - mulx r8,rcx,r8 - adc r11,rcx - shlx rcx,r9,rsi - adc r8,0 - shrx rax,r9,rsi - - - add r10,rcx - adc r11,rax - - mulx r9,rcx,r9 - adc r8,rcx - shlx rcx,r10,rsi - adc r9,0 - shrx rax,r10,rsi - - - add r11,rcx - adc r8,rax - - mulx r10,rcx,r10 - adc r9,rcx - shlx rcx,r11,rsi - adc r10,0 - shrx rax,r11,rsi - - - add r8,rcx - adc r9,rax - - mulx r11,rcx,r11 - adc r10,rcx - adc r11,0 - - xor rdx,rdx - add r12,r8 - mov rsi,QWORD[(($L$poly+8))] - adc r13,r9 - mov r8,r12 - adc r14,r10 - adc r15,r11 - mov r9,r13 - adc rdx,0 - - sub r12,-1 - mov r10,r14 - sbb r13,rsi - sbb r14,0 - mov r11,r15 - sbb r15,rbp - sbb rdx,0 - - cmovc r12,r8 - cmovc r13,r9 - mov QWORD[rdi],r12 - cmovc r14,r10 - mov QWORD[8+rdi],r13 - cmovc r15,r11 - mov QWORD[16+rdi],r14 - mov QWORD[24+rdi],r15 - - ret - - - - -global ecp_nistz256_select_w5_nohw - -ALIGN 32 -ecp_nistz256_select_w5_nohw: - -_CET_ENDBR - lea rax,[((-136))+rsp] -$L$SEH_begin_ecp_nistz256_select_w5_nohw: - DB 0x48,0x8d,0x60,0xe0 - DB 0x0f,0x29,0x70,0xe0 - DB 0x0f,0x29,0x78,0xf0 - DB 0x44,0x0f,0x29,0x00 - DB 0x44,0x0f,0x29,0x48,0x10 - DB 0x44,0x0f,0x29,0x50,0x20 - DB 0x44,0x0f,0x29,0x58,0x30 - DB 0x44,0x0f,0x29,0x60,0x40 - DB 0x44,0x0f,0x29,0x68,0x50 - DB 0x44,0x0f,0x29,0x70,0x60 - DB 0x44,0x0f,0x29,0x78,0x70 - movdqa xmm0,XMMWORD[$L$One] - movd xmm1,r8d - - pxor xmm2,xmm2 - pxor xmm3,xmm3 - pxor xmm4,xmm4 - pxor xmm5,xmm5 - pxor xmm6,xmm6 - pxor xmm7,xmm7 - - movdqa xmm8,xmm0 - pshufd xmm1,xmm1,0 - - mov rax,16 -$L$select_loop_sse_w5: - - movdqa xmm15,xmm8 - paddd xmm8,xmm0 - pcmpeqd xmm15,xmm1 - - movdqa xmm9,XMMWORD[rdx] - movdqa xmm10,XMMWORD[16+rdx] - movdqa xmm11,XMMWORD[32+rdx] - movdqa xmm12,XMMWORD[48+rdx] - movdqa xmm13,XMMWORD[64+rdx] - movdqa xmm14,XMMWORD[80+rdx] - lea rdx,[96+rdx] - - pand xmm9,xmm15 - pand xmm10,xmm15 - por xmm2,xmm9 - pand xmm11,xmm15 - por xmm3,xmm10 - pand xmm12,xmm15 - por xmm4,xmm11 - pand xmm13,xmm15 - por xmm5,xmm12 - pand xmm14,xmm15 - por xmm6,xmm13 - por xmm7,xmm14 - - dec rax - jnz NEAR $L$select_loop_sse_w5 - - movdqu XMMWORD[rcx],xmm2 - movdqu XMMWORD[16+rcx],xmm3 - movdqu XMMWORD[32+rcx],xmm4 - movdqu XMMWORD[48+rcx],xmm5 - movdqu XMMWORD[64+rcx],xmm6 - movdqu XMMWORD[80+rcx],xmm7 - movaps xmm6,XMMWORD[rsp] - movaps xmm7,XMMWORD[16+rsp] - movaps xmm8,XMMWORD[32+rsp] - movaps xmm9,XMMWORD[48+rsp] - movaps xmm10,XMMWORD[64+rsp] - movaps xmm11,XMMWORD[80+rsp] - movaps xmm12,XMMWORD[96+rsp] - movaps xmm13,XMMWORD[112+rsp] - movaps xmm14,XMMWORD[128+rsp] - movaps xmm15,XMMWORD[144+rsp] - lea rsp,[168+rsp] - ret - -$L$SEH_end_ecp_nistz256_select_w5_nohw: - - - - -global ecp_nistz256_select_w7_nohw - -ALIGN 32 -ecp_nistz256_select_w7_nohw: - -_CET_ENDBR - lea rax,[((-136))+rsp] -$L$SEH_begin_ecp_nistz256_select_w7_nohw: - DB 0x48,0x8d,0x60,0xe0 - DB 0x0f,0x29,0x70,0xe0 - DB 0x0f,0x29,0x78,0xf0 - DB 0x44,0x0f,0x29,0x00 - DB 0x44,0x0f,0x29,0x48,0x10 - DB 0x44,0x0f,0x29,0x50,0x20 - DB 0x44,0x0f,0x29,0x58,0x30 - DB 0x44,0x0f,0x29,0x60,0x40 - DB 0x44,0x0f,0x29,0x68,0x50 - DB 0x44,0x0f,0x29,0x70,0x60 - DB 0x44,0x0f,0x29,0x78,0x70 - movdqa xmm8,XMMWORD[$L$One] - movd xmm1,r8d - - pxor xmm2,xmm2 - pxor xmm3,xmm3 - pxor xmm4,xmm4 - pxor xmm5,xmm5 - - movdqa xmm0,xmm8 - pshufd xmm1,xmm1,0 - mov rax,64 - -$L$select_loop_sse_w7: - movdqa xmm15,xmm8 - paddd xmm8,xmm0 - movdqa xmm9,XMMWORD[rdx] - movdqa xmm10,XMMWORD[16+rdx] - pcmpeqd xmm15,xmm1 - movdqa xmm11,XMMWORD[32+rdx] - movdqa xmm12,XMMWORD[48+rdx] - lea rdx,[64+rdx] - - pand xmm9,xmm15 - pand xmm10,xmm15 - por xmm2,xmm9 - pand xmm11,xmm15 - por xmm3,xmm10 - pand xmm12,xmm15 - por xmm4,xmm11 - prefetcht0 [255+rdx] - por xmm5,xmm12 - - dec rax - jnz NEAR $L$select_loop_sse_w7 - - movdqu XMMWORD[rcx],xmm2 - movdqu XMMWORD[16+rcx],xmm3 - movdqu XMMWORD[32+rcx],xmm4 - movdqu XMMWORD[48+rcx],xmm5 - movaps xmm6,XMMWORD[rsp] - movaps xmm7,XMMWORD[16+rsp] - movaps xmm8,XMMWORD[32+rsp] - movaps xmm9,XMMWORD[48+rsp] - movaps xmm10,XMMWORD[64+rsp] - movaps xmm11,XMMWORD[80+rsp] - movaps xmm12,XMMWORD[96+rsp] - movaps xmm13,XMMWORD[112+rsp] - movaps xmm14,XMMWORD[128+rsp] - movaps xmm15,XMMWORD[144+rsp] - lea rsp,[168+rsp] - ret - -$L$SEH_end_ecp_nistz256_select_w7_nohw: - - - -global ecp_nistz256_select_w5_avx2 - -ALIGN 32 -ecp_nistz256_select_w5_avx2: - -_CET_ENDBR - vzeroupper - lea rax,[((-136))+rsp] - mov r11,rsp -$L$SEH_begin_ecp_nistz256_select_w5_avx2: - DB 0x48,0x8d,0x60,0xe0 - DB 0xc5,0xf8,0x29,0x70,0xe0 - DB 0xc5,0xf8,0x29,0x78,0xf0 - DB 0xc5,0x78,0x29,0x40,0x00 - DB 0xc5,0x78,0x29,0x48,0x10 - DB 0xc5,0x78,0x29,0x50,0x20 - DB 0xc5,0x78,0x29,0x58,0x30 - DB 0xc5,0x78,0x29,0x60,0x40 - DB 0xc5,0x78,0x29,0x68,0x50 - DB 0xc5,0x78,0x29,0x70,0x60 - DB 0xc5,0x78,0x29,0x78,0x70 - vmovdqa ymm0,YMMWORD[$L$Two] - - vpxor ymm2,ymm2,ymm2 - vpxor ymm3,ymm3,ymm3 - vpxor ymm4,ymm4,ymm4 - - vmovdqa ymm5,YMMWORD[$L$One] - vmovdqa ymm10,YMMWORD[$L$Two] - - vmovd xmm1,r8d - vpermd ymm1,ymm2,ymm1 - - mov rax,8 -$L$select_loop_avx2_w5: - - vmovdqa ymm6,YMMWORD[rdx] - vmovdqa ymm7,YMMWORD[32+rdx] - vmovdqa ymm8,YMMWORD[64+rdx] - - vmovdqa ymm11,YMMWORD[96+rdx] - vmovdqa ymm12,YMMWORD[128+rdx] - vmovdqa ymm13,YMMWORD[160+rdx] - - vpcmpeqd ymm9,ymm5,ymm1 - vpcmpeqd ymm14,ymm10,ymm1 - - vpaddd ymm5,ymm5,ymm0 - vpaddd ymm10,ymm10,ymm0 - lea rdx,[192+rdx] - - vpand ymm6,ymm6,ymm9 - vpand ymm7,ymm7,ymm9 - vpand ymm8,ymm8,ymm9 - vpand ymm11,ymm11,ymm14 - vpand ymm12,ymm12,ymm14 - vpand ymm13,ymm13,ymm14 - - vpxor ymm2,ymm2,ymm6 - vpxor ymm3,ymm3,ymm7 - vpxor ymm4,ymm4,ymm8 - vpxor ymm2,ymm2,ymm11 - vpxor ymm3,ymm3,ymm12 - vpxor ymm4,ymm4,ymm13 - - dec rax - jnz NEAR $L$select_loop_avx2_w5 - - vmovdqu YMMWORD[rcx],ymm2 - vmovdqu YMMWORD[32+rcx],ymm3 - vmovdqu YMMWORD[64+rcx],ymm4 - vzeroupper - movaps xmm6,XMMWORD[rsp] - movaps xmm7,XMMWORD[16+rsp] - movaps xmm8,XMMWORD[32+rsp] - movaps xmm9,XMMWORD[48+rsp] - movaps xmm10,XMMWORD[64+rsp] - movaps xmm11,XMMWORD[80+rsp] - movaps xmm12,XMMWORD[96+rsp] - movaps xmm13,XMMWORD[112+rsp] - movaps xmm14,XMMWORD[128+rsp] - movaps xmm15,XMMWORD[144+rsp] - lea rsp,[r11] - ret - -$L$SEH_end_ecp_nistz256_select_w5_avx2: - - - - -global ecp_nistz256_select_w7_avx2 - -ALIGN 32 -ecp_nistz256_select_w7_avx2: - -_CET_ENDBR - vzeroupper - mov r11,rsp - lea rax,[((-136))+rsp] -$L$SEH_begin_ecp_nistz256_select_w7_avx2: - DB 0x48,0x8d,0x60,0xe0 - DB 0xc5,0xf8,0x29,0x70,0xe0 - DB 0xc5,0xf8,0x29,0x78,0xf0 - DB 0xc5,0x78,0x29,0x40,0x00 - DB 0xc5,0x78,0x29,0x48,0x10 - DB 0xc5,0x78,0x29,0x50,0x20 - DB 0xc5,0x78,0x29,0x58,0x30 - DB 0xc5,0x78,0x29,0x60,0x40 - DB 0xc5,0x78,0x29,0x68,0x50 - DB 0xc5,0x78,0x29,0x70,0x60 - DB 0xc5,0x78,0x29,0x78,0x70 - vmovdqa ymm0,YMMWORD[$L$Three] - - vpxor ymm2,ymm2,ymm2 - vpxor ymm3,ymm3,ymm3 - - vmovdqa ymm4,YMMWORD[$L$One] - vmovdqa ymm8,YMMWORD[$L$Two] - vmovdqa ymm12,YMMWORD[$L$Three] - - vmovd xmm1,r8d - vpermd ymm1,ymm2,ymm1 - - - mov rax,21 -$L$select_loop_avx2_w7: - - vmovdqa ymm5,YMMWORD[rdx] - vmovdqa ymm6,YMMWORD[32+rdx] - - vmovdqa ymm9,YMMWORD[64+rdx] - vmovdqa ymm10,YMMWORD[96+rdx] - - vmovdqa ymm13,YMMWORD[128+rdx] - vmovdqa ymm14,YMMWORD[160+rdx] - - vpcmpeqd ymm7,ymm4,ymm1 - vpcmpeqd ymm11,ymm8,ymm1 - vpcmpeqd ymm15,ymm12,ymm1 - - vpaddd ymm4,ymm4,ymm0 - vpaddd ymm8,ymm8,ymm0 - vpaddd ymm12,ymm12,ymm0 - lea rdx,[192+rdx] - - vpand ymm5,ymm5,ymm7 - vpand ymm6,ymm6,ymm7 - vpand ymm9,ymm9,ymm11 - vpand ymm10,ymm10,ymm11 - vpand ymm13,ymm13,ymm15 - vpand ymm14,ymm14,ymm15 - - vpxor ymm2,ymm2,ymm5 - vpxor ymm3,ymm3,ymm6 - vpxor ymm2,ymm2,ymm9 - vpxor ymm3,ymm3,ymm10 - vpxor ymm2,ymm2,ymm13 - vpxor ymm3,ymm3,ymm14 - - dec rax - jnz NEAR $L$select_loop_avx2_w7 - - - vmovdqa ymm5,YMMWORD[rdx] - vmovdqa ymm6,YMMWORD[32+rdx] - - vpcmpeqd ymm7,ymm4,ymm1 - - vpand ymm5,ymm5,ymm7 - vpand ymm6,ymm6,ymm7 - - vpxor ymm2,ymm2,ymm5 - vpxor ymm3,ymm3,ymm6 - - vmovdqu YMMWORD[rcx],ymm2 - vmovdqu YMMWORD[32+rcx],ymm3 - vzeroupper - movaps xmm6,XMMWORD[rsp] - movaps xmm7,XMMWORD[16+rsp] - movaps xmm8,XMMWORD[32+rsp] - movaps xmm9,XMMWORD[48+rsp] - movaps xmm10,XMMWORD[64+rsp] - movaps xmm11,XMMWORD[80+rsp] - movaps xmm12,XMMWORD[96+rsp] - movaps xmm13,XMMWORD[112+rsp] - movaps xmm14,XMMWORD[128+rsp] - movaps xmm15,XMMWORD[144+rsp] - lea rsp,[r11] - ret - -$L$SEH_end_ecp_nistz256_select_w7_avx2: - - -ALIGN 32 -__ecp_nistz256_add_toq: - - xor r11,r11 - add r12,QWORD[rbx] - adc r13,QWORD[8+rbx] - mov rax,r12 - adc r8,QWORD[16+rbx] - adc r9,QWORD[24+rbx] - mov rbp,r13 - adc r11,0 - - sub r12,-1 - mov rcx,r8 - sbb r13,r14 - sbb r8,0 - mov r10,r9 - sbb r9,r15 - sbb r11,0 - - cmovc r12,rax - cmovc r13,rbp - mov QWORD[rdi],r12 - cmovc r8,rcx - mov QWORD[8+rdi],r13 - cmovc r9,r10 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - - ret - - - - -ALIGN 32 -__ecp_nistz256_sub_fromq: - - sub r12,QWORD[rbx] - sbb r13,QWORD[8+rbx] - mov rax,r12 - sbb r8,QWORD[16+rbx] - sbb r9,QWORD[24+rbx] - mov rbp,r13 - sbb r11,r11 - - add r12,-1 - mov rcx,r8 - adc r13,r14 - adc r8,0 - mov r10,r9 - adc r9,r15 - test r11,r11 - - cmovz r12,rax - cmovz r13,rbp - mov QWORD[rdi],r12 - cmovz r8,rcx - mov QWORD[8+rdi],r13 - cmovz r9,r10 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - - ret - - - - -ALIGN 32 -__ecp_nistz256_subq: - - sub rax,r12 - sbb rbp,r13 - mov r12,rax - sbb rcx,r8 - sbb r10,r9 - mov r13,rbp - sbb r11,r11 - - add rax,-1 - mov r8,rcx - adc rbp,r14 - adc rcx,0 - mov r9,r10 - adc r10,r15 - test r11,r11 - - cmovnz r12,rax - cmovnz r13,rbp - cmovnz r8,rcx - cmovnz r9,r10 - - ret - - - - -ALIGN 32 -__ecp_nistz256_mul_by_2q: - - xor r11,r11 - add r12,r12 - adc r13,r13 - mov rax,r12 - adc r8,r8 - adc r9,r9 - mov rbp,r13 - adc r11,0 - - sub r12,-1 - mov rcx,r8 - sbb r13,r14 - sbb r8,0 - mov r10,r9 - sbb r9,r15 - sbb r11,0 - - cmovc r12,rax - cmovc r13,rbp - mov QWORD[rdi],r12 - cmovc r8,rcx - mov QWORD[8+rdi],r13 - cmovc r9,r10 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - - ret - - -global ecp_nistz256_point_double_nohw - -ALIGN 32 -ecp_nistz256_point_double_nohw: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_point_double_nohw: - mov rdi,rcx - mov rsi,rdx - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,32*5+8 - -$L$point_doubleq_body: - -$L$point_double_shortcutq: - movdqu xmm0,XMMWORD[rsi] - mov rbx,rsi - movdqu xmm1,XMMWORD[16+rsi] - mov r12,QWORD[((32+0))+rsi] - mov r13,QWORD[((32+8))+rsi] - mov r8,QWORD[((32+16))+rsi] - mov r9,QWORD[((32+24))+rsi] - mov r14,QWORD[(($L$poly+8))] - mov r15,QWORD[(($L$poly+24))] - movdqa XMMWORD[96+rsp],xmm0 - movdqa XMMWORD[(96+16)+rsp],xmm1 - lea r10,[32+rdi] - lea r11,[64+rdi] - movq xmm0,rdi - movq xmm1,r10 - movq xmm2,r11 - - lea rdi,[rsp] - call __ecp_nistz256_mul_by_2q - - mov rax,QWORD[((64+0))+rsi] - mov r14,QWORD[((64+8))+rsi] - mov r15,QWORD[((64+16))+rsi] - mov r8,QWORD[((64+24))+rsi] - lea rsi,[((64-0))+rsi] - lea rdi,[64+rsp] - call __ecp_nistz256_sqr_montq - - mov rax,QWORD[((0+0))+rsp] - mov r14,QWORD[((8+0))+rsp] - lea rsi,[((0+0))+rsp] - mov r15,QWORD[((16+0))+rsp] - mov r8,QWORD[((24+0))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_sqr_montq - - mov rax,QWORD[32+rbx] - mov r9,QWORD[((64+0))+rbx] - mov r10,QWORD[((64+8))+rbx] - mov r11,QWORD[((64+16))+rbx] - mov r12,QWORD[((64+24))+rbx] - lea rsi,[((64-0))+rbx] - lea rbx,[32+rbx] - movq rdi,xmm2 - call __ecp_nistz256_mul_montq - call __ecp_nistz256_mul_by_2q - - mov r12,QWORD[((96+0))+rsp] - mov r13,QWORD[((96+8))+rsp] - lea rbx,[64+rsp] - mov r8,QWORD[((96+16))+rsp] - mov r9,QWORD[((96+24))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_add_toq - - mov r12,QWORD[((96+0))+rsp] - mov r13,QWORD[((96+8))+rsp] - lea rbx,[64+rsp] - mov r8,QWORD[((96+16))+rsp] - mov r9,QWORD[((96+24))+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromq - - mov rax,QWORD[((0+0))+rsp] - mov r14,QWORD[((8+0))+rsp] - lea rsi,[((0+0))+rsp] - mov r15,QWORD[((16+0))+rsp] - mov r8,QWORD[((24+0))+rsp] - movq rdi,xmm1 - call __ecp_nistz256_sqr_montq - xor r9,r9 - mov rax,r12 - add r12,-1 - mov r10,r13 - adc r13,rsi - mov rcx,r14 - adc r14,0 - mov r8,r15 - adc r15,rbp - adc r9,0 - xor rsi,rsi - test rax,1 - - cmovz r12,rax - cmovz r13,r10 - cmovz r14,rcx - cmovz r15,r8 - cmovz r9,rsi - - mov rax,r13 - shr r12,1 - shl rax,63 - mov r10,r14 - shr r13,1 - or r12,rax - shl r10,63 - mov rcx,r15 - shr r14,1 - or r13,r10 - shl rcx,63 - mov QWORD[rdi],r12 - shr r15,1 - mov QWORD[8+rdi],r13 - shl r9,63 - or r14,rcx - or r15,r9 - mov QWORD[16+rdi],r14 - mov QWORD[24+rdi],r15 - mov rax,QWORD[64+rsp] - lea rbx,[64+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((0+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montq - - lea rdi,[128+rsp] - call __ecp_nistz256_mul_by_2q - - lea rbx,[32+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_add_toq - - mov rax,QWORD[96+rsp] - lea rbx,[96+rsp] - mov r9,QWORD[((0+0))+rsp] - mov r10,QWORD[((8+0))+rsp] - lea rsi,[((0+0))+rsp] - mov r11,QWORD[((16+0))+rsp] - mov r12,QWORD[((24+0))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_mul_montq - - lea rdi,[128+rsp] - call __ecp_nistz256_mul_by_2q - - mov rax,QWORD[((0+32))+rsp] - mov r14,QWORD[((8+32))+rsp] - lea rsi,[((0+32))+rsp] - mov r15,QWORD[((16+32))+rsp] - mov r8,QWORD[((24+32))+rsp] - movq rdi,xmm0 - call __ecp_nistz256_sqr_montq - - lea rbx,[128+rsp] - mov r8,r14 - mov r9,r15 - mov r14,rsi - mov r15,rbp - call __ecp_nistz256_sub_fromq - - mov rax,QWORD[((0+0))+rsp] - mov rbp,QWORD[((0+8))+rsp] - mov rcx,QWORD[((0+16))+rsp] - mov r10,QWORD[((0+24))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_subq - - mov rax,QWORD[32+rsp] - lea rbx,[32+rsp] - mov r14,r12 - xor ecx,ecx - mov QWORD[((0+0))+rsp],r12 - mov r10,r13 - mov QWORD[((0+8))+rsp],r13 - cmovz r11,r8 - mov QWORD[((0+16))+rsp],r8 - lea rsi,[((0-0))+rsp] - cmovz r12,r9 - mov QWORD[((0+24))+rsp],r9 - mov r9,r14 - lea rdi,[rsp] - call __ecp_nistz256_mul_montq - - movq rbx,xmm1 - movq rdi,xmm1 - call __ecp_nistz256_sub_fromq - - lea rsi,[((160+56))+rsp] - - mov r15,QWORD[((-48))+rsi] - - mov r14,QWORD[((-40))+rsi] - - mov r13,QWORD[((-32))+rsi] - - mov r12,QWORD[((-24))+rsi] - - mov rbx,QWORD[((-16))+rsi] - - mov rbp,QWORD[((-8))+rsi] - - lea rsp,[rsi] - -$L$point_doubleq_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_point_double_nohw: -global ecp_nistz256_point_add_nohw - -ALIGN 32 -ecp_nistz256_point_add_nohw: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_point_add_nohw: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,32*18+8 - -$L$point_addq_body: - - movdqu xmm0,XMMWORD[rsi] - movdqu xmm1,XMMWORD[16+rsi] - movdqu xmm2,XMMWORD[32+rsi] - movdqu xmm3,XMMWORD[48+rsi] - movdqu xmm4,XMMWORD[64+rsi] - movdqu xmm5,XMMWORD[80+rsi] - mov rbx,rsi - mov rsi,rdx - movdqa XMMWORD[384+rsp],xmm0 - movdqa XMMWORD[(384+16)+rsp],xmm1 - movdqa XMMWORD[416+rsp],xmm2 - movdqa XMMWORD[(416+16)+rsp],xmm3 - movdqa XMMWORD[448+rsp],xmm4 - movdqa XMMWORD[(448+16)+rsp],xmm5 - por xmm5,xmm4 - - movdqu xmm0,XMMWORD[rsi] - pshufd xmm3,xmm5,0xb1 - movdqu xmm1,XMMWORD[16+rsi] - movdqu xmm2,XMMWORD[32+rsi] - por xmm5,xmm3 - movdqu xmm3,XMMWORD[48+rsi] - mov rax,QWORD[((64+0))+rsi] - mov r14,QWORD[((64+8))+rsi] - mov r15,QWORD[((64+16))+rsi] - mov r8,QWORD[((64+24))+rsi] - movdqa XMMWORD[480+rsp],xmm0 - pshufd xmm4,xmm5,0x1e - movdqa XMMWORD[(480+16)+rsp],xmm1 - movdqu xmm0,XMMWORD[64+rsi] - movdqu xmm1,XMMWORD[80+rsi] - movdqa XMMWORD[512+rsp],xmm2 - movdqa XMMWORD[(512+16)+rsp],xmm3 - por xmm5,xmm4 - pxor xmm4,xmm4 - por xmm1,xmm0 - movq xmm0,rdi - - lea rsi,[((64-0))+rsi] - mov QWORD[((544+0))+rsp],rax - mov QWORD[((544+8))+rsp],r14 - mov QWORD[((544+16))+rsp],r15 - mov QWORD[((544+24))+rsp],r8 - lea rdi,[96+rsp] - call __ecp_nistz256_sqr_montq - - pcmpeqd xmm5,xmm4 - pshufd xmm4,xmm1,0xb1 - por xmm4,xmm1 - pshufd xmm5,xmm5,0 - pshufd xmm3,xmm4,0x1e - por xmm4,xmm3 - pxor xmm3,xmm3 - pcmpeqd xmm4,xmm3 - pshufd xmm4,xmm4,0 - mov rax,QWORD[((64+0))+rbx] - mov r14,QWORD[((64+8))+rbx] - mov r15,QWORD[((64+16))+rbx] - mov r8,QWORD[((64+24))+rbx] - movq xmm1,rbx - - lea rsi,[((64-0))+rbx] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montq - - mov rax,QWORD[544+rsp] - lea rbx,[544+rsp] - mov r9,QWORD[((0+96))+rsp] - mov r10,QWORD[((8+96))+rsp] - lea rsi,[((0+96))+rsp] - mov r11,QWORD[((16+96))+rsp] - mov r12,QWORD[((24+96))+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[448+rsp] - lea rbx,[448+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((0+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[416+rsp] - lea rbx,[416+rsp] - mov r9,QWORD[((0+224))+rsp] - mov r10,QWORD[((8+224))+rsp] - lea rsi,[((0+224))+rsp] - mov r11,QWORD[((16+224))+rsp] - mov r12,QWORD[((24+224))+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[512+rsp] - lea rbx,[512+rsp] - mov r9,QWORD[((0+256))+rsp] - mov r10,QWORD[((8+256))+rsp] - lea rsi,[((0+256))+rsp] - mov r11,QWORD[((16+256))+rsp] - mov r12,QWORD[((24+256))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montq - - lea rbx,[224+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromq - - or r12,r13 - movdqa xmm2,xmm4 - or r12,r8 - or r12,r9 - por xmm2,xmm5 - movq xmm3,r12 - - mov rax,QWORD[384+rsp] - lea rbx,[384+rsp] - mov r9,QWORD[((0+96))+rsp] - mov r10,QWORD[((8+96))+rsp] - lea rsi,[((0+96))+rsp] - mov r11,QWORD[((16+96))+rsp] - mov r12,QWORD[((24+96))+rsp] - lea rdi,[160+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[480+rsp] - lea rbx,[480+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((0+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_mul_montq - - lea rbx,[160+rsp] - lea rdi,[rsp] - call __ecp_nistz256_sub_fromq - - or r12,r13 - or r12,r8 - or r12,r9 - - movq r8,xmm2 - movq r9,xmm3 - or r12,r8 - DB 0x3e - jnz NEAR $L$add_proceedq - - - - test r9,r9 - jz NEAR $L$add_doubleq - - - - - - - movq rdi,xmm0 - pxor xmm0,xmm0 - movdqu XMMWORD[rdi],xmm0 - movdqu XMMWORD[16+rdi],xmm0 - movdqu XMMWORD[32+rdi],xmm0 - movdqu XMMWORD[48+rdi],xmm0 - movdqu XMMWORD[64+rdi],xmm0 - movdqu XMMWORD[80+rdi],xmm0 - jmp NEAR $L$add_doneq - -ALIGN 32 -$L$add_doubleq: - movq rsi,xmm1 - movq rdi,xmm0 - add rsp,416 - - jmp NEAR $L$point_double_shortcutq - - -ALIGN 32 -$L$add_proceedq: - mov rax,QWORD[((0+64))+rsp] - mov r14,QWORD[((8+64))+rsp] - lea rsi,[((0+64))+rsp] - mov r15,QWORD[((16+64))+rsp] - mov r8,QWORD[((24+64))+rsp] - lea rdi,[96+rsp] - call __ecp_nistz256_sqr_montq - - mov rax,QWORD[448+rsp] - lea rbx,[448+rsp] - mov r9,QWORD[((0+0))+rsp] - mov r10,QWORD[((8+0))+rsp] - lea rsi,[((0+0))+rsp] - mov r11,QWORD[((16+0))+rsp] - mov r12,QWORD[((24+0))+rsp] - lea rdi,[352+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[((0+0))+rsp] - mov r14,QWORD[((8+0))+rsp] - lea rsi,[((0+0))+rsp] - mov r15,QWORD[((16+0))+rsp] - mov r8,QWORD[((24+0))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montq - - mov rax,QWORD[544+rsp] - lea rbx,[544+rsp] - mov r9,QWORD[((0+352))+rsp] - mov r10,QWORD[((8+352))+rsp] - lea rsi,[((0+352))+rsp] - mov r11,QWORD[((16+352))+rsp] - mov r12,QWORD[((24+352))+rsp] - lea rdi,[352+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[rsp] - lea rbx,[rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((0+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[128+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[160+rsp] - lea rbx,[160+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((0+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_mul_montq - - - - - xor r11,r11 - add r12,r12 - lea rsi,[96+rsp] - adc r13,r13 - mov rax,r12 - adc r8,r8 - adc r9,r9 - mov rbp,r13 - adc r11,0 - - sub r12,-1 - mov rcx,r8 - sbb r13,r14 - sbb r8,0 - mov r10,r9 - sbb r9,r15 - sbb r11,0 - - cmovc r12,rax - mov rax,QWORD[rsi] - cmovc r13,rbp - mov rbp,QWORD[8+rsi] - cmovc r8,rcx - mov rcx,QWORD[16+rsi] - cmovc r9,r10 - mov r10,QWORD[24+rsi] - - call __ecp_nistz256_subq - - lea rbx,[128+rsp] - lea rdi,[288+rsp] - call __ecp_nistz256_sub_fromq - - mov rax,QWORD[((192+0))+rsp] - mov rbp,QWORD[((192+8))+rsp] - mov rcx,QWORD[((192+16))+rsp] - mov r10,QWORD[((192+24))+rsp] - lea rdi,[320+rsp] - - call __ecp_nistz256_subq - - mov QWORD[rdi],r12 - mov QWORD[8+rdi],r13 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - mov rax,QWORD[128+rsp] - lea rbx,[128+rsp] - mov r9,QWORD[((0+224))+rsp] - mov r10,QWORD[((8+224))+rsp] - lea rsi,[((0+224))+rsp] - mov r11,QWORD[((16+224))+rsp] - mov r12,QWORD[((24+224))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[320+rsp] - lea rbx,[320+rsp] - mov r9,QWORD[((0+64))+rsp] - mov r10,QWORD[((8+64))+rsp] - lea rsi,[((0+64))+rsp] - mov r11,QWORD[((16+64))+rsp] - mov r12,QWORD[((24+64))+rsp] - lea rdi,[320+rsp] - call __ecp_nistz256_mul_montq - - lea rbx,[256+rsp] - lea rdi,[320+rsp] - call __ecp_nistz256_sub_fromq - - movq rdi,xmm0 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[352+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((352+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[544+rsp] - pand xmm3,XMMWORD[((544+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[448+rsp] - pand xmm3,XMMWORD[((448+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[64+rdi],xmm2 - movdqu XMMWORD[80+rdi],xmm3 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[288+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((288+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[480+rsp] - pand xmm3,XMMWORD[((480+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[384+rsp] - pand xmm3,XMMWORD[((384+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[rdi],xmm2 - movdqu XMMWORD[16+rdi],xmm3 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[320+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((320+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[512+rsp] - pand xmm3,XMMWORD[((512+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[416+rsp] - pand xmm3,XMMWORD[((416+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[32+rdi],xmm2 - movdqu XMMWORD[48+rdi],xmm3 - -$L$add_doneq: - lea rsi,[((576+56))+rsp] - - mov r15,QWORD[((-48))+rsi] - - mov r14,QWORD[((-40))+rsi] - - mov r13,QWORD[((-32))+rsi] - - mov r12,QWORD[((-24))+rsi] - - mov rbx,QWORD[((-16))+rsi] - - mov rbp,QWORD[((-8))+rsi] - - lea rsp,[rsi] - -$L$point_addq_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_point_add_nohw: -global ecp_nistz256_point_add_affine_nohw - -ALIGN 32 -ecp_nistz256_point_add_affine_nohw: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_point_add_affine_nohw: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,32*15+8 - -$L$add_affineq_body: - - movdqu xmm0,XMMWORD[rsi] - mov rbx,rdx - movdqu xmm1,XMMWORD[16+rsi] - movdqu xmm2,XMMWORD[32+rsi] - movdqu xmm3,XMMWORD[48+rsi] - movdqu xmm4,XMMWORD[64+rsi] - movdqu xmm5,XMMWORD[80+rsi] - mov rax,QWORD[((64+0))+rsi] - mov r14,QWORD[((64+8))+rsi] - mov r15,QWORD[((64+16))+rsi] - mov r8,QWORD[((64+24))+rsi] - movdqa XMMWORD[320+rsp],xmm0 - movdqa XMMWORD[(320+16)+rsp],xmm1 - movdqa XMMWORD[352+rsp],xmm2 - movdqa XMMWORD[(352+16)+rsp],xmm3 - movdqa XMMWORD[384+rsp],xmm4 - movdqa XMMWORD[(384+16)+rsp],xmm5 - por xmm5,xmm4 - - movdqu xmm0,XMMWORD[rbx] - pshufd xmm3,xmm5,0xb1 - movdqu xmm1,XMMWORD[16+rbx] - movdqu xmm2,XMMWORD[32+rbx] - por xmm5,xmm3 - movdqu xmm3,XMMWORD[48+rbx] - movdqa XMMWORD[416+rsp],xmm0 - pshufd xmm4,xmm5,0x1e - movdqa XMMWORD[(416+16)+rsp],xmm1 - por xmm1,xmm0 - movq xmm0,rdi - movdqa XMMWORD[448+rsp],xmm2 - movdqa XMMWORD[(448+16)+rsp],xmm3 - por xmm3,xmm2 - por xmm5,xmm4 - pxor xmm4,xmm4 - por xmm3,xmm1 - - lea rsi,[((64-0))+rsi] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montq - - pcmpeqd xmm5,xmm4 - pshufd xmm4,xmm3,0xb1 - mov rax,QWORD[rbx] - - mov r9,r12 - por xmm4,xmm3 - pshufd xmm5,xmm5,0 - pshufd xmm3,xmm4,0x1e - mov r10,r13 - por xmm4,xmm3 - pxor xmm3,xmm3 - mov r11,r14 - pcmpeqd xmm4,xmm3 - pshufd xmm4,xmm4,0 - - lea rsi,[((32-0))+rsp] - mov r12,r15 - lea rdi,[rsp] - call __ecp_nistz256_mul_montq - - lea rbx,[320+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromq - - mov rax,QWORD[384+rsp] - lea rbx,[384+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((0+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[384+rsp] - lea rbx,[384+rsp] - mov r9,QWORD[((0+64))+rsp] - mov r10,QWORD[((8+64))+rsp] - lea rsi,[((0+64))+rsp] - mov r11,QWORD[((16+64))+rsp] - mov r12,QWORD[((24+64))+rsp] - lea rdi,[288+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[448+rsp] - lea rbx,[448+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((0+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montq - - lea rbx,[352+rsp] - lea rdi,[96+rsp] - call __ecp_nistz256_sub_fromq - - mov rax,QWORD[((0+64))+rsp] - mov r14,QWORD[((8+64))+rsp] - lea rsi,[((0+64))+rsp] - mov r15,QWORD[((16+64))+rsp] - mov r8,QWORD[((24+64))+rsp] - lea rdi,[128+rsp] - call __ecp_nistz256_sqr_montq - - mov rax,QWORD[((0+96))+rsp] - mov r14,QWORD[((8+96))+rsp] - lea rsi,[((0+96))+rsp] - mov r15,QWORD[((16+96))+rsp] - mov r8,QWORD[((24+96))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_sqr_montq - - mov rax,QWORD[128+rsp] - lea rbx,[128+rsp] - mov r9,QWORD[((0+64))+rsp] - mov r10,QWORD[((8+64))+rsp] - lea rsi,[((0+64))+rsp] - mov r11,QWORD[((16+64))+rsp] - mov r12,QWORD[((24+64))+rsp] - lea rdi,[160+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[320+rsp] - lea rbx,[320+rsp] - mov r9,QWORD[((0+128))+rsp] - mov r10,QWORD[((8+128))+rsp] - lea rsi,[((0+128))+rsp] - mov r11,QWORD[((16+128))+rsp] - mov r12,QWORD[((24+128))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_mul_montq - - - - - xor r11,r11 - add r12,r12 - lea rsi,[192+rsp] - adc r13,r13 - mov rax,r12 - adc r8,r8 - adc r9,r9 - mov rbp,r13 - adc r11,0 - - sub r12,-1 - mov rcx,r8 - sbb r13,r14 - sbb r8,0 - mov r10,r9 - sbb r9,r15 - sbb r11,0 - - cmovc r12,rax - mov rax,QWORD[rsi] - cmovc r13,rbp - mov rbp,QWORD[8+rsi] - cmovc r8,rcx - mov rcx,QWORD[16+rsi] - cmovc r9,r10 - mov r10,QWORD[24+rsi] - - call __ecp_nistz256_subq - - lea rbx,[160+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_sub_fromq - - mov rax,QWORD[((0+0))+rsp] - mov rbp,QWORD[((0+8))+rsp] - mov rcx,QWORD[((0+16))+rsp] - mov r10,QWORD[((0+24))+rsp] - lea rdi,[64+rsp] - - call __ecp_nistz256_subq - - mov QWORD[rdi],r12 - mov QWORD[8+rdi],r13 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - mov rax,QWORD[352+rsp] - lea rbx,[352+rsp] - mov r9,QWORD[((0+160))+rsp] - mov r10,QWORD[((8+160))+rsp] - lea rsi,[((0+160))+rsp] - mov r11,QWORD[((16+160))+rsp] - mov r12,QWORD[((24+160))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montq - - mov rax,QWORD[96+rsp] - lea rbx,[96+rsp] - mov r9,QWORD[((0+64))+rsp] - mov r10,QWORD[((8+64))+rsp] - lea rsi,[((0+64))+rsp] - mov r11,QWORD[((16+64))+rsp] - mov r12,QWORD[((24+64))+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_mul_montq - - lea rbx,[32+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_sub_fromq - - movq rdi,xmm0 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[288+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((288+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[$L$ONE_mont] - pand xmm3,XMMWORD[(($L$ONE_mont+16))] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[384+rsp] - pand xmm3,XMMWORD[((384+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[64+rdi],xmm2 - movdqu XMMWORD[80+rdi],xmm3 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[224+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((224+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[416+rsp] - pand xmm3,XMMWORD[((416+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[320+rsp] - pand xmm3,XMMWORD[((320+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[rdi],xmm2 - movdqu XMMWORD[16+rdi],xmm3 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[256+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((256+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[448+rsp] - pand xmm3,XMMWORD[((448+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[352+rsp] - pand xmm3,XMMWORD[((352+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[32+rdi],xmm2 - movdqu XMMWORD[48+rdi],xmm3 - - lea rsi,[((480+56))+rsp] - - mov r15,QWORD[((-48))+rsi] - - mov r14,QWORD[((-40))+rsi] - - mov r13,QWORD[((-32))+rsi] - - mov r12,QWORD[((-24))+rsi] - - mov rbx,QWORD[((-16))+rsi] - - mov rbp,QWORD[((-8))+rsi] - - lea rsp,[rsi] - -$L$add_affineq_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_point_add_affine_nohw: - -ALIGN 32 -__ecp_nistz256_add_tox: - - xor r11,r11 - adc r12,QWORD[rbx] - adc r13,QWORD[8+rbx] - mov rax,r12 - adc r8,QWORD[16+rbx] - adc r9,QWORD[24+rbx] - mov rbp,r13 - adc r11,0 - - xor r10,r10 - sbb r12,-1 - mov rcx,r8 - sbb r13,r14 - sbb r8,0 - mov r10,r9 - sbb r9,r15 - sbb r11,0 - - cmovc r12,rax - cmovc r13,rbp - mov QWORD[rdi],r12 - cmovc r8,rcx - mov QWORD[8+rdi],r13 - cmovc r9,r10 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - - ret - - - - -ALIGN 32 -__ecp_nistz256_sub_fromx: - - xor r11,r11 - sbb r12,QWORD[rbx] - sbb r13,QWORD[8+rbx] - mov rax,r12 - sbb r8,QWORD[16+rbx] - sbb r9,QWORD[24+rbx] - mov rbp,r13 - sbb r11,0 - - xor r10,r10 - adc r12,-1 - mov rcx,r8 - adc r13,r14 - adc r8,0 - mov r10,r9 - adc r9,r15 - - bt r11,0 - cmovnc r12,rax - cmovnc r13,rbp - mov QWORD[rdi],r12 - cmovnc r8,rcx - mov QWORD[8+rdi],r13 - cmovnc r9,r10 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - - ret - - - - -ALIGN 32 -__ecp_nistz256_subx: - - xor r11,r11 - sbb rax,r12 - sbb rbp,r13 - mov r12,rax - sbb rcx,r8 - sbb r10,r9 - mov r13,rbp - sbb r11,0 - - xor r9,r9 - adc rax,-1 - mov r8,rcx - adc rbp,r14 - adc rcx,0 - mov r9,r10 - adc r10,r15 - - bt r11,0 - cmovc r12,rax - cmovc r13,rbp - cmovc r8,rcx - cmovc r9,r10 - - ret - - - - -ALIGN 32 -__ecp_nistz256_mul_by_2x: - - xor r11,r11 - adc r12,r12 - adc r13,r13 - mov rax,r12 - adc r8,r8 - adc r9,r9 - mov rbp,r13 - adc r11,0 - - xor r10,r10 - sbb r12,-1 - mov rcx,r8 - sbb r13,r14 - sbb r8,0 - mov r10,r9 - sbb r9,r15 - sbb r11,0 - - cmovc r12,rax - cmovc r13,rbp - mov QWORD[rdi],r12 - cmovc r8,rcx - mov QWORD[8+rdi],r13 - cmovc r9,r10 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - - ret - - -global ecp_nistz256_point_double_adx - -ALIGN 32 -ecp_nistz256_point_double_adx: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_point_double_adx: - mov rdi,rcx - mov rsi,rdx - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,32*5+8 - -$L$point_doublex_body: - -$L$point_double_shortcutx: - movdqu xmm0,XMMWORD[rsi] - mov rbx,rsi - movdqu xmm1,XMMWORD[16+rsi] - mov r12,QWORD[((32+0))+rsi] - mov r13,QWORD[((32+8))+rsi] - mov r8,QWORD[((32+16))+rsi] - mov r9,QWORD[((32+24))+rsi] - mov r14,QWORD[(($L$poly+8))] - mov r15,QWORD[(($L$poly+24))] - movdqa XMMWORD[96+rsp],xmm0 - movdqa XMMWORD[(96+16)+rsp],xmm1 - lea r10,[32+rdi] - lea r11,[64+rdi] - movq xmm0,rdi - movq xmm1,r10 - movq xmm2,r11 - - lea rdi,[rsp] - call __ecp_nistz256_mul_by_2x - - mov rdx,QWORD[((64+0))+rsi] - mov r14,QWORD[((64+8))+rsi] - mov r15,QWORD[((64+16))+rsi] - mov r8,QWORD[((64+24))+rsi] - lea rsi,[((64-128))+rsi] - lea rdi,[64+rsp] - call __ecp_nistz256_sqr_montx - - mov rdx,QWORD[((0+0))+rsp] - mov r14,QWORD[((8+0))+rsp] - lea rsi,[((-128+0))+rsp] - mov r15,QWORD[((16+0))+rsp] - mov r8,QWORD[((24+0))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_sqr_montx - - mov rdx,QWORD[32+rbx] - mov r9,QWORD[((64+0))+rbx] - mov r10,QWORD[((64+8))+rbx] - mov r11,QWORD[((64+16))+rbx] - mov r12,QWORD[((64+24))+rbx] - lea rsi,[((64-128))+rbx] - lea rbx,[32+rbx] - movq rdi,xmm2 - call __ecp_nistz256_mul_montx - call __ecp_nistz256_mul_by_2x - - mov r12,QWORD[((96+0))+rsp] - mov r13,QWORD[((96+8))+rsp] - lea rbx,[64+rsp] - mov r8,QWORD[((96+16))+rsp] - mov r9,QWORD[((96+24))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_add_tox - - mov r12,QWORD[((96+0))+rsp] - mov r13,QWORD[((96+8))+rsp] - lea rbx,[64+rsp] - mov r8,QWORD[((96+16))+rsp] - mov r9,QWORD[((96+24))+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromx - - mov rdx,QWORD[((0+0))+rsp] - mov r14,QWORD[((8+0))+rsp] - lea rsi,[((-128+0))+rsp] - mov r15,QWORD[((16+0))+rsp] - mov r8,QWORD[((24+0))+rsp] - movq rdi,xmm1 - call __ecp_nistz256_sqr_montx - xor r9,r9 - mov rax,r12 - add r12,-1 - mov r10,r13 - adc r13,rsi - mov rcx,r14 - adc r14,0 - mov r8,r15 - adc r15,rbp - adc r9,0 - xor rsi,rsi - test rax,1 - - cmovz r12,rax - cmovz r13,r10 - cmovz r14,rcx - cmovz r15,r8 - cmovz r9,rsi - - mov rax,r13 - shr r12,1 - shl rax,63 - mov r10,r14 - shr r13,1 - or r12,rax - shl r10,63 - mov rcx,r15 - shr r14,1 - or r13,r10 - shl rcx,63 - mov QWORD[rdi],r12 - shr r15,1 - mov QWORD[8+rdi],r13 - shl r9,63 - or r14,rcx - or r15,r9 - mov QWORD[16+rdi],r14 - mov QWORD[24+rdi],r15 - mov rdx,QWORD[64+rsp] - lea rbx,[64+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((-128+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montx - - lea rdi,[128+rsp] - call __ecp_nistz256_mul_by_2x - - lea rbx,[32+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_add_tox - - mov rdx,QWORD[96+rsp] - lea rbx,[96+rsp] - mov r9,QWORD[((0+0))+rsp] - mov r10,QWORD[((8+0))+rsp] - lea rsi,[((-128+0))+rsp] - mov r11,QWORD[((16+0))+rsp] - mov r12,QWORD[((24+0))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_mul_montx - - lea rdi,[128+rsp] - call __ecp_nistz256_mul_by_2x - - mov rdx,QWORD[((0+32))+rsp] - mov r14,QWORD[((8+32))+rsp] - lea rsi,[((-128+32))+rsp] - mov r15,QWORD[((16+32))+rsp] - mov r8,QWORD[((24+32))+rsp] - movq rdi,xmm0 - call __ecp_nistz256_sqr_montx - - lea rbx,[128+rsp] - mov r8,r14 - mov r9,r15 - mov r14,rsi - mov r15,rbp - call __ecp_nistz256_sub_fromx - - mov rax,QWORD[((0+0))+rsp] - mov rbp,QWORD[((0+8))+rsp] - mov rcx,QWORD[((0+16))+rsp] - mov r10,QWORD[((0+24))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_subx - - mov rdx,QWORD[32+rsp] - lea rbx,[32+rsp] - mov r14,r12 - xor ecx,ecx - mov QWORD[((0+0))+rsp],r12 - mov r10,r13 - mov QWORD[((0+8))+rsp],r13 - cmovz r11,r8 - mov QWORD[((0+16))+rsp],r8 - lea rsi,[((0-128))+rsp] - cmovz r12,r9 - mov QWORD[((0+24))+rsp],r9 - mov r9,r14 - lea rdi,[rsp] - call __ecp_nistz256_mul_montx - - movq rbx,xmm1 - movq rdi,xmm1 - call __ecp_nistz256_sub_fromx - - lea rsi,[((160+56))+rsp] - - mov r15,QWORD[((-48))+rsi] - - mov r14,QWORD[((-40))+rsi] - - mov r13,QWORD[((-32))+rsi] - - mov r12,QWORD[((-24))+rsi] - - mov rbx,QWORD[((-16))+rsi] - - mov rbp,QWORD[((-8))+rsi] - - lea rsp,[rsi] - -$L$point_doublex_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_point_double_adx: -global ecp_nistz256_point_add_adx - -ALIGN 32 -ecp_nistz256_point_add_adx: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_point_add_adx: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,32*18+8 - -$L$point_addx_body: - - movdqu xmm0,XMMWORD[rsi] - movdqu xmm1,XMMWORD[16+rsi] - movdqu xmm2,XMMWORD[32+rsi] - movdqu xmm3,XMMWORD[48+rsi] - movdqu xmm4,XMMWORD[64+rsi] - movdqu xmm5,XMMWORD[80+rsi] - mov rbx,rsi - mov rsi,rdx - movdqa XMMWORD[384+rsp],xmm0 - movdqa XMMWORD[(384+16)+rsp],xmm1 - movdqa XMMWORD[416+rsp],xmm2 - movdqa XMMWORD[(416+16)+rsp],xmm3 - movdqa XMMWORD[448+rsp],xmm4 - movdqa XMMWORD[(448+16)+rsp],xmm5 - por xmm5,xmm4 - - movdqu xmm0,XMMWORD[rsi] - pshufd xmm3,xmm5,0xb1 - movdqu xmm1,XMMWORD[16+rsi] - movdqu xmm2,XMMWORD[32+rsi] - por xmm5,xmm3 - movdqu xmm3,XMMWORD[48+rsi] - mov rdx,QWORD[((64+0))+rsi] - mov r14,QWORD[((64+8))+rsi] - mov r15,QWORD[((64+16))+rsi] - mov r8,QWORD[((64+24))+rsi] - movdqa XMMWORD[480+rsp],xmm0 - pshufd xmm4,xmm5,0x1e - movdqa XMMWORD[(480+16)+rsp],xmm1 - movdqu xmm0,XMMWORD[64+rsi] - movdqu xmm1,XMMWORD[80+rsi] - movdqa XMMWORD[512+rsp],xmm2 - movdqa XMMWORD[(512+16)+rsp],xmm3 - por xmm5,xmm4 - pxor xmm4,xmm4 - por xmm1,xmm0 - movq xmm0,rdi - - lea rsi,[((64-128))+rsi] - mov QWORD[((544+0))+rsp],rdx - mov QWORD[((544+8))+rsp],r14 - mov QWORD[((544+16))+rsp],r15 - mov QWORD[((544+24))+rsp],r8 - lea rdi,[96+rsp] - call __ecp_nistz256_sqr_montx - - pcmpeqd xmm5,xmm4 - pshufd xmm4,xmm1,0xb1 - por xmm4,xmm1 - pshufd xmm5,xmm5,0 - pshufd xmm3,xmm4,0x1e - por xmm4,xmm3 - pxor xmm3,xmm3 - pcmpeqd xmm4,xmm3 - pshufd xmm4,xmm4,0 - mov rdx,QWORD[((64+0))+rbx] - mov r14,QWORD[((64+8))+rbx] - mov r15,QWORD[((64+16))+rbx] - mov r8,QWORD[((64+24))+rbx] - movq xmm1,rbx - - lea rsi,[((64-128))+rbx] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montx - - mov rdx,QWORD[544+rsp] - lea rbx,[544+rsp] - mov r9,QWORD[((0+96))+rsp] - mov r10,QWORD[((8+96))+rsp] - lea rsi,[((-128+96))+rsp] - mov r11,QWORD[((16+96))+rsp] - mov r12,QWORD[((24+96))+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[448+rsp] - lea rbx,[448+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((-128+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[416+rsp] - lea rbx,[416+rsp] - mov r9,QWORD[((0+224))+rsp] - mov r10,QWORD[((8+224))+rsp] - lea rsi,[((-128+224))+rsp] - mov r11,QWORD[((16+224))+rsp] - mov r12,QWORD[((24+224))+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[512+rsp] - lea rbx,[512+rsp] - mov r9,QWORD[((0+256))+rsp] - mov r10,QWORD[((8+256))+rsp] - lea rsi,[((-128+256))+rsp] - mov r11,QWORD[((16+256))+rsp] - mov r12,QWORD[((24+256))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montx - - lea rbx,[224+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromx - - or r12,r13 - movdqa xmm2,xmm4 - or r12,r8 - or r12,r9 - por xmm2,xmm5 - movq xmm3,r12 - - mov rdx,QWORD[384+rsp] - lea rbx,[384+rsp] - mov r9,QWORD[((0+96))+rsp] - mov r10,QWORD[((8+96))+rsp] - lea rsi,[((-128+96))+rsp] - mov r11,QWORD[((16+96))+rsp] - mov r12,QWORD[((24+96))+rsp] - lea rdi,[160+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[480+rsp] - lea rbx,[480+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((-128+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_mul_montx - - lea rbx,[160+rsp] - lea rdi,[rsp] - call __ecp_nistz256_sub_fromx - - or r12,r13 - or r12,r8 - or r12,r9 - - movq r8,xmm2 - movq r9,xmm3 - or r12,r8 - DB 0x3e - jnz NEAR $L$add_proceedx - - - - test r9,r9 - jz NEAR $L$add_doublex - - - - - - - movq rdi,xmm0 - pxor xmm0,xmm0 - movdqu XMMWORD[rdi],xmm0 - movdqu XMMWORD[16+rdi],xmm0 - movdqu XMMWORD[32+rdi],xmm0 - movdqu XMMWORD[48+rdi],xmm0 - movdqu XMMWORD[64+rdi],xmm0 - movdqu XMMWORD[80+rdi],xmm0 - jmp NEAR $L$add_donex - -ALIGN 32 -$L$add_doublex: - movq rsi,xmm1 - movq rdi,xmm0 - add rsp,416 - - jmp NEAR $L$point_double_shortcutx - - -ALIGN 32 -$L$add_proceedx: - mov rdx,QWORD[((0+64))+rsp] - mov r14,QWORD[((8+64))+rsp] - lea rsi,[((-128+64))+rsp] - mov r15,QWORD[((16+64))+rsp] - mov r8,QWORD[((24+64))+rsp] - lea rdi,[96+rsp] - call __ecp_nistz256_sqr_montx - - mov rdx,QWORD[448+rsp] - lea rbx,[448+rsp] - mov r9,QWORD[((0+0))+rsp] - mov r10,QWORD[((8+0))+rsp] - lea rsi,[((-128+0))+rsp] - mov r11,QWORD[((16+0))+rsp] - mov r12,QWORD[((24+0))+rsp] - lea rdi,[352+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[((0+0))+rsp] - mov r14,QWORD[((8+0))+rsp] - lea rsi,[((-128+0))+rsp] - mov r15,QWORD[((16+0))+rsp] - mov r8,QWORD[((24+0))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montx - - mov rdx,QWORD[544+rsp] - lea rbx,[544+rsp] - mov r9,QWORD[((0+352))+rsp] - mov r10,QWORD[((8+352))+rsp] - lea rsi,[((-128+352))+rsp] - mov r11,QWORD[((16+352))+rsp] - mov r12,QWORD[((24+352))+rsp] - lea rdi,[352+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[rsp] - lea rbx,[rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((-128+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[128+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[160+rsp] - lea rbx,[160+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((-128+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_mul_montx - - - - - xor r11,r11 - add r12,r12 - lea rsi,[96+rsp] - adc r13,r13 - mov rax,r12 - adc r8,r8 - adc r9,r9 - mov rbp,r13 - adc r11,0 - - sub r12,-1 - mov rcx,r8 - sbb r13,r14 - sbb r8,0 - mov r10,r9 - sbb r9,r15 - sbb r11,0 - - cmovc r12,rax - mov rax,QWORD[rsi] - cmovc r13,rbp - mov rbp,QWORD[8+rsi] - cmovc r8,rcx - mov rcx,QWORD[16+rsi] - cmovc r9,r10 - mov r10,QWORD[24+rsi] - - call __ecp_nistz256_subx - - lea rbx,[128+rsp] - lea rdi,[288+rsp] - call __ecp_nistz256_sub_fromx - - mov rax,QWORD[((192+0))+rsp] - mov rbp,QWORD[((192+8))+rsp] - mov rcx,QWORD[((192+16))+rsp] - mov r10,QWORD[((192+24))+rsp] - lea rdi,[320+rsp] - - call __ecp_nistz256_subx - - mov QWORD[rdi],r12 - mov QWORD[8+rdi],r13 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - mov rdx,QWORD[128+rsp] - lea rbx,[128+rsp] - mov r9,QWORD[((0+224))+rsp] - mov r10,QWORD[((8+224))+rsp] - lea rsi,[((-128+224))+rsp] - mov r11,QWORD[((16+224))+rsp] - mov r12,QWORD[((24+224))+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[320+rsp] - lea rbx,[320+rsp] - mov r9,QWORD[((0+64))+rsp] - mov r10,QWORD[((8+64))+rsp] - lea rsi,[((-128+64))+rsp] - mov r11,QWORD[((16+64))+rsp] - mov r12,QWORD[((24+64))+rsp] - lea rdi,[320+rsp] - call __ecp_nistz256_mul_montx - - lea rbx,[256+rsp] - lea rdi,[320+rsp] - call __ecp_nistz256_sub_fromx - - movq rdi,xmm0 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[352+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((352+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[544+rsp] - pand xmm3,XMMWORD[((544+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[448+rsp] - pand xmm3,XMMWORD[((448+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[64+rdi],xmm2 - movdqu XMMWORD[80+rdi],xmm3 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[288+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((288+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[480+rsp] - pand xmm3,XMMWORD[((480+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[384+rsp] - pand xmm3,XMMWORD[((384+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[rdi],xmm2 - movdqu XMMWORD[16+rdi],xmm3 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[320+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((320+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[512+rsp] - pand xmm3,XMMWORD[((512+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[416+rsp] - pand xmm3,XMMWORD[((416+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[32+rdi],xmm2 - movdqu XMMWORD[48+rdi],xmm3 - -$L$add_donex: - lea rsi,[((576+56))+rsp] - - mov r15,QWORD[((-48))+rsi] - - mov r14,QWORD[((-40))+rsi] - - mov r13,QWORD[((-32))+rsi] - - mov r12,QWORD[((-24))+rsi] - - mov rbx,QWORD[((-16))+rsi] - - mov rbp,QWORD[((-8))+rsi] - - lea rsp,[rsi] - -$L$point_addx_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_point_add_adx: -global ecp_nistz256_point_add_affine_adx - -ALIGN 32 -ecp_nistz256_point_add_affine_adx: - mov QWORD[8+rsp],rdi ;WIN64 prologue - mov QWORD[16+rsp],rsi - mov rax,rsp -$L$SEH_begin_ecp_nistz256_point_add_affine_adx: - mov rdi,rcx - mov rsi,rdx - mov rdx,r8 - - - -_CET_ENDBR - push rbp - - push rbx - - push r12 - - push r13 - - push r14 - - push r15 - - sub rsp,32*15+8 - -$L$add_affinex_body: - - movdqu xmm0,XMMWORD[rsi] - mov rbx,rdx - movdqu xmm1,XMMWORD[16+rsi] - movdqu xmm2,XMMWORD[32+rsi] - movdqu xmm3,XMMWORD[48+rsi] - movdqu xmm4,XMMWORD[64+rsi] - movdqu xmm5,XMMWORD[80+rsi] - mov rdx,QWORD[((64+0))+rsi] - mov r14,QWORD[((64+8))+rsi] - mov r15,QWORD[((64+16))+rsi] - mov r8,QWORD[((64+24))+rsi] - movdqa XMMWORD[320+rsp],xmm0 - movdqa XMMWORD[(320+16)+rsp],xmm1 - movdqa XMMWORD[352+rsp],xmm2 - movdqa XMMWORD[(352+16)+rsp],xmm3 - movdqa XMMWORD[384+rsp],xmm4 - movdqa XMMWORD[(384+16)+rsp],xmm5 - por xmm5,xmm4 - - movdqu xmm0,XMMWORD[rbx] - pshufd xmm3,xmm5,0xb1 - movdqu xmm1,XMMWORD[16+rbx] - movdqu xmm2,XMMWORD[32+rbx] - por xmm5,xmm3 - movdqu xmm3,XMMWORD[48+rbx] - movdqa XMMWORD[416+rsp],xmm0 - pshufd xmm4,xmm5,0x1e - movdqa XMMWORD[(416+16)+rsp],xmm1 - por xmm1,xmm0 - movq xmm0,rdi - movdqa XMMWORD[448+rsp],xmm2 - movdqa XMMWORD[(448+16)+rsp],xmm3 - por xmm3,xmm2 - por xmm5,xmm4 - pxor xmm4,xmm4 - por xmm3,xmm1 - - lea rsi,[((64-128))+rsi] - lea rdi,[32+rsp] - call __ecp_nistz256_sqr_montx - - pcmpeqd xmm5,xmm4 - pshufd xmm4,xmm3,0xb1 - mov rdx,QWORD[rbx] - - mov r9,r12 - por xmm4,xmm3 - pshufd xmm5,xmm5,0 - pshufd xmm3,xmm4,0x1e - mov r10,r13 - por xmm4,xmm3 - pxor xmm3,xmm3 - mov r11,r14 - pcmpeqd xmm4,xmm3 - pshufd xmm4,xmm4,0 - - lea rsi,[((32-128))+rsp] - mov r12,r15 - lea rdi,[rsp] - call __ecp_nistz256_mul_montx - - lea rbx,[320+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_sub_fromx - - mov rdx,QWORD[384+rsp] - lea rbx,[384+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((-128+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[384+rsp] - lea rbx,[384+rsp] - mov r9,QWORD[((0+64))+rsp] - mov r10,QWORD[((8+64))+rsp] - lea rsi,[((-128+64))+rsp] - mov r11,QWORD[((16+64))+rsp] - mov r12,QWORD[((24+64))+rsp] - lea rdi,[288+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[448+rsp] - lea rbx,[448+rsp] - mov r9,QWORD[((0+32))+rsp] - mov r10,QWORD[((8+32))+rsp] - lea rsi,[((-128+32))+rsp] - mov r11,QWORD[((16+32))+rsp] - mov r12,QWORD[((24+32))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montx - - lea rbx,[352+rsp] - lea rdi,[96+rsp] - call __ecp_nistz256_sub_fromx - - mov rdx,QWORD[((0+64))+rsp] - mov r14,QWORD[((8+64))+rsp] - lea rsi,[((-128+64))+rsp] - mov r15,QWORD[((16+64))+rsp] - mov r8,QWORD[((24+64))+rsp] - lea rdi,[128+rsp] - call __ecp_nistz256_sqr_montx - - mov rdx,QWORD[((0+96))+rsp] - mov r14,QWORD[((8+96))+rsp] - lea rsi,[((-128+96))+rsp] - mov r15,QWORD[((16+96))+rsp] - mov r8,QWORD[((24+96))+rsp] - lea rdi,[192+rsp] - call __ecp_nistz256_sqr_montx - - mov rdx,QWORD[128+rsp] - lea rbx,[128+rsp] - mov r9,QWORD[((0+64))+rsp] - mov r10,QWORD[((8+64))+rsp] - lea rsi,[((-128+64))+rsp] - mov r11,QWORD[((16+64))+rsp] - mov r12,QWORD[((24+64))+rsp] - lea rdi,[160+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[320+rsp] - lea rbx,[320+rsp] - mov r9,QWORD[((0+128))+rsp] - mov r10,QWORD[((8+128))+rsp] - lea rsi,[((-128+128))+rsp] - mov r11,QWORD[((16+128))+rsp] - mov r12,QWORD[((24+128))+rsp] - lea rdi,[rsp] - call __ecp_nistz256_mul_montx - - - - - xor r11,r11 - add r12,r12 - lea rsi,[192+rsp] - adc r13,r13 - mov rax,r12 - adc r8,r8 - adc r9,r9 - mov rbp,r13 - adc r11,0 - - sub r12,-1 - mov rcx,r8 - sbb r13,r14 - sbb r8,0 - mov r10,r9 - sbb r9,r15 - sbb r11,0 - - cmovc r12,rax - mov rax,QWORD[rsi] - cmovc r13,rbp - mov rbp,QWORD[8+rsi] - cmovc r8,rcx - mov rcx,QWORD[16+rsi] - cmovc r9,r10 - mov r10,QWORD[24+rsi] - - call __ecp_nistz256_subx - - lea rbx,[160+rsp] - lea rdi,[224+rsp] - call __ecp_nistz256_sub_fromx - - mov rax,QWORD[((0+0))+rsp] - mov rbp,QWORD[((0+8))+rsp] - mov rcx,QWORD[((0+16))+rsp] - mov r10,QWORD[((0+24))+rsp] - lea rdi,[64+rsp] - - call __ecp_nistz256_subx - - mov QWORD[rdi],r12 - mov QWORD[8+rdi],r13 - mov QWORD[16+rdi],r8 - mov QWORD[24+rdi],r9 - mov rdx,QWORD[352+rsp] - lea rbx,[352+rsp] - mov r9,QWORD[((0+160))+rsp] - mov r10,QWORD[((8+160))+rsp] - lea rsi,[((-128+160))+rsp] - mov r11,QWORD[((16+160))+rsp] - mov r12,QWORD[((24+160))+rsp] - lea rdi,[32+rsp] - call __ecp_nistz256_mul_montx - - mov rdx,QWORD[96+rsp] - lea rbx,[96+rsp] - mov r9,QWORD[((0+64))+rsp] - mov r10,QWORD[((8+64))+rsp] - lea rsi,[((-128+64))+rsp] - mov r11,QWORD[((16+64))+rsp] - mov r12,QWORD[((24+64))+rsp] - lea rdi,[64+rsp] - call __ecp_nistz256_mul_montx - - lea rbx,[32+rsp] - lea rdi,[256+rsp] - call __ecp_nistz256_sub_fromx - - movq rdi,xmm0 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[288+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((288+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[$L$ONE_mont] - pand xmm3,XMMWORD[(($L$ONE_mont+16))] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[384+rsp] - pand xmm3,XMMWORD[((384+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[64+rdi],xmm2 - movdqu XMMWORD[80+rdi],xmm3 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[224+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((224+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[416+rsp] - pand xmm3,XMMWORD[((416+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[320+rsp] - pand xmm3,XMMWORD[((320+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[rdi],xmm2 - movdqu XMMWORD[16+rdi],xmm3 - - movdqa xmm0,xmm5 - movdqa xmm1,xmm5 - pandn xmm0,XMMWORD[256+rsp] - movdqa xmm2,xmm5 - pandn xmm1,XMMWORD[((256+16))+rsp] - movdqa xmm3,xmm5 - pand xmm2,XMMWORD[448+rsp] - pand xmm3,XMMWORD[((448+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - - movdqa xmm0,xmm4 - movdqa xmm1,xmm4 - pandn xmm0,xmm2 - movdqa xmm2,xmm4 - pandn xmm1,xmm3 - movdqa xmm3,xmm4 - pand xmm2,XMMWORD[352+rsp] - pand xmm3,XMMWORD[((352+16))+rsp] - por xmm2,xmm0 - por xmm3,xmm1 - movdqu XMMWORD[32+rdi],xmm2 - movdqu XMMWORD[48+rdi],xmm3 - - lea rsi,[((480+56))+rsp] - - mov r15,QWORD[((-48))+rsi] - - mov r14,QWORD[((-40))+rsi] - - mov r13,QWORD[((-32))+rsi] - - mov r12,QWORD[((-24))+rsi] - - mov rbx,QWORD[((-16))+rsi] - - mov rbp,QWORD[((-8))+rsi] - - lea rsp,[rsi] - -$L$add_affinex_epilogue: - mov rdi,QWORD[8+rsp] ;WIN64 epilogue - mov rsi,QWORD[16+rsp] - ret - -$L$SEH_end_ecp_nistz256_point_add_affine_adx: EXTERN __imp_RtlVirtualUnwind @@ -4892,10 +1262,6 @@ section .pdata rdata align=4 ALIGN 4 - DD $L$SEH_begin_ecp_nistz256_neg wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_neg wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_neg wrt ..imagebase - DD $L$SEH_begin_ecp_nistz256_ord_mul_mont_nohw wrt ..imagebase DD $L$SEH_end_ecp_nistz256_ord_mul_mont_nohw wrt ..imagebase DD $L$SEH_info_ecp_nistz256_ord_mul_mont_nohw wrt ..imagebase @@ -4910,63 +1276,9 @@ DD $L$SEH_begin_ecp_nistz256_ord_sqr_mont_adx wrt ..imagebase DD $L$SEH_end_ecp_nistz256_ord_sqr_mont_adx wrt ..imagebase DD $L$SEH_info_ecp_nistz256_ord_sqr_mont_adx wrt ..imagebase - DD $L$SEH_begin_ecp_nistz256_mul_mont_nohw wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_mul_mont_nohw wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_mul_mont_nohw wrt ..imagebase - - DD $L$SEH_begin_ecp_nistz256_sqr_mont_nohw wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_sqr_mont_nohw wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_sqr_mont_nohw wrt ..imagebase - DD $L$SEH_begin_ecp_nistz256_mul_mont_adx wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_mul_mont_adx wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_mul_mont_adx wrt ..imagebase - - DD $L$SEH_begin_ecp_nistz256_sqr_mont_adx wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_sqr_mont_adx wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_sqr_mont_adx wrt ..imagebase - DD $L$SEH_begin_ecp_nistz256_select_w5_nohw wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_select_w5_nohw wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_select_wX_nohw wrt ..imagebase - - DD $L$SEH_begin_ecp_nistz256_select_w7_nohw wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_select_w7_nohw wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_select_wX_nohw wrt ..imagebase - DD $L$SEH_begin_ecp_nistz256_select_w5_avx2 wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_select_w5_avx2 wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_select_wX_avx2 wrt ..imagebase - - DD $L$SEH_begin_ecp_nistz256_select_w7_avx2 wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_select_w7_avx2 wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_select_wX_avx2 wrt ..imagebase - DD $L$SEH_begin_ecp_nistz256_point_double_nohw wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_point_double_nohw wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_point_double_nohw wrt ..imagebase - - DD $L$SEH_begin_ecp_nistz256_point_add_nohw wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_point_add_nohw wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_point_add_nohw wrt ..imagebase - - DD $L$SEH_begin_ecp_nistz256_point_add_affine_nohw wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_point_add_affine_nohw wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_point_add_affine_nohw wrt ..imagebase - DD $L$SEH_begin_ecp_nistz256_point_double_adx wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_point_double_adx wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_point_double_adx wrt ..imagebase - - DD $L$SEH_begin_ecp_nistz256_point_add_adx wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_point_add_adx wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_point_add_adx wrt ..imagebase - - DD $L$SEH_begin_ecp_nistz256_point_add_affine_adx wrt ..imagebase - DD $L$SEH_end_ecp_nistz256_point_add_affine_adx wrt ..imagebase - DD $L$SEH_info_ecp_nistz256_point_add_affine_adx wrt ..imagebase section .xdata rdata align=8 ALIGN 8 -$L$SEH_info_ecp_nistz256_neg: - DB 9,0,0,0 - DD short_handler wrt ..imagebase - DD $L$neg_body wrt ..imagebase,$L$neg_epilogue wrt ..imagebase $L$SEH_info_ecp_nistz256_ord_mul_mont_nohw: DB 9,0,0,0 DD full_handler wrt ..imagebase @@ -4987,86 +1299,6 @@ DD full_handler wrt ..imagebase DD $L$ord_sqrx_body wrt ..imagebase,$L$ord_sqrx_epilogue wrt ..imagebase DD 48,0 -$L$SEH_info_ecp_nistz256_mul_mont_nohw: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$mul_body wrt ..imagebase,$L$mul_epilogue wrt ..imagebase - DD 48,0 -$L$SEH_info_ecp_nistz256_sqr_mont_nohw: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$sqr_body wrt ..imagebase,$L$sqr_epilogue wrt ..imagebase - DD 48,0 -$L$SEH_info_ecp_nistz256_mul_mont_adx: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$mulx_body wrt ..imagebase,$L$mulx_epilogue wrt ..imagebase - DD 48,0 -$L$SEH_info_ecp_nistz256_sqr_mont_adx: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$sqrx_body wrt ..imagebase,$L$sqrx_epilogue wrt ..imagebase - DD 48,0 -$L$SEH_info_ecp_nistz256_select_wX_nohw: - DB 0x01,0x33,0x16,0x00 - DB 0x33,0xf8,0x09,0x00 - DB 0x2e,0xe8,0x08,0x00 - DB 0x29,0xd8,0x07,0x00 - DB 0x24,0xc8,0x06,0x00 - DB 0x1f,0xb8,0x05,0x00 - DB 0x1a,0xa8,0x04,0x00 - DB 0x15,0x98,0x03,0x00 - DB 0x10,0x88,0x02,0x00 - DB 0x0c,0x78,0x01,0x00 - DB 0x08,0x68,0x00,0x00 - DB 0x04,0x01,0x15,0x00 -ALIGN 8 -$L$SEH_info_ecp_nistz256_select_wX_avx2: - DB 0x01,0x36,0x17,0x0b - DB 0x36,0xf8,0x09,0x00 - DB 0x31,0xe8,0x08,0x00 - DB 0x2c,0xd8,0x07,0x00 - DB 0x27,0xc8,0x06,0x00 - DB 0x22,0xb8,0x05,0x00 - DB 0x1d,0xa8,0x04,0x00 - DB 0x18,0x98,0x03,0x00 - DB 0x13,0x88,0x02,0x00 - DB 0x0e,0x78,0x01,0x00 - DB 0x09,0x68,0x00,0x00 - DB 0x04,0x01,0x15,0x00 - DB 0x00,0xb3,0x00,0x00 -ALIGN 8 -$L$SEH_info_ecp_nistz256_point_double_nohw: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$point_doubleq_body wrt ..imagebase,$L$point_doubleq_epilogue wrt ..imagebase - DD 32*5+56,0 -$L$SEH_info_ecp_nistz256_point_add_nohw: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$point_addq_body wrt ..imagebase,$L$point_addq_epilogue wrt ..imagebase - DD 32*18+56,0 -$L$SEH_info_ecp_nistz256_point_add_affine_nohw: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$add_affineq_body wrt ..imagebase,$L$add_affineq_epilogue wrt ..imagebase - DD 32*15+56,0 -ALIGN 8 -$L$SEH_info_ecp_nistz256_point_double_adx: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$point_doublex_body wrt ..imagebase,$L$point_doublex_epilogue wrt ..imagebase - DD 32*5+56,0 -$L$SEH_info_ecp_nistz256_point_add_adx: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$point_addx_body wrt ..imagebase,$L$point_addx_epilogue wrt ..imagebase - DD 32*18+56,0 -$L$SEH_info_ecp_nistz256_point_add_affine_adx: - DB 9,0,0,0 - DD full_handler wrt ..imagebase - DD $L$add_affinex_body wrt ..imagebase,$L$add_affinex_epilogue wrt ..imagebase - DD 32*15+56,0 %else ; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738 ret
diff --git a/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc b/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc index e29d4be..7ab85df 100644 --- a/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc +++ b/gen/boringssl_prefix_symbols_internal_x86_64_win_asm.inc
@@ -227,38 +227,14 @@ %define chacha20_poly1305_seal BORINGSSL_PREFIX %+ _chacha20_poly1305_seal %define chacha20_poly1305_seal_avx2 BORINGSSL_PREFIX %+ _chacha20_poly1305_seal_avx2 %define chacha20_poly1305_seal_sse41 BORINGSSL_PREFIX %+ _chacha20_poly1305_seal_sse41 -%define ecp_nistz256_div_by_2 BORINGSSL_PREFIX %+ _ecp_nistz256_div_by_2 -%define ecp_nistz256_mul_by_2 BORINGSSL_PREFIX %+ _ecp_nistz256_mul_by_2 -%define ecp_nistz256_mul_by_3 BORINGSSL_PREFIX %+ _ecp_nistz256_mul_by_3 %define ecp_nistz256_mul_mont BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont -%define ecp_nistz256_mul_mont_adx BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont_adx -%define ecp_nistz256_mul_mont_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont_nohw -%define ecp_nistz256_neg BORINGSSL_PREFIX %+ _ecp_nistz256_neg %define ecp_nistz256_ord_mul_mont BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont %define ecp_nistz256_ord_mul_mont_adx BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont_adx %define ecp_nistz256_ord_mul_mont_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont_nohw %define ecp_nistz256_ord_sqr_mont BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont %define ecp_nistz256_ord_sqr_mont_adx BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont_adx %define ecp_nistz256_ord_sqr_mont_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont_nohw -%define ecp_nistz256_point_add BORINGSSL_PREFIX %+ _ecp_nistz256_point_add -%define ecp_nistz256_point_add_adx BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_adx -%define ecp_nistz256_point_add_affine BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine -%define ecp_nistz256_point_add_affine_adx BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine_adx -%define ecp_nistz256_point_add_affine_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine_nohw -%define ecp_nistz256_point_add_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_nohw -%define ecp_nistz256_point_double BORINGSSL_PREFIX %+ _ecp_nistz256_point_double -%define ecp_nistz256_point_double_adx BORINGSSL_PREFIX %+ _ecp_nistz256_point_double_adx -%define ecp_nistz256_point_double_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_point_double_nohw -%define ecp_nistz256_select_w5 BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5 -%define ecp_nistz256_select_w5_avx2 BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5_avx2 -%define ecp_nistz256_select_w5_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5_nohw -%define ecp_nistz256_select_w7 BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7 -%define ecp_nistz256_select_w7_avx2 BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7_avx2 -%define ecp_nistz256_select_w7_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7_nohw %define ecp_nistz256_sqr_mont BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont -%define ecp_nistz256_sqr_mont_adx BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont_adx -%define ecp_nistz256_sqr_mont_nohw BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont_nohw -%define ecp_nistz256_sub BORINGSSL_PREFIX %+ _ecp_nistz256_sub %define fiat_curve25519_adx_mul BORINGSSL_PREFIX %+ _fiat_curve25519_adx_mul %define fiat_curve25519_adx_square BORINGSSL_PREFIX %+ _fiat_curve25519_adx_square %define fiat_p256_adx_mul BORINGSSL_PREFIX %+ _fiat_p256_adx_mul
diff --git a/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc b/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc index 060a50b..24eb900 100644 --- a/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc +++ b/gen/boringssl_prefix_symbols_internal_x86_win_asm.inc
@@ -227,38 +227,14 @@ %define _chacha20_poly1305_seal _ %+ BORINGSSL_PREFIX %+ _chacha20_poly1305_seal %define _chacha20_poly1305_seal_avx2 _ %+ BORINGSSL_PREFIX %+ _chacha20_poly1305_seal_avx2 %define _chacha20_poly1305_seal_sse41 _ %+ BORINGSSL_PREFIX %+ _chacha20_poly1305_seal_sse41 -%define _ecp_nistz256_div_by_2 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_div_by_2 -%define _ecp_nistz256_mul_by_2 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_by_2 -%define _ecp_nistz256_mul_by_3 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_by_3 %define _ecp_nistz256_mul_mont _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont -%define _ecp_nistz256_mul_mont_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont_adx -%define _ecp_nistz256_mul_mont_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_mul_mont_nohw -%define _ecp_nistz256_neg _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_neg %define _ecp_nistz256_ord_mul_mont _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont %define _ecp_nistz256_ord_mul_mont_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont_adx %define _ecp_nistz256_ord_mul_mont_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_mul_mont_nohw %define _ecp_nistz256_ord_sqr_mont _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont %define _ecp_nistz256_ord_sqr_mont_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont_adx %define _ecp_nistz256_ord_sqr_mont_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_ord_sqr_mont_nohw -%define _ecp_nistz256_point_add _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add -%define _ecp_nistz256_point_add_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_adx -%define _ecp_nistz256_point_add_affine _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine -%define _ecp_nistz256_point_add_affine_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine_adx -%define _ecp_nistz256_point_add_affine_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_affine_nohw -%define _ecp_nistz256_point_add_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_add_nohw -%define _ecp_nistz256_point_double _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_double -%define _ecp_nistz256_point_double_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_double_adx -%define _ecp_nistz256_point_double_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_point_double_nohw -%define _ecp_nistz256_select_w5 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5 -%define _ecp_nistz256_select_w5_avx2 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5_avx2 -%define _ecp_nistz256_select_w5_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w5_nohw -%define _ecp_nistz256_select_w7 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7 -%define _ecp_nistz256_select_w7_avx2 _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7_avx2 -%define _ecp_nistz256_select_w7_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_select_w7_nohw %define _ecp_nistz256_sqr_mont _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont -%define _ecp_nistz256_sqr_mont_adx _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont_adx -%define _ecp_nistz256_sqr_mont_nohw _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_sqr_mont_nohw -%define _ecp_nistz256_sub _ %+ BORINGSSL_PREFIX %+ _ecp_nistz256_sub %define _fiat_curve25519_adx_mul _ %+ BORINGSSL_PREFIX %+ _fiat_curve25519_adx_mul %define _fiat_curve25519_adx_square _ %+ BORINGSSL_PREFIX %+ _fiat_curve25519_adx_square %define _fiat_p256_adx_mul _ %+ BORINGSSL_PREFIX %+ _fiat_p256_adx_mul
diff --git a/gen/sources.bzl b/gen/sources.bzl index a7ff8d4..a92991a 100644 --- a/gen/sources.bzl +++ b/gen/sources.bzl
@@ -65,7 +65,6 @@ "crypto/fipsmodule/ec/ec_montgomery.cc.inc", "crypto/fipsmodule/ec/felem.cc.inc", "crypto/fipsmodule/ec/oct.cc.inc", - "crypto/fipsmodule/ec/p256-nistz.cc.inc", "crypto/fipsmodule/ec/p256.cc.inc", "crypto/fipsmodule/ec/scalar.cc.inc", "crypto/fipsmodule/ec/simple.cc.inc", @@ -657,7 +656,7 @@ "crypto/fipsmodule/ec/builtin_curves.h", "crypto/fipsmodule/ec/internal.h", "crypto/fipsmodule/ec/p256-nistz-table.h", - "crypto/fipsmodule/ec/p256-nistz.h", + "crypto/fipsmodule/ec/p256_internal.h", "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/entropy/internal.h", @@ -704,7 +703,6 @@ "third_party/fiat/p256_32.h", "third_party/fiat/p256_64.h", "third_party/fiat/p256_64_msvc.h", - "third_party/fiat/p256_field.c.inc", "third_party/fiat/p256_field_32.br.c.inc", "third_party/fiat/p256_field_64.br.c.inc", "third_party/fiat/p256_point.br.c.inc", @@ -781,7 +779,6 @@ "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", - "crypto/fipsmodule/ec/p256-nistz_test.cc", "crypto/fipsmodule/ec/p256_test.cc", "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/entropy/jitter_test.cc", @@ -895,7 +892,7 @@ "crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt", "crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt", "crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt", - "crypto/fipsmodule/ec/p256-nistz_tests.txt", + "crypto/fipsmodule/ec/p256-tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt", "crypto/fipsmodule/keccak/keccak_tests.txt",
diff --git a/gen/sources.cmake b/gen/sources.cmake index e04e968..36e9796 100644 --- a/gen/sources.cmake +++ b/gen/sources.cmake
@@ -69,7 +69,6 @@ crypto/fipsmodule/ec/ec_montgomery.cc.inc crypto/fipsmodule/ec/felem.cc.inc crypto/fipsmodule/ec/oct.cc.inc - crypto/fipsmodule/ec/p256-nistz.cc.inc crypto/fipsmodule/ec/p256.cc.inc crypto/fipsmodule/ec/scalar.cc.inc crypto/fipsmodule/ec/simple.cc.inc @@ -679,7 +678,7 @@ crypto/fipsmodule/ec/builtin_curves.h crypto/fipsmodule/ec/internal.h crypto/fipsmodule/ec/p256-nistz-table.h - crypto/fipsmodule/ec/p256-nistz.h + crypto/fipsmodule/ec/p256_internal.h crypto/fipsmodule/ec/p256_table.h crypto/fipsmodule/ecdsa/internal.h crypto/fipsmodule/entropy/internal.h @@ -726,7 +725,6 @@ third_party/fiat/p256_32.h third_party/fiat/p256_64.h third_party/fiat/p256_64_msvc.h - third_party/fiat/p256_field.c.inc third_party/fiat/p256_field_32.br.c.inc third_party/fiat/p256_field_64.br.c.inc third_party/fiat/p256_point.br.c.inc @@ -809,7 +807,6 @@ crypto/fipsmodule/bn/bn_test.cc crypto/fipsmodule/cmac/cmac_test.cc crypto/fipsmodule/ec/ec_test.cc - crypto/fipsmodule/ec/p256-nistz_test.cc crypto/fipsmodule/ec/p256_test.cc crypto/fipsmodule/ecdsa/ecdsa_test.cc crypto/fipsmodule/entropy/jitter_test.cc @@ -927,7 +924,7 @@ crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt - crypto/fipsmodule/ec/p256-nistz_tests.txt + crypto/fipsmodule/ec/p256-tests.txt crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt crypto/fipsmodule/keccak/keccak_tests.txt
diff --git a/gen/sources.gni b/gen/sources.gni index 611aab5..8172b50 100644 --- a/gen/sources.gni +++ b/gen/sources.gni
@@ -65,7 +65,6 @@ "crypto/fipsmodule/ec/ec_montgomery.cc.inc", "crypto/fipsmodule/ec/felem.cc.inc", "crypto/fipsmodule/ec/oct.cc.inc", - "crypto/fipsmodule/ec/p256-nistz.cc.inc", "crypto/fipsmodule/ec/p256.cc.inc", "crypto/fipsmodule/ec/scalar.cc.inc", "crypto/fipsmodule/ec/simple.cc.inc", @@ -657,7 +656,7 @@ "crypto/fipsmodule/ec/builtin_curves.h", "crypto/fipsmodule/ec/internal.h", "crypto/fipsmodule/ec/p256-nistz-table.h", - "crypto/fipsmodule/ec/p256-nistz.h", + "crypto/fipsmodule/ec/p256_internal.h", "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/entropy/internal.h", @@ -704,7 +703,6 @@ "third_party/fiat/p256_32.h", "third_party/fiat/p256_64.h", "third_party/fiat/p256_64_msvc.h", - "third_party/fiat/p256_field.c.inc", "third_party/fiat/p256_field_32.br.c.inc", "third_party/fiat/p256_field_64.br.c.inc", "third_party/fiat/p256_point.br.c.inc", @@ -781,7 +779,6 @@ "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", - "crypto/fipsmodule/ec/p256-nistz_test.cc", "crypto/fipsmodule/ec/p256_test.cc", "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/entropy/jitter_test.cc", @@ -895,7 +892,7 @@ "crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt", "crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt", "crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt", - "crypto/fipsmodule/ec/p256-nistz_tests.txt", + "crypto/fipsmodule/ec/p256-tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt", "crypto/fipsmodule/keccak/keccak_tests.txt",
diff --git a/gen/sources.json b/gen/sources.json index 6309f27..966e727 100644 --- a/gen/sources.json +++ b/gen/sources.json
@@ -50,7 +50,6 @@ "crypto/fipsmodule/ec/ec_montgomery.cc.inc", "crypto/fipsmodule/ec/felem.cc.inc", "crypto/fipsmodule/ec/oct.cc.inc", - "crypto/fipsmodule/ec/p256-nistz.cc.inc", "crypto/fipsmodule/ec/p256.cc.inc", "crypto/fipsmodule/ec/scalar.cc.inc", "crypto/fipsmodule/ec/simple.cc.inc", @@ -639,7 +638,7 @@ "crypto/fipsmodule/ec/builtin_curves.h", "crypto/fipsmodule/ec/internal.h", "crypto/fipsmodule/ec/p256-nistz-table.h", - "crypto/fipsmodule/ec/p256-nistz.h", + "crypto/fipsmodule/ec/p256_internal.h", "crypto/fipsmodule/ec/p256_table.h", "crypto/fipsmodule/ecdsa/internal.h", "crypto/fipsmodule/entropy/internal.h", @@ -686,7 +685,6 @@ "third_party/fiat/p256_32.h", "third_party/fiat/p256_64.h", "third_party/fiat/p256_64_msvc.h", - "third_party/fiat/p256_field.c.inc", "third_party/fiat/p256_field_32.br.c.inc", "third_party/fiat/p256_field_64.br.c.inc", "third_party/fiat/p256_point.br.c.inc" @@ -762,7 +760,6 @@ "crypto/fipsmodule/bn/bn_test.cc", "crypto/fipsmodule/cmac/cmac_test.cc", "crypto/fipsmodule/ec/ec_test.cc", - "crypto/fipsmodule/ec/p256-nistz_test.cc", "crypto/fipsmodule/ec/p256_test.cc", "crypto/fipsmodule/ecdsa/ecdsa_test.cc", "crypto/fipsmodule/entropy/jitter_test.cc", @@ -874,7 +871,7 @@ "crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt", "crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt", "crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt", - "crypto/fipsmodule/ec/p256-nistz_tests.txt", + "crypto/fipsmodule/ec/p256-tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt", "crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt", "crypto/fipsmodule/keccak/keccak_tests.txt",
diff --git a/gen/sources.mk b/gen/sources.mk index 87b3d53..f24cef4 100644 --- a/gen/sources.mk +++ b/gen/sources.mk
@@ -64,7 +64,6 @@ crypto/fipsmodule/ec/ec_montgomery.cc.inc \ crypto/fipsmodule/ec/felem.cc.inc \ crypto/fipsmodule/ec/oct.cc.inc \ - crypto/fipsmodule/ec/p256-nistz.cc.inc \ crypto/fipsmodule/ec/p256.cc.inc \ crypto/fipsmodule/ec/scalar.cc.inc \ crypto/fipsmodule/ec/simple.cc.inc \ @@ -647,7 +646,7 @@ crypto/fipsmodule/ec/builtin_curves.h \ crypto/fipsmodule/ec/internal.h \ crypto/fipsmodule/ec/p256-nistz-table.h \ - crypto/fipsmodule/ec/p256-nistz.h \ + crypto/fipsmodule/ec/p256_internal.h \ crypto/fipsmodule/ec/p256_table.h \ crypto/fipsmodule/ecdsa/internal.h \ crypto/fipsmodule/entropy/internal.h \ @@ -694,7 +693,6 @@ third_party/fiat/p256_32.h \ third_party/fiat/p256_64.h \ third_party/fiat/p256_64_msvc.h \ - third_party/fiat/p256_field.c.inc \ third_party/fiat/p256_field_32.br.c.inc \ third_party/fiat/p256_field_64.br.c.inc \ third_party/fiat/p256_point.br.c.inc @@ -768,7 +766,6 @@ crypto/fipsmodule/bn/bn_test.cc \ crypto/fipsmodule/cmac/cmac_test.cc \ crypto/fipsmodule/ec/ec_test.cc \ - crypto/fipsmodule/ec/p256-nistz_test.cc \ crypto/fipsmodule/ec/p256_test.cc \ crypto/fipsmodule/ecdsa/ecdsa_test.cc \ crypto/fipsmodule/entropy/jitter_test.cc \ @@ -880,7 +877,7 @@ crypto/fipsmodule/cmac/cavp_aes192_cmac_tests.txt \ crypto/fipsmodule/cmac/cavp_aes256_cmac_tests.txt \ crypto/fipsmodule/ec/ec_scalar_base_mult_tests.txt \ - crypto/fipsmodule/ec/p256-nistz_tests.txt \ + crypto/fipsmodule/ec/p256-tests.txt \ crypto/fipsmodule/ecdsa/ecdsa_sign_tests.txt \ crypto/fipsmodule/ecdsa/ecdsa_verify_tests.txt \ crypto/fipsmodule/keccak/keccak_tests.txt \
diff --git a/include/openssl/prefix_symbols_internal_S.h b/include/openssl/prefix_symbols_internal_S.h index 4e5a51b..170ac1c 100644 --- a/include/openssl/prefix_symbols_internal_S.h +++ b/include/openssl/prefix_symbols_internal_S.h
@@ -231,38 +231,14 @@ #define _chacha20_poly1305_seal BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal) #define _chacha20_poly1305_seal_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal_avx2) #define _chacha20_poly1305_seal_sse41 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal_sse41) -#define _ecp_nistz256_div_by_2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_div_by_2) -#define _ecp_nistz256_mul_by_2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_by_2) -#define _ecp_nistz256_mul_by_3 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_by_3) #define _ecp_nistz256_mul_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont) -#define _ecp_nistz256_mul_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont_adx) -#define _ecp_nistz256_mul_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont_nohw) -#define _ecp_nistz256_neg BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_neg) #define _ecp_nistz256_ord_mul_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont) #define _ecp_nistz256_ord_mul_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont_adx) #define _ecp_nistz256_ord_mul_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont_nohw) #define _ecp_nistz256_ord_sqr_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont) #define _ecp_nistz256_ord_sqr_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont_adx) #define _ecp_nistz256_ord_sqr_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont_nohw) -#define _ecp_nistz256_point_add BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add) -#define _ecp_nistz256_point_add_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_adx) -#define _ecp_nistz256_point_add_affine BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine) -#define _ecp_nistz256_point_add_affine_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine_adx) -#define _ecp_nistz256_point_add_affine_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine_nohw) -#define _ecp_nistz256_point_add_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_nohw) -#define _ecp_nistz256_point_double BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double) -#define _ecp_nistz256_point_double_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double_adx) -#define _ecp_nistz256_point_double_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double_nohw) -#define _ecp_nistz256_select_w5 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5) -#define _ecp_nistz256_select_w5_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5_avx2) -#define _ecp_nistz256_select_w5_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5_nohw) -#define _ecp_nistz256_select_w7 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7) -#define _ecp_nistz256_select_w7_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7_avx2) -#define _ecp_nistz256_select_w7_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7_nohw) #define _ecp_nistz256_sqr_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont) -#define _ecp_nistz256_sqr_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont_adx) -#define _ecp_nistz256_sqr_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont_nohw) -#define _ecp_nistz256_sub BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sub) #define _fiat_curve25519_adx_mul BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_curve25519_adx_mul) #define _fiat_curve25519_adx_square BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_curve25519_adx_square) #define _fiat_p256_adx_mul BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_p256_adx_mul) @@ -533,38 +509,14 @@ #define chacha20_poly1305_seal BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal) #define chacha20_poly1305_seal_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal_avx2) #define chacha20_poly1305_seal_sse41 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(chacha20_poly1305_seal_sse41) -#define ecp_nistz256_div_by_2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_div_by_2) -#define ecp_nistz256_mul_by_2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_by_2) -#define ecp_nistz256_mul_by_3 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_by_3) #define ecp_nistz256_mul_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont) -#define ecp_nistz256_mul_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont_adx) -#define ecp_nistz256_mul_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_mul_mont_nohw) -#define ecp_nistz256_neg BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_neg) #define ecp_nistz256_ord_mul_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont) #define ecp_nistz256_ord_mul_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont_adx) #define ecp_nistz256_ord_mul_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_mul_mont_nohw) #define ecp_nistz256_ord_sqr_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont) #define ecp_nistz256_ord_sqr_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont_adx) #define ecp_nistz256_ord_sqr_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_ord_sqr_mont_nohw) -#define ecp_nistz256_point_add BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add) -#define ecp_nistz256_point_add_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_adx) -#define ecp_nistz256_point_add_affine BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine) -#define ecp_nistz256_point_add_affine_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine_adx) -#define ecp_nistz256_point_add_affine_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_affine_nohw) -#define ecp_nistz256_point_add_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_add_nohw) -#define ecp_nistz256_point_double BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double) -#define ecp_nistz256_point_double_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double_adx) -#define ecp_nistz256_point_double_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_point_double_nohw) -#define ecp_nistz256_select_w5 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5) -#define ecp_nistz256_select_w5_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5_avx2) -#define ecp_nistz256_select_w5_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w5_nohw) -#define ecp_nistz256_select_w7 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7) -#define ecp_nistz256_select_w7_avx2 BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7_avx2) -#define ecp_nistz256_select_w7_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_select_w7_nohw) #define ecp_nistz256_sqr_mont BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont) -#define ecp_nistz256_sqr_mont_adx BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont_adx) -#define ecp_nistz256_sqr_mont_nohw BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sqr_mont_nohw) -#define ecp_nistz256_sub BORINGSSL_ADD_USER_LABEL_AND_PREFIX(ecp_nistz256_sub) #define fiat_curve25519_adx_mul BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_curve25519_adx_mul) #define fiat_curve25519_adx_square BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_curve25519_adx_square) #define fiat_p256_adx_mul BORINGSSL_ADD_USER_LABEL_AND_PREFIX(fiat_p256_adx_mul)
diff --git a/include/openssl/prefix_symbols_internal_c.h b/include/openssl/prefix_symbols_internal_c.h index 6e84071..ef12452 100644 --- a/include/openssl/prefix_symbols_internal_c.h +++ b/include/openssl/prefix_symbols_internal_c.h
@@ -231,38 +231,14 @@ #define chacha20_poly1305_seal BORINGSSL_ADD_PREFIX(chacha20_poly1305_seal) #define chacha20_poly1305_seal_avx2 BORINGSSL_ADD_PREFIX(chacha20_poly1305_seal_avx2) #define chacha20_poly1305_seal_sse41 BORINGSSL_ADD_PREFIX(chacha20_poly1305_seal_sse41) -#define ecp_nistz256_div_by_2 BORINGSSL_ADD_PREFIX(ecp_nistz256_div_by_2) -#define ecp_nistz256_mul_by_2 BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_by_2) -#define ecp_nistz256_mul_by_3 BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_by_3) #define ecp_nistz256_mul_mont BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_mont) -#define ecp_nistz256_mul_mont_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_mont_adx) -#define ecp_nistz256_mul_mont_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_mul_mont_nohw) -#define ecp_nistz256_neg BORINGSSL_ADD_PREFIX(ecp_nistz256_neg) #define ecp_nistz256_ord_mul_mont BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_mul_mont) #define ecp_nistz256_ord_mul_mont_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_mul_mont_adx) #define ecp_nistz256_ord_mul_mont_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_mul_mont_nohw) #define ecp_nistz256_ord_sqr_mont BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_sqr_mont) #define ecp_nistz256_ord_sqr_mont_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_sqr_mont_adx) #define ecp_nistz256_ord_sqr_mont_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_ord_sqr_mont_nohw) -#define ecp_nistz256_point_add BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add) -#define ecp_nistz256_point_add_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_adx) -#define ecp_nistz256_point_add_affine BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_affine) -#define ecp_nistz256_point_add_affine_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_affine_adx) -#define ecp_nistz256_point_add_affine_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_affine_nohw) -#define ecp_nistz256_point_add_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_point_add_nohw) -#define ecp_nistz256_point_double BORINGSSL_ADD_PREFIX(ecp_nistz256_point_double) -#define ecp_nistz256_point_double_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_point_double_adx) -#define ecp_nistz256_point_double_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_point_double_nohw) -#define ecp_nistz256_select_w5 BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w5) -#define ecp_nistz256_select_w5_avx2 BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w5_avx2) -#define ecp_nistz256_select_w5_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w5_nohw) -#define ecp_nistz256_select_w7 BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w7) -#define ecp_nistz256_select_w7_avx2 BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w7_avx2) -#define ecp_nistz256_select_w7_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_select_w7_nohw) #define ecp_nistz256_sqr_mont BORINGSSL_ADD_PREFIX(ecp_nistz256_sqr_mont) -#define ecp_nistz256_sqr_mont_adx BORINGSSL_ADD_PREFIX(ecp_nistz256_sqr_mont_adx) -#define ecp_nistz256_sqr_mont_nohw BORINGSSL_ADD_PREFIX(ecp_nistz256_sqr_mont_nohw) -#define ecp_nistz256_sub BORINGSSL_ADD_PREFIX(ecp_nistz256_sub) #define fiat_curve25519_adx_mul BORINGSSL_ADD_PREFIX(fiat_curve25519_adx_mul) #define fiat_curve25519_adx_square BORINGSSL_ADD_PREFIX(fiat_curve25519_adx_square) #define fiat_p256_adx_mul BORINGSSL_ADD_PREFIX(fiat_p256_adx_mul)
diff --git a/third_party/fiat/bedrock_unverified_platform.c.inc b/third_party/fiat/bedrock_unverified_platform.c.inc index d2b03c1..cb9490a 100644 --- a/third_party/fiat/bedrock_unverified_platform.c.inc +++ b/third_party/fiat/bedrock_unverified_platform.c.inc
@@ -118,6 +118,16 @@ : [vz] "r"(vz), [c] "r"(c) : "cc"); return vnz; +#elif !defined(OPENSSL_NO_ASM) && (defined(__GNUC__) || defined(__clang__)) && \ + defined(__aarch64__) + br_word_t ret; + __asm__( + "cmp %[c], #0\n" + "csel %[ret], %[vz], %[vnz], EQ" + : [ret] "=r"(ret) + : [vz] "r"(vz), [vnz] "r"(vnz), [c] "r"(c) + : "cc"); + return ret; #else br_word_t m = br_broadcast_nonzero(c); return (m & vnz) | (~m & vz);
diff --git a/third_party/fiat/p256_64.h b/third_party/fiat/p256_64.h index 8cad169..ee07a45 100644 --- a/third_party/fiat/p256_64.h +++ b/third_party/fiat/p256_64.h
@@ -1,15 +1,3 @@ -#include <openssl/base.h> -#include "bedrock_unverified_platform.c.inc" -#include "p256_field_64.br.c.inc" -#include "../../crypto/internal.h" - -#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(__x86_64__) -extern "C" { -void fiat_p256_adx_mul(uint64_t*, const uint64_t*, const uint64_t*); -void fiat_p256_adx_sqr(uint64_t*, const uint64_t*); -} -#endif - /* Autogenerated: 'src/ExtractionOCaml/word_by_word_montgomery' --inline --static --use-value-barrier p256 64 '2^256 - 2^224 + 2^192 + 2^96 - 1' mul square add sub opp from_montgomery to_montgomery nonzero selectznz to_bytes from_bytes one msat divstep divstep_precomp */ /* curve description: p256 */ /* machine_wordsize = 64 (from "64") */ @@ -177,14 +165,6 @@ * */ static FIAT_P256_FIAT_INLINE void fiat_p256_mul(fiat_p256_montgomery_domain_field_element out1, const fiat_p256_montgomery_domain_field_element arg1, const fiat_p256_montgomery_domain_field_element arg2) { - // NOTE: edited by hand, see third_party/fiat/README.md -#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(__x86_64__) - if (bssl::CRYPTO_is_BMI1_capable() && bssl::CRYPTO_is_BMI2_capable() && - bssl::CRYPTO_is_ADX_capable()) { - fiat_p256_adx_mul(out1, arg1, arg2); - return; - } -#endif uint64_t x1; uint64_t x2; uint64_t x3; @@ -492,14 +472,6 @@ * */ static FIAT_P256_FIAT_INLINE void fiat_p256_square(fiat_p256_montgomery_domain_field_element out1, const fiat_p256_montgomery_domain_field_element arg1) { - // NOTE: edited by hand, see third_party/fiat/README.md -#if !defined(OPENSSL_NO_ASM) && defined(__GNUC__) && defined(__x86_64__) - if (bssl::CRYPTO_is_BMI1_capable() && bssl::CRYPTO_is_BMI2_capable() && - bssl::CRYPTO_is_ADX_capable()) { - fiat_p256_adx_sqr(out1, arg1); - return; - } -#endif uint64_t x1; uint64_t x2; uint64_t x3;
diff --git a/third_party/fiat/p256_64_msvc.h b/third_party/fiat/p256_64_msvc.h index aff43d3..abed672 100644 --- a/third_party/fiat/p256_64_msvc.h +++ b/third_party/fiat/p256_64_msvc.h
@@ -17,8 +17,6 @@ /* twos_complement_eval z = let x1 := z[0] + (z[1] << 64) + (z[2] << 128) + (z[3] << 192) in */ /* if x1 & (2^256-1) < 2^255 then x1 & (2^256-1) else (x1 & (2^256-1)) - 2^256 */ -#include "bedrock_unverified_platform.c.inc" -#include "p256_field_64.br.c.inc" #include <stdint.h> #include <intrin.h> #if defined(_M_X64)
diff --git a/third_party/fiat/p256_field.c.inc b/third_party/fiat/p256_field.c.inc index bbd25de..e69de29 100644 --- a/third_party/fiat/p256_field.c.inc +++ b/third_party/fiat/p256_field.c.inc
@@ -1,26 +0,0 @@ -#include <openssl/base.h> -#include "bedrock_unverified_platform.c.inc" - -#if defined(BORINGSSL_HAS_UINT128) -#include "p256_64.h" -#elif defined(OPENSSL_64_BIT) -#include "p256_64_msvc.h" -#else -#include "p256_field_32.br.c.inc" -#include "p256_32.h" -// the 32-bit Bedrock-generated field halving calls Fiat-C code for add, sub -static inline void p256_coord_add(br_word_t out, br_word_t x, br_word_t y) { - fiat_p256_add((uint32_t*)out, (const uint32_t*)x, (const uint32_t*)y); -} -static inline void p256_coord_sub(br_word_t out, br_word_t x, br_word_t y) { - fiat_p256_sub((uint32_t*)out, (const uint32_t*)x, (const uint32_t*)y); -} -#endif - -// the Bedrock-generated point operations call Fiat-C or Fiat-x86 mul, sqr -static inline void p256_coord_mul(br_word_t out, br_word_t x, br_word_t y) { - fiat_p256_mul((br_word_t*)out, (const br_word_t*)x, (const br_word_t*)y); -} -static inline void p256_coord_sqr(br_word_t out, br_word_t x) { - fiat_p256_square((br_word_t*)out, (const br_word_t*)x); -}
diff --git a/third_party/fiat/p256_point.br.c.inc b/third_party/fiat/p256_point.br.c.inc index 117547b..4928034 100644 --- a/third_party/fiat/p256_point.br.c.inc +++ b/third_party/fiat/p256_point.br.c.inc
@@ -110,3 +110,57 @@ } br_memcpy(p_out, p_sel, (br_word_t)96); } + +static inline br_word_t p256_point_add_affine_nz_nz_neq(br_word_t p_out, br_word_t p_P, br_word_t p_Q) { + br_word_t z1z1, Hsqr, ok, different_x, different_y, u2, Hcub, r, h, s2; + uint8_t _br_stackalloc_z1z1[32] = {0}; z1z1 = (br_word_t)&_br_stackalloc_z1z1; + uint8_t _br_stackalloc_u2[32] = {0}; u2 = (br_word_t)&_br_stackalloc_u2; + uint8_t _br_stackalloc_h[32] = {0}; h = (br_word_t)&_br_stackalloc_h; + uint8_t _br_stackalloc_s2[32] = {0}; s2 = (br_word_t)&_br_stackalloc_s2; + uint8_t _br_stackalloc_r[32] = {0}; r = (br_word_t)&_br_stackalloc_r; + uint8_t _br_stackalloc_Hsqr[32] = {0}; Hsqr = (br_word_t)&_br_stackalloc_Hsqr; + uint8_t _br_stackalloc_Hcub[32] = {0}; Hcub = (br_word_t)&_br_stackalloc_Hcub; + p256_coord_sqr(z1z1, (p_P+32)+32); + p256_coord_mul(u2, p_Q, z1z1); + p256_coord_sub(h, u2, p_P); + p256_coord_mul(s2, (p_P+32)+32, z1z1); + p256_coord_mul((p_out+32)+32, h, (p_P+32)+32); + p256_coord_mul(s2, s2, p_Q+32); + p256_coord_sub(r, s2, p_P+32); + p256_coord_sqr(Hsqr, h); + p256_coord_sqr(p_out, r); + p256_coord_mul(Hcub, Hsqr, h); + p256_coord_mul(u2, p_P, Hsqr); + different_x = p256_coord_nonzero(Hcub); + different_y = p256_coord_nonzero(p_out); + ok = br_value_barrier(different_x|different_y); + p256_coord_sub(p_out, p_out, Hcub); + p256_coord_sub(p_out, p_out, u2); + p256_coord_sub(p_out, p_out, u2); + p256_coord_sub(h, u2, p_out); + p256_coord_mul(s2, Hcub, p_P+32); + p256_coord_mul(h, h, r); + p256_coord_sub(p_out+32, h, s2); + return ok; +} + +static inline void p256_point_add_affinenz_conditional_vartime_if_doubling(br_word_t p_out, br_word_t p_P, br_word_t p_Q, br_word_t c) { + br_word_t nzQ, p_tmp, zeroP, zeroQ, ok, p_buf; + zeroP = p256_point_iszero(p_P); + nzQ = br_broadcast_nonzero(c); + zeroQ = ~nzQ; + uint8_t _br_stackalloc_p_tmp[96] = {0}; p_tmp = (br_word_t)&_br_stackalloc_p_tmp; + ok = p256_point_add_affine_nz_nz_neq(p_tmp, p_P, p_Q); + ok = br_declassify((zeroP|zeroQ)|ok); + uint8_t _br_stackalloc_p_buf[96] = {0}; p_buf = (br_word_t)&_br_stackalloc_p_buf; + br_memset(p_buf, (br_word_t)0, (br_word_t)3*32); + br_memcxor(p_buf, p_tmp, (br_word_t)3*32, (~zeroP)&(~zeroQ)); + br_memcxor(p_buf, p_P, (br_word_t)3*32, (~zeroP)&zeroQ); + br_memcxor(p_buf, p_Q, (br_word_t)3*32, zeroP&(~zeroQ)); + if (ok) { + /*skip*/ + } else { + p256_point_double(p_buf, p_P); + } + br_memcpy(p_out, p_buf, (br_word_t)96); +}