Rearrange bn/generic.c
In preparation for adding aarch64 bn_add_words and bn_sub_words
implementations, rearrange this so we first define BN_ADD_ASM and
BN_MUL_ASM defines, and then gate fallbacks on that. This also required
moving some functions around to group the add/mul functions together.
Change-Id: I59281706db35ad3fb1186a4afd345a820f5542d2
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/56965
Reviewed-by: Bob Beck <bbe@google.com>
Commit-Queue: Bob Beck <bbe@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
Auto-Submit: David Benjamin <davidben@google.com>
diff --git a/crypto/fipsmodule/bn/generic.c b/crypto/fipsmodule/bn/generic.c
index ee80a3c..628cc53 100644
--- a/crypto/fipsmodule/bn/generic.c
+++ b/crypto/fipsmodule/bn/generic.c
@@ -61,11 +61,20 @@
#include "internal.h"
-// This file has two other implementations: x86 assembly language in
-// asm/bn-586.pl and x86_64 inline assembly in asm/x86_64-gcc.c.
-#if defined(OPENSSL_NO_ASM) || \
- !(defined(OPENSSL_X86) || \
- (defined(OPENSSL_X86_64) && (defined(__GNUC__) || defined(__clang__))))
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86)
+// See asm/bn-586.pl.
+#define BN_ADD_ASM
+#define BN_MUL_ASM
+#endif
+
+#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_X86_64) && \
+ (defined(__GNUC__) || defined(__clang__))
+// See asm/x86_64-gcc.c
+#define BN_ADD_ASM
+#define BN_MUL_ASM
+#endif
+
+#if !defined(BN_MUL_ASM)
#ifdef BN_ULLONG
#define mul_add(r, a, w, c) \
@@ -201,157 +210,6 @@
}
}
-#ifdef BN_ULLONG
-BN_ULONG bn_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
- size_t n) {
- BN_ULLONG ll = 0;
-
- if (n == 0) {
- return 0;
- }
-
- while (n & ~3) {
- ll += (BN_ULLONG)a[0] + b[0];
- r[0] = (BN_ULONG)ll;
- ll >>= BN_BITS2;
- ll += (BN_ULLONG)a[1] + b[1];
- r[1] = (BN_ULONG)ll;
- ll >>= BN_BITS2;
- ll += (BN_ULLONG)a[2] + b[2];
- r[2] = (BN_ULONG)ll;
- ll >>= BN_BITS2;
- ll += (BN_ULLONG)a[3] + b[3];
- r[3] = (BN_ULONG)ll;
- ll >>= BN_BITS2;
- a += 4;
- b += 4;
- r += 4;
- n -= 4;
- }
- while (n) {
- ll += (BN_ULLONG)a[0] + b[0];
- r[0] = (BN_ULONG)ll;
- ll >>= BN_BITS2;
- a++;
- b++;
- r++;
- n--;
- }
- return (BN_ULONG)ll;
-}
-
-#else // !BN_ULLONG
-
-BN_ULONG bn_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
- size_t n) {
- BN_ULONG c, l, t;
-
- if (n == 0) {
- return (BN_ULONG)0;
- }
-
- c = 0;
- while (n & ~3) {
- t = a[0];
- t += c;
- c = (t < c);
- l = t + b[0];
- c += (l < t);
- r[0] = l;
- t = a[1];
- t += c;
- c = (t < c);
- l = t + b[1];
- c += (l < t);
- r[1] = l;
- t = a[2];
- t += c;
- c = (t < c);
- l = t + b[2];
- c += (l < t);
- r[2] = l;
- t = a[3];
- t += c;
- c = (t < c);
- l = t + b[3];
- c += (l < t);
- r[3] = l;
- a += 4;
- b += 4;
- r += 4;
- n -= 4;
- }
- while (n) {
- t = a[0];
- t += c;
- c = (t < c);
- l = t + b[0];
- c += (l < t);
- r[0] = l;
- a++;
- b++;
- r++;
- n--;
- }
- return (BN_ULONG)c;
-}
-
-#endif // !BN_ULLONG
-
-BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
- size_t n) {
- BN_ULONG t1, t2;
- int c = 0;
-
- if (n == 0) {
- return (BN_ULONG)0;
- }
-
- while (n & ~3) {
- t1 = a[0];
- t2 = b[0];
- r[0] = t1 - t2 - c;
- if (t1 != t2) {
- c = (t1 < t2);
- }
- t1 = a[1];
- t2 = b[1];
- r[1] = t1 - t2 - c;
- if (t1 != t2) {
- c = (t1 < t2);
- }
- t1 = a[2];
- t2 = b[2];
- r[2] = t1 - t2 - c;
- if (t1 != t2) {
- c = (t1 < t2);
- }
- t1 = a[3];
- t2 = b[3];
- r[3] = t1 - t2 - c;
- if (t1 != t2) {
- c = (t1 < t2);
- }
- a += 4;
- b += 4;
- r += 4;
- n -= 4;
- }
- while (n) {
- t1 = a[0];
- t2 = b[0];
- r[0] = t1 - t2 - c;
- if (t1 != t2) {
- c = (t1 < t2);
- }
- a++;
- b++;
- r++;
- n--;
- }
- return c;
-}
-
// mul_add_c(a,b,c0,c1,c2) -- c+=a*b for three word number c=(c2,c1,c0)
// mul_add_c2(a,b,c0,c1,c2) -- c+=2*a*b for three word number c=(c2,c1,c0)
// sqr_add_c(a,i,c0,c1,c2) -- c+=a[i]^2 for three word number c=(c2,c1,c0)
@@ -708,4 +566,159 @@
#undef sqr_add_c
#undef sqr_add_c2
-#endif
+#endif // !BN_MUL_ASM
+
+#if !defined(BN_ADD_ASM)
+
+#ifdef BN_ULLONG
+BN_ULONG bn_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
+ size_t n) {
+ BN_ULLONG ll = 0;
+
+ if (n == 0) {
+ return 0;
+ }
+
+ while (n & ~3) {
+ ll += (BN_ULLONG)a[0] + b[0];
+ r[0] = (BN_ULONG)ll;
+ ll >>= BN_BITS2;
+ ll += (BN_ULLONG)a[1] + b[1];
+ r[1] = (BN_ULONG)ll;
+ ll >>= BN_BITS2;
+ ll += (BN_ULLONG)a[2] + b[2];
+ r[2] = (BN_ULONG)ll;
+ ll >>= BN_BITS2;
+ ll += (BN_ULLONG)a[3] + b[3];
+ r[3] = (BN_ULONG)ll;
+ ll >>= BN_BITS2;
+ a += 4;
+ b += 4;
+ r += 4;
+ n -= 4;
+ }
+ while (n) {
+ ll += (BN_ULLONG)a[0] + b[0];
+ r[0] = (BN_ULONG)ll;
+ ll >>= BN_BITS2;
+ a++;
+ b++;
+ r++;
+ n--;
+ }
+ return (BN_ULONG)ll;
+}
+
+#else // !BN_ULLONG
+
+BN_ULONG bn_add_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
+ size_t n) {
+ BN_ULONG c, l, t;
+
+ if (n == 0) {
+ return (BN_ULONG)0;
+ }
+
+ c = 0;
+ while (n & ~3) {
+ t = a[0];
+ t += c;
+ c = (t < c);
+ l = t + b[0];
+ c += (l < t);
+ r[0] = l;
+ t = a[1];
+ t += c;
+ c = (t < c);
+ l = t + b[1];
+ c += (l < t);
+ r[1] = l;
+ t = a[2];
+ t += c;
+ c = (t < c);
+ l = t + b[2];
+ c += (l < t);
+ r[2] = l;
+ t = a[3];
+ t += c;
+ c = (t < c);
+ l = t + b[3];
+ c += (l < t);
+ r[3] = l;
+ a += 4;
+ b += 4;
+ r += 4;
+ n -= 4;
+ }
+ while (n) {
+ t = a[0];
+ t += c;
+ c = (t < c);
+ l = t + b[0];
+ c += (l < t);
+ r[0] = l;
+ a++;
+ b++;
+ r++;
+ n--;
+ }
+ return (BN_ULONG)c;
+}
+
+#endif // !BN_ULLONG
+
+BN_ULONG bn_sub_words(BN_ULONG *r, const BN_ULONG *a, const BN_ULONG *b,
+ size_t n) {
+ BN_ULONG t1, t2;
+ int c = 0;
+
+ if (n == 0) {
+ return (BN_ULONG)0;
+ }
+
+ while (n & ~3) {
+ t1 = a[0];
+ t2 = b[0];
+ r[0] = t1 - t2 - c;
+ if (t1 != t2) {
+ c = (t1 < t2);
+ }
+ t1 = a[1];
+ t2 = b[1];
+ r[1] = t1 - t2 - c;
+ if (t1 != t2) {
+ c = (t1 < t2);
+ }
+ t1 = a[2];
+ t2 = b[2];
+ r[2] = t1 - t2 - c;
+ if (t1 != t2) {
+ c = (t1 < t2);
+ }
+ t1 = a[3];
+ t2 = b[3];
+ r[3] = t1 - t2 - c;
+ if (t1 != t2) {
+ c = (t1 < t2);
+ }
+ a += 4;
+ b += 4;
+ r += 4;
+ n -= 4;
+ }
+ while (n) {
+ t1 = a[0];
+ t2 = b[0];
+ r[0] = t1 - t2 - c;
+ if (t1 != t2) {
+ c = (t1 < t2);
+ }
+ a++;
+ b++;
+ r++;
+ n--;
+ }
+ return c;
+}
+
+#endif // !BN_ADD_ASM
