bn/asm/x86_64-mont5.pl: fix carry bug in bn_sqr8x_internal.

CVE-2017-3732

(Imported from upstream's 3f4bcf5bb664b47ed369a70b99fac4e0ad141bb3 and
3e7a496307ab1174c1f8f64eed4454c1c9cde1a8.)

Change-Id: I40255fdf4184e3b919758a72c3d3a7486d91ff65
Reviewed-on: https://boringssl-review.googlesource.com/13360
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: Adam Langley <agl@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
diff --git a/crypto/bn/bn_test.cc b/crypto/bn/bn_test.cc
index 8f93ad0..a152cdf 100644
--- a/crypto/bn/bn_test.cc
+++ b/crypto/bn/bn_test.cc
@@ -515,6 +515,54 @@
   return true;
 }
 
+static bool TestModSquare(FileTest *t, BN_CTX *ctx) {
+  bssl::UniquePtr<BIGNUM> a = GetBIGNUM(t, "A");
+  bssl::UniquePtr<BIGNUM> m = GetBIGNUM(t, "M");
+  bssl::UniquePtr<BIGNUM> mod_square = GetBIGNUM(t, "ModSquare");
+  if (!a || !m || !mod_square) {
+    return false;
+  }
+
+  bssl::UniquePtr<BIGNUM> a_copy(BN_new());
+  bssl::UniquePtr<BIGNUM> ret(BN_new());
+  if (!ret || !a_copy ||
+      !BN_mod_mul(ret.get(), a.get(), a.get(), m.get(), ctx) ||
+      !ExpectBIGNUMsEqual(t, "A * A (mod M)", mod_square.get(), ret.get()) ||
+      // Repeat the operation with |a_copy|.
+      !BN_copy(a_copy.get(), a.get()) ||
+      !BN_mod_mul(ret.get(), a.get(), a_copy.get(), m.get(), ctx) ||
+      !ExpectBIGNUMsEqual(t, "A * A_copy (mod M)", mod_square.get(),
+                          ret.get())) {
+    return false;
+  }
+
+  if (BN_is_odd(m.get())) {
+    // Reduce |a| and test the Montgomery version.
+    bssl::UniquePtr<BN_MONT_CTX> mont(BN_MONT_CTX_new());
+    bssl::UniquePtr<BIGNUM> a_tmp(BN_new());
+    if (!mont || !a_tmp ||
+        !BN_MONT_CTX_set(mont.get(), m.get(), ctx) ||
+        !BN_nnmod(a_tmp.get(), a.get(), m.get(), ctx) ||
+        !BN_to_montgomery(a_tmp.get(), a_tmp.get(), mont.get(), ctx) ||
+        !BN_mod_mul_montgomery(ret.get(), a_tmp.get(), a_tmp.get(), mont.get(),
+                               ctx) ||
+        !BN_from_montgomery(ret.get(), ret.get(), mont.get(), ctx) ||
+        !ExpectBIGNUMsEqual(t, "A * A (mod M) (Montgomery)",
+                            mod_square.get(), ret.get()) ||
+        // Repeat the operation with |a_copy|.
+        !BN_copy(a_copy.get(), a_tmp.get()) ||
+        !BN_mod_mul_montgomery(ret.get(), a_tmp.get(), a_copy.get(), mont.get(),
+                               ctx) ||
+        !BN_from_montgomery(ret.get(), ret.get(), mont.get(), ctx) ||
+        !ExpectBIGNUMsEqual(t, "A * A_copy (mod M) (Montgomery)",
+                            mod_square.get(), ret.get())) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 static bool TestModExp(FileTest *t, BN_CTX *ctx) {
   bssl::UniquePtr<BIGNUM> a = GetBIGNUM(t, "A");
   bssl::UniquePtr<BIGNUM> e = GetBIGNUM(t, "E");
@@ -649,6 +697,7 @@
     {"Product", TestProduct},
     {"Quotient", TestQuotient},
     {"ModMul", TestModMul},
+    {"ModSquare", TestModSquare},
     {"ModExp", TestModExp},
     {"Exp", TestExp},
     {"ModSqrt", TestModSqrt},