Handle BN_mod_word failures.

As of 67cb49d045f04973ddba0f92fe8a8ad483c7da89 and the corresponding upstream
change, BN_mod_word may fail, like BN_div_word. Handle this properly and
document in bn.h. Thanks to Brian Smith for pointing this out.

Change-Id: I6d4f32dc37bcabf70847c9a8b417d55d31b3a380
Reviewed-on: https://boringssl-review.googlesource.com/8491
Reviewed-by: Adam Langley <agl@google.com>

crypto/bn/bn_test.cc

diff --git a/crypto/bn/bn_test.cc b/crypto/bn/bn_test.cc
index d909ee2..25207e7 100644
--- a/crypto/bn/bn_test.cc
+++ b/crypto/bn/bn_test.cc
@@ -852,7 +852,7 @@
     BN_ULONG s = b->d[0];
     BN_ULONG rmod = BN_mod_word(b.get(), s);
     BN_ULONG r = BN_div_word(b.get(), s);
-    if (r == (BN_ULONG)-1) {
+    if (r == (BN_ULONG)-1 || rmod == (BN_ULONG)-1) {
       return false;
     }
 

crypto/bn/prime.c

diff --git a/crypto/bn/prime.c b/crypto/bn/prime.c
index d07e609..98a46a9 100644
--- a/crypto/bn/prime.c
+++ b/crypto/bn/prime.c
@@ -496,7 +496,11 @@
 
   if (do_trial_division) {
     for (i = 1; i < NUMPRIMES; i++) {
-      if (BN_mod_word(a, primes[i]) == 0) {
+      BN_ULONG mod = BN_mod_word(a, primes[i]);
+      if (mod == (BN_ULONG)-1) {
+        goto err;
+      }
+      if (mod == 0) {
         return 0;
       }
     }
@@ -653,7 +657,11 @@
 
   /* we now have a random number 'rnd' to test. */
   for (i = 1; i < NUMPRIMES; i++) {
-    mods[i] = (uint16_t)BN_mod_word(rnd, (BN_ULONG)primes[i]);
+    BN_ULONG mod = BN_mod_word(rnd, (BN_ULONG)primes[i]);
+    if (mod == (BN_ULONG)-1) {
+      return 0;
+    }
+    mods[i] = (uint16_t)mod;
   }
   /* If bits is so small that it fits into a single word then we
    * additionally don't want to exceed that many bits. */
@@ -753,7 +761,11 @@
 loop:
   for (i = 1; i < NUMPRIMES; i++) {
     /* check that rnd is a prime */
-    if (BN_mod_word(rnd, (BN_ULONG)primes[i]) <= 1) {
+    BN_ULONG mod = BN_mod_word(rnd, (BN_ULONG)primes[i]);
+    if (mod == (BN_ULONG)-1) {
+      goto err;
+    }
+    if (mod <= 1) {
       if (!BN_add(rnd, rnd, add)) {
         goto err;
       }
@@ -825,8 +837,12 @@
     /* check that p and q are prime */
     /* check that for p and q
      * gcd(p-1,primes) == 1 (except for 2) */
-    if ((BN_mod_word(p, (BN_ULONG)primes[i]) == 0) ||
-        (BN_mod_word(q, (BN_ULONG)primes[i]) == 0)) {
+    BN_ULONG pmod = BN_mod_word(p, (BN_ULONG)primes[i]);
+    BN_ULONG qmod = BN_mod_word(q, (BN_ULONG)primes[i]);
+    if (pmod == (BN_ULONG)-1 || qmod == (BN_ULONG)-1) {
+      goto err;
+    }
+    if (pmod == 0 || qmod == 0) {
       if (!BN_add(p, p, padd)) {
         goto err;
       }

crypto/dh/check.c

diff --git a/crypto/dh/check.c b/crypto/dh/check.c
index fbe58a5..ed1508f 100644
--- a/crypto/dh/check.c
+++ b/crypto/dh/check.c
@@ -173,11 +173,17 @@
     }
   } else if (BN_is_word(dh->g, DH_GENERATOR_2)) {
     l = BN_mod_word(dh->p, 24);
+    if (l == (BN_ULONG)-1) {
+      goto err;
+    }
     if (l != 11) {
       *ret |= DH_CHECK_NOT_SUITABLE_GENERATOR;
     }
   } else if (BN_is_word(dh->g, DH_GENERATOR_5)) {
     l = BN_mod_word(dh->p, 10);
+    if (l == (BN_ULONG)-1) {
+      goto err;
+    }
     if (l != 3 && l != 7) {
       *ret |= DH_CHECK_NOT_SUITABLE_GENERATOR;
     }

include/openssl/bn.h

diff --git a/include/openssl/bn.h b/include/openssl/bn.h
index e9be0f5..9ee2237 100644
--- a/include/openssl/bn.h
+++ b/include/openssl/bn.h
@@ -502,7 +502,7 @@
 
 /* Modulo arithmetic. */
 
-/* BN_mod_word returns |a| mod |w|. */
+/* BN_mod_word returns |a| mod |w| or (BN_ULONG)-1 on error. */
 OPENSSL_EXPORT BN_ULONG BN_mod_word(const BIGNUM *a, BN_ULONG w);
 
 /* BN_mod is a helper macro that calls |BN_div| and discards the quotient. */