commit 53409ee3d7595ed37da472bc73b010cd2c8a5ffd
author David Benjamin <davidben@google.com> Thu Jun 23 16:02:21 2016 -0400
committer Adam Langley <agl@google.com> Thu Jun 23 21:22:33 2016 +0000
tree c200aac060e90f2cd666119896fc2faebcc998fb
parent ff594ca8c818db5e682c9f9bf5abcc88d49c36c4

Fix BN_is_prime* calls.

This function returns a tri-state -1 on error. We should check this.

Change-Id: I6fe130c11d10690923aac5ac7a6dfe3e3ff3f5e9
Reviewed-on: https://boringssl-review.googlesource.com/8490
Reviewed-by: Adam Langley <agl@google.com>

crypto/dh/check.c

diff --git a/crypto/dh/check.c b/crypto/dh/check.c
index d27fdf1..fbe58a5 100644
--- a/crypto/dh/check.c
+++ b/crypto/dh/check.c
@@ -120,7 +120,7 @@
    *   for 5, p mod 10 == 3 or 7
    * should hold.
    */
-  int ok = 0;
+  int ok = 0, r;
   BN_CTX *ctx = NULL;
   BN_ULONG l;
   BIGNUM *t1 = NULL, *t2 = NULL;
@@ -154,7 +154,11 @@
         *ret |= DH_CHECK_NOT_SUITABLE_GENERATOR;
       }
     }
-    if (!BN_is_prime_ex(dh->q, BN_prime_checks, ctx, NULL)) {
+    r = BN_is_prime_ex(dh->q, BN_prime_checks, ctx, NULL);
+    if (r < 0) {
+      goto err;
+    }
+    if (!r) {
       *ret |= DH_CHECK_Q_NOT_PRIME;
     }
     /* Check p == 1 mod q  i.e. q divides p - 1 */
@@ -181,13 +185,21 @@
     *ret |= DH_CHECK_UNABLE_TO_CHECK_GENERATOR;
   }
 
-  if (!BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL)) {
+  r = BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL);
+  if (r < 0) {
+    goto err;
+  }
+  if (!r) {
     *ret |= DH_CHECK_P_NOT_PRIME;
   } else if (!dh->q) {
     if (!BN_rshift1(t1, dh->p)) {
       goto err;
     }
-    if (!BN_is_prime_ex(t1, BN_prime_checks, ctx, NULL)) {
+    r = BN_is_prime_ex(t1, BN_prime_checks, ctx, NULL);
+    if (r < 0) {
+      goto err;
+    }
+    if (!r) {
       *ret |= DH_CHECK_P_NOT_SAFE_PRIME;
     }
   }