commit 42574efaff5466104bc04692e48f0900cd1f971a
author David Benjamin <davidben@chromium.org> Wed Mar 18 20:37:12 2015 -0400
committer Adam Langley <agl@google.com> Thu Mar 19 11:36:49 2015 +0000
tree a13c2a5e680b33d8a1303267fde5e3dcdae73f2d
parent cdcecf546b8ea59cd0ac00a53cef7f31c3169480

Avoid undefined behavior in probable_prime.

(Imported from upstream's e4676e900f165f5272991443225813002300b09b.)

Change-Id: I678e158c223daf2f7f9114f4e743d531fe2e2a93
Reviewed-on: https://boringssl-review.googlesource.com/4044
Reviewed-by: Adam Langley <agl@google.com>

crypto/bn/prime.c

diff --git a/crypto/bn/prime.c b/crypto/bn/prime.c
index 6bdc921..cf3afcf 100644
--- a/crypto/bn/prime.c
+++ b/crypto/bn/prime.c
@@ -659,7 +659,13 @@
   /* If bits is so small that it fits into a single word then we
    * additionally don't want to exceed that many bits. */
   if (is_single_word) {
-    BN_ULONG size_limit = (((BN_ULONG)1) << bits) - get_word(rnd) - 1;
+    BN_ULONG size_limit;
+    if (bits == BN_BITS2) {
+      /* Avoid undefined behavior. */
+      size_limit = ~((BN_ULONG)0) - get_word(rnd);
+    } else {
+      size_limit = (((BN_ULONG)1) << bits) - get_word(rnd) - 1;
+    }
     if (size_limit < maxdelta) {
       maxdelta = size_limit;
     }