Tidy up primality code. We BN_cmp with 1 at the top, so the absolute value code never runs. This simplifies the BN_CTX business considerably. Also add a test for negative prime numbers. Change-Id: I500a56bc285c2f75576947cfb518e75c9e6861ce Reviewed-on: https://boringssl-review.googlesource.com/15367 Commit-Queue: Steven Valdez <svaldez@google.com> Reviewed-by: Steven Valdez <svaldez@google.com> CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
diff --git a/crypto/bn/bn_test.cc b/crypto/bn/bn_test.cc index fc48339..2d92276 100644 --- a/crypto/bn/bn_test.cc +++ b/crypto/bn/bn_test.cc
@@ -2739,6 +2739,26 @@ } } + // Negative numbers are not prime. + if (!BN_set_word(p.get(), 7)) { + return false; + } + BN_set_negative(p.get(), 1); + if (!BN_primality_test(&is_probably_prime_1, p.get(), BN_prime_checks, ctx, + false /* do_trial_division */, + nullptr /* callback */) || + is_probably_prime_1 != 0 || + !BN_primality_test(&is_probably_prime_2, p.get(), BN_prime_checks, ctx, + true /* do_trial_division */, + nullptr /* callback */) || + is_probably_prime_2 != 0) { + fprintf(stderr, + "TestPrimeChecking failed for -7 (is_prime: 0 vs %d without " + "trial division vs %d with it)\n", + is_probably_prime_1, is_probably_prime_2); + return false; + } + return true; }