Use consistent RSA keygen and import limits

RSA keygen currently checks for a limit of 256, but anything below 512
will fail later anyway. Use the same constant for both.

Update-Note: Trying to generate RSA key sizes between 256 and 511 bits
will still fail earlier. Before it would generate a key and then throw
it away.

Bug: 42290480
Change-Id: Ic2407e65cf54a8cd3828a888a5037c7ea4e9c070
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/86588
Auto-Submit: David Benjamin <davidben@google.com>
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: David Benjamin <davidben@google.com>

crypto/fipsmodule/rsa/internal.h

diff --git a/crypto/fipsmodule/rsa/internal.h b/crypto/fipsmodule/rsa/internal.h
index 9720cbf..449e8fa 100644
--- a/crypto/fipsmodule/rsa/internal.h
+++ b/crypto/fipsmodule/rsa/internal.h
@@ -27,6 +27,9 @@
 #endif
 
 
+// TODO(crbug.com/42290480): Raise this limit. 512-bit RSA was factored in 1999.
+#define OPENSSL_RSA_MIN_MODULUS_BITS 512
+
 // TODO(davidben): This is inside BCM because |RSA| is inside BCM, but BCM never
 // uses this. Split the RSA type in two.
 enum rsa_pss_params_t {

crypto/fipsmodule/rsa/rsa_impl.cc.inc

diff --git a/crypto/fipsmodule/rsa/rsa_impl.cc.inc b/crypto/fipsmodule/rsa/rsa_impl.cc.inc
index 2f0eeb0..8a43501 100644
--- a/crypto/fipsmodule/rsa/rsa_impl.cc.inc
+++ b/crypto/fipsmodule/rsa/rsa_impl.cc.inc
@@ -48,9 +48,7 @@
     return 0;
   }
 
-  // TODO(crbug.com/boringssl/607): Raise this limit. 512-bit RSA was factored
-  // in 1999.
-  if (n_bits < 512) {
+  if (n_bits < OPENSSL_RSA_MIN_MODULUS_BITS) {
     OPENSSL_PUT_ERROR(RSA, RSA_R_KEY_SIZE_TOO_SMALL);
     return 0;
   }
@@ -815,7 +813,7 @@
   bits &= ~127;
 
   // Reject excessively small keys.
-  if (bits < 256) {
+  if (bits < OPENSSL_RSA_MIN_MODULUS_BITS) {
     OPENSSL_PUT_ERROR(RSA, RSA_R_KEY_SIZE_TOO_SMALL);
     return 0;
   }

crypto/rsa/rsa_test.cc

diff --git a/crypto/rsa/rsa_test.cc b/crypto/rsa/rsa_test.cc
index d9172c8..f235377 100644
--- a/crypto/rsa/rsa_test.cc
+++ b/crypto/rsa/rsa_test.cc
@@ -667,19 +667,6 @@
   ERR_clear_error();
 }
 
-// Attempting to generate an excessively small key should fail.
-TEST(RSATest, GenerateSmallKey) {
-  bssl::UniquePtr<RSA> rsa(RSA_new());
-  ASSERT_TRUE(rsa);
-  bssl::UniquePtr<BIGNUM> e(BN_new());
-  ASSERT_TRUE(e);
-  ASSERT_TRUE(BN_set_word(e.get(), RSA_F4));
-
-  EXPECT_FALSE(RSA_generate_key_ex(rsa.get(), 255, e.get(), nullptr));
-  EXPECT_TRUE(
-      ErrorEquals(ERR_get_error(), ERR_LIB_RSA, RSA_R_KEY_SIZE_TOO_SMALL));
-}
-
 // Attempting to generate an funny RSA key length should round down.
 TEST(RSATest, RoundKeyLengths) {
   bssl::UniquePtr<BIGNUM> e(BN_new());
@@ -1286,11 +1273,23 @@
     return bssl::UniquePtr<RSA>(
         PEM_read_bio_RSA_PUBKEY(bio.get(), nullptr, nullptr, nullptr));
   };
+  auto generate_key = [](unsigned bits) -> bssl::UniquePtr<RSA> {
+    bssl::UniquePtr<RSA> rsa(RSA_new());
+    bssl::UniquePtr<BIGNUM> e(BN_new());
+    if (!rsa || !e || !BN_set_word(e.get(), RSA_F4) ||
+        !RSA_generate_key_ex(rsa.get(), bits, e.get(), nullptr)) {
+      return nullptr;
+    }
+    return rsa;
+  };
 
   // We support RSA-512 through RSA-8192.
   //
-  // TODO(crbug.com/boringssl/42290480): Raise this limit. 512-bit RSA was
-  // factored in 1999.
+  // TODO(crbug.com/42290480): Raise this limit. 512-bit RSA was factored in
+  // 1999.
+  EXPECT_FALSE(generate_key(511u));
+  EXPECT_TRUE(
+      ErrorEquals(ERR_get_error(), ERR_LIB_RSA, RSA_R_KEY_SIZE_TOO_SMALL));
   EXPECT_FALSE(read_private_key("crypto/rsa/test/rsa511.pem"));
   EXPECT_FALSE(read_public_key("crypto/rsa/test/rsa511pub.pem"));
 
@@ -1300,6 +1299,9 @@
   rsa = read_public_key("crypto/rsa/test/rsa512pub.pem");
   ASSERT_TRUE(rsa);
   EXPECT_EQ(RSA_bits(rsa.get()), 512u);
+  rsa = generate_key(512u);
+  ASSERT_TRUE(rsa);
+  EXPECT_EQ(RSA_bits(rsa.get()), 512u);
 
   rsa = read_private_key("crypto/rsa/test/rsa8192.pem");
   ASSERT_TRUE(rsa);
@@ -1307,6 +1309,7 @@
   rsa = read_public_key("crypto/rsa/test/rsa8192pub.pem");
   ASSERT_TRUE(rsa);
   EXPECT_EQ(RSA_bits(rsa.get()), 8192u);
+  // RSA-8192 takes too long to generate, so skip this.
 
   EXPECT_FALSE(read_private_key("crypto/rsa/test/rsa8193.pem"));
   EXPECT_FALSE(read_public_key("crypto/rsa/test/rsa8193pub.pem"));