Also check OPENSSL_RSA_MAX_MODULUS_BITS in RSA keygen

This doesn't change whether the function succeeds. It will just burn a
*TON* of CPU and then never return anything.

Change-Id: I997a1a3df4d805278df6ca98aef416946c466256
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/86927
Reviewed-by: Adam Langley <agl@google.com>
Auto-Submit: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>

crypto/fipsmodule/rsa/rsa_impl.cc.inc

diff --git a/crypto/fipsmodule/rsa/rsa_impl.cc.inc b/crypto/fipsmodule/rsa/rsa_impl.cc.inc
index d1b58e7..77cbca9 100644
--- a/crypto/fipsmodule/rsa/rsa_impl.cc.inc
+++ b/crypto/fipsmodule/rsa/rsa_impl.cc.inc
@@ -726,6 +726,11 @@
 // probability of about 2^-20.
 static int rsa_generate_key_impl(RSA *rsa, int bits, const BIGNUM *e_value,
                                  BN_GENCB *cb) {
+  if (bits > OPENSSL_RSA_MAX_MODULUS_BITS) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_MODULUS_TOO_LARGE);
+    return 0;
+  }
+
   // Always generate RSA keys which are a multiple of 128 bits. Round |bits|
   // down as needed.
   bits &= ~127;

crypto/rsa/rsa_test.cc

diff --git a/crypto/rsa/rsa_test.cc b/crypto/rsa/rsa_test.cc
index cff453d..81ba889 100644
--- a/crypto/rsa/rsa_test.cc
+++ b/crypto/rsa/rsa_test.cc
@@ -1314,6 +1314,7 @@
 
   EXPECT_FALSE(read_private_key("crypto/rsa/test/rsa8193.pem"));
   EXPECT_FALSE(read_public_key("crypto/rsa/test/rsa8193pub.pem"));
+  EXPECT_FALSE(generate_key(8193u));
 }
 
 #if defined(OPENSSL_THREADS)