commit e55b32ddff4e1d4f580f196485944bcbb5218b70
author David Benjamin <davidben@google.com> Thu Jun 22 10:53:25 2017 -0400
committer CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org> Thu Jun 22 15:20:15 2017 +0000
tree 1fccfc4f07b45da9155717e5c3114f9bd78fcacb
parent 6fff386492d9f316f5f780ff9d0ddaf1700f98a9

Don't crash when decrypting with public keys.

Public and private RSA keys have the same type in OpenSSL, so it's
probably prudent for us to catch this case with an error rather than
crash. (As we do if you, say, configure RSA-PSS parameters on an Ed25519
EVP_PKEY.) Bindings libraries, in particular, tend to hit this sort of
then when their callers do silly things.

Change-Id: I2555e9bfe716a9f15273abd887a8459c682432dd
Reviewed-on: https://boringssl-review.googlesource.com/17325
Commit-Queue: Adam Langley <agl@google.com>
Reviewed-by: Adam Langley <agl@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

crypto/fipsmodule/rsa/rsa_impl.c

diff --git a/crypto/fipsmodule/rsa/rsa_impl.c b/crypto/fipsmodule/rsa/rsa_impl.c
index 342f1c2..c79f864 100644
--- a/crypto/fipsmodule/rsa/rsa_impl.c
+++ b/crypto/fipsmodule/rsa/rsa_impl.c
@@ -533,6 +533,11 @@
 
 int rsa_default_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
                                   size_t len) {
+  if (rsa->n == NULL || rsa->d == NULL) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_VALUE_MISSING);
+    return 0;
+  }
+
   BIGNUM *f, *result;
   BN_CTX *ctx = NULL;
   unsigned blinding_index = 0;

crypto/rsa_extra/rsa_test.cc

diff --git a/crypto/rsa_extra/rsa_test.cc b/crypto/rsa_extra/rsa_test.cc
index 162ac05..1cc71ea 100644
--- a/crypto/rsa_extra/rsa_test.cc
+++ b/crypto/rsa_extra/rsa_test.cc
@@ -697,6 +697,22 @@
       RSA_verify(NID_sha256, kZeros, sizeof(kZeros), sig, sig_len, rsa.get()));
 }
 
+// Test that decrypting with a public key fails gracefully rather than crashing.
+TEST(RSATest, DecryptPublic) {
+  bssl::UniquePtr<RSA> pub(
+      RSA_public_key_from_bytes(kFIPSPublicKey, sizeof(kFIPSPublicKey) - 1));
+  ASSERT_TRUE(pub);
+  ASSERT_EQ(1024u / 8u, RSA_size(pub.get()));
+
+  size_t len;
+  uint8_t in[1024 / 8] = {0}, out[1024 / 8];
+  EXPECT_FALSE(RSA_decrypt(pub.get(), &len, out, sizeof(out), in, sizeof(in),
+                           RSA_PKCS1_PADDING));
+  uint32_t err = ERR_get_error();
+  EXPECT_EQ(ERR_LIB_RSA, ERR_GET_LIB(err));
+  EXPECT_EQ(RSA_R_VALUE_MISSING, ERR_GET_REASON(err));
+}
+
 #if !defined(BORINGSSL_SHARED_LIBRARY)
 TEST(RSATest, SqrtTwo) {
   bssl::UniquePtr<BIGNUM> sqrt(BN_new()), pow2(BN_new());