Avoid a copy when using RSA_PADDING_NONE.

RSA_PADDING_NONE is actually the important one for RSA_decrypt since OAEP isn't
used much and RSA_PKCS1_PADDING is unsafe to use due to timing constraints.
(The SSL stack uses RSA_PADDING_NONE and does the padding check separately.)

Change-Id: I5f9d168e7c34796a41bf01fc1878022742b63501
Reviewed-on: https://boringssl-review.googlesource.com/5641
Reviewed-by: Adam Langley <agl@google.com>

crypto/rsa/internal.h

diff --git a/crypto/rsa/internal.h b/crypto/rsa/internal.h
index 06fd387..c0044c3 100644
--- a/crypto/rsa/internal.h
+++ b/crypto/rsa/internal.h
@@ -107,8 +107,6 @@
                                       const EVP_MD *md, const EVP_MD *mgf1md);
 int RSA_padding_add_none(uint8_t *to, unsigned to_len, const uint8_t *from,
                          unsigned from_len);
-int RSA_padding_check_none(uint8_t *to, unsigned to_len, const uint8_t *from,
-                           unsigned from_len);
 
 /* RSA_private_transform calls either the method-specific |private_transform|
  * function (if given) or the generic one. See the comment for

crypto/rsa/padding.c

diff --git a/crypto/rsa/padding.c b/crypto/rsa/padding.c
index 6ac41ba..5a42e24 100644
--- a/crypto/rsa/padding.c
+++ b/crypto/rsa/padding.c
@@ -300,17 +300,6 @@
   return 1;
 }
 
-int RSA_padding_check_none(uint8_t *to, unsigned tlen, const uint8_t *from,
-                           unsigned flen) {
-  if (flen > tlen) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
-    return -1;
-  }
-
-  memcpy(to, from, flen);
-  return flen;
-}
-
 int PKCS1_MGF1(uint8_t *mask, unsigned len, const uint8_t *seed,
                unsigned seedlen, const EVP_MD *dgst) {
   unsigned outlen = 0;

crypto/rsa/rsa_impl.c

diff --git a/crypto/rsa/rsa_impl.c b/crypto/rsa/rsa_impl.c
index e1dcaf3..aa1b70f 100644
--- a/crypto/rsa/rsa_impl.c
+++ b/crypto/rsa/rsa_impl.c
@@ -372,10 +372,15 @@
     return 0;
   }
 
-  buf = OPENSSL_malloc(rsa_size);
-  if (buf == NULL) {
-    OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
-    goto err;
+  if (padding == RSA_NO_PADDING) {
+    buf = out;
+  } else {
+    /* Allocate a temporary buffer to hold the padded plaintext. */
+    buf = OPENSSL_malloc(rsa_size);
+    if (buf == NULL) {
+      OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
+      goto err;
+    }
   }
 
   if (in_len != rsa_size) {
@@ -397,7 +402,7 @@
                                             NULL, 0, NULL, NULL);
       break;
     case RSA_NO_PADDING:
-      r = RSA_padding_check_none(out, rsa_size, buf, rsa_size);
+      r = rsa_size;
       break;
     default:
       OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
@@ -412,7 +417,7 @@
   }
 
 err:
-  if (buf != NULL) {
+  if (padding != RSA_NO_PADDING && buf != NULL) {
     OPENSSL_cleanse(buf, rsa_size);
     OPENSSL_free(buf);
   }
@@ -459,8 +464,17 @@
   BN_CTX_start(ctx);
   f = BN_CTX_get(ctx);
   result = BN_CTX_get(ctx);
-  buf = OPENSSL_malloc(rsa_size);
-  if (!f || !result || !buf) {
+  if (padding == RSA_NO_PADDING) {
+    buf = out;
+  } else {
+    /* Allocate a temporary buffer to hold the padded plaintext. */
+    buf = OPENSSL_malloc(rsa_size);
+    if (buf == NULL) {
+      OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
+      goto err;
+    }
+  }
+  if (!f || !result) {
     OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
     goto err;
   }
@@ -501,7 +515,7 @@
       r = RSA_padding_check_PKCS1_type_1(out, rsa_size, buf, rsa_size);
       break;
     case RSA_NO_PADDING:
-      r = RSA_padding_check_none(out, rsa_size, buf, rsa_size);
+      r = rsa_size;
       break;
     default:
       OPENSSL_PUT_ERROR(RSA, RSA_R_UNKNOWN_PADDING_TYPE);
@@ -520,7 +534,7 @@
     BN_CTX_end(ctx);
     BN_CTX_free(ctx);
   }
-  if (buf != NULL) {
+  if (padding != RSA_NO_PADDING && buf != NULL) {
     OPENSSL_cleanse(buf, rsa_size);
     OPENSSL_free(buf);
   }