size_t RSA_private_decrypt's input.

Change-Id: If05761052e235b38d9798b2fe4d8ba44293af891
Reviewed-on: https://boringssl-review.googlesource.com/5944
Reviewed-by: Adam Langley <agl@google.com>

crypto/rsa/rsa.c

diff --git a/crypto/rsa/rsa.c b/crypto/rsa/rsa.c
index 2f23165..63eb170 100644
--- a/crypto/rsa/rsa.c
+++ b/crypto/rsa/rsa.c
@@ -56,6 +56,7 @@
 
 #include <openssl/rsa.h>
 
+#include <limits.h>
 #include <string.h>
 
 #include <openssl/bn.h>
@@ -240,7 +241,7 @@
                                     padding);
 }
 
-int RSA_private_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
+int RSA_private_decrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
                         int padding) {
   size_t out_len;
 
@@ -248,6 +249,10 @@
     return -1;
   }
 
+  if (out_len > INT_MAX) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
+    return -1;
+  }
   return out_len;
 }
 

include/openssl/rsa.h

diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index 2be50dc..e44e228 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -164,7 +164,7 @@
  *
  * WARNING: this function is dangerous because it breaks the usual return value
  * convention. Use |RSA_decrypt| instead. */
-OPENSSL_EXPORT int RSA_private_decrypt(int flen, const uint8_t *from,
+OPENSSL_EXPORT int RSA_private_decrypt(size_t flen, const uint8_t *from,
                                        uint8_t *to, RSA *rsa, int padding);
 
 /* RSA_message_index_PKCS1_type_2 performs the first step of a PKCS #1 padding