size_t RSA functions.

This extends 79c59a30 to |RSA_public_encrypt|, |RSA_private_encrypt|,
and |RSA_public_decrypt|.  It benefits Conscrypt, which expects these
functions to have the same signature as |RSA_public_private_decrypt|.

Change-Id: Id1ce3118e8f20a9f43fd4f7bfc478c72a0c64e4b
Reviewed-on: https://boringssl-review.googlesource.com/6286
Reviewed-by: David Benjamin <davidben@chromium.org>
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/rsa/rsa.c b/crypto/rsa/rsa.c
index 63eb170..2a0b4ce 100644
--- a/crypto/rsa/rsa.c
+++ b/crypto/rsa/rsa.c
@@ -199,7 +199,7 @@
                                     padding);
 }
 
-int RSA_public_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
+int RSA_public_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
                        int padding) {
   size_t out_len;
 
@@ -207,6 +207,10 @@
     return -1;
   }
 
+  if (out_len > INT_MAX) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
+    return -1;
+  }
   return out_len;
 }
 
@@ -220,7 +224,7 @@
                                      padding);
 }
 
-int RSA_private_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
+int RSA_private_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
                         int padding) {
   size_t out_len;
 
@@ -228,6 +232,10 @@
     return -1;
   }
 
+  if (out_len > INT_MAX) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
+    return -1;
+  }
   return out_len;
 }
 
@@ -266,7 +274,7 @@
                                        padding);
 }
 
-int RSA_public_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
+int RSA_public_decrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
                        int padding) {
   size_t out_len;
 
@@ -274,6 +282,10 @@
     return -1;
   }
 
+  if (out_len > INT_MAX) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
+    return -1;
+  }
   return out_len;
 }
 
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index e44e228..e0c4368 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -152,7 +152,7 @@
  *
  * WARNING: this function is dangerous because it breaks the usual return value
  * convention. Use |RSA_encrypt| instead. */
-OPENSSL_EXPORT int RSA_public_encrypt(int flen, const uint8_t *from,
+OPENSSL_EXPORT int RSA_public_encrypt(size_t flen, const uint8_t *from,
                                       uint8_t *to, RSA *rsa, int padding);
 
 /* RSA_private_decrypt decrypts |flen| bytes from |from| with the public key in
@@ -244,7 +244,7 @@
  *
  * WARNING: this function is dangerous because it breaks the usual return value
  * convention. Use |RSA_sign_raw| instead. */
-OPENSSL_EXPORT int RSA_private_encrypt(int flen, const uint8_t *from,
+OPENSSL_EXPORT int RSA_private_encrypt(size_t flen, const uint8_t *from,
                                        uint8_t *to, RSA *rsa, int padding);
 
 /* RSA_public_decrypt verifies |flen| bytes of signature from |from| using the
@@ -255,7 +255,7 @@
  *
  * WARNING: this function is dangerous because it breaks the usual return value
  * convention. Use |RSA_verify_raw| instead. */
-OPENSSL_EXPORT int RSA_public_decrypt(int flen, const uint8_t *from,
+OPENSSL_EXPORT int RSA_public_decrypt(size_t flen, const uint8_t *from,
                                       uint8_t *to, RSA *rsa, int padding);