size_t RSA functions.
This extends 79c59a30 to |RSA_public_encrypt|, |RSA_private_encrypt|,
and |RSA_public_decrypt|. It benefits Conscrypt, which expects these
functions to have the same signature as |RSA_public_private_decrypt|.
Change-Id: Id1ce3118e8f20a9f43fd4f7bfc478c72a0c64e4b
Reviewed-on: https://boringssl-review.googlesource.com/6286
Reviewed-by: David Benjamin <davidben@chromium.org>
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/rsa/rsa.c b/crypto/rsa/rsa.c
index 63eb170..2a0b4ce 100644
--- a/crypto/rsa/rsa.c
+++ b/crypto/rsa/rsa.c
@@ -199,7 +199,7 @@
padding);
}
-int RSA_public_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
+int RSA_public_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
int padding) {
size_t out_len;
@@ -207,6 +207,10 @@
return -1;
}
+ if (out_len > INT_MAX) {
+ OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
+ return -1;
+ }
return out_len;
}
@@ -220,7 +224,7 @@
padding);
}
-int RSA_private_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
+int RSA_private_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
int padding) {
size_t out_len;
@@ -228,6 +232,10 @@
return -1;
}
+ if (out_len > INT_MAX) {
+ OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
+ return -1;
+ }
return out_len;
}
@@ -266,7 +274,7 @@
padding);
}
-int RSA_public_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
+int RSA_public_decrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
int padding) {
size_t out_len;
@@ -274,6 +282,10 @@
return -1;
}
+ if (out_len > INT_MAX) {
+ OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
+ return -1;
+ }
return out_len;
}
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index e44e228..e0c4368 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -152,7 +152,7 @@
*
* WARNING: this function is dangerous because it breaks the usual return value
* convention. Use |RSA_encrypt| instead. */
-OPENSSL_EXPORT int RSA_public_encrypt(int flen, const uint8_t *from,
+OPENSSL_EXPORT int RSA_public_encrypt(size_t flen, const uint8_t *from,
uint8_t *to, RSA *rsa, int padding);
/* RSA_private_decrypt decrypts |flen| bytes from |from| with the public key in
@@ -244,7 +244,7 @@
*
* WARNING: this function is dangerous because it breaks the usual return value
* convention. Use |RSA_sign_raw| instead. */
-OPENSSL_EXPORT int RSA_private_encrypt(int flen, const uint8_t *from,
+OPENSSL_EXPORT int RSA_private_encrypt(size_t flen, const uint8_t *from,
uint8_t *to, RSA *rsa, int padding);
/* RSA_public_decrypt verifies |flen| bytes of signature from |from| using the
@@ -255,7 +255,7 @@
*
* WARNING: this function is dangerous because it breaks the usual return value
* convention. Use |RSA_verify_raw| instead. */
-OPENSSL_EXPORT int RSA_public_decrypt(int flen, const uint8_t *from,
+OPENSSL_EXPORT int RSA_public_decrypt(size_t flen, const uint8_t *from,
uint8_t *to, RSA *rsa, int padding);