Split off private_transform function in RSA.
This change extracts two, common parts of RSA_decrypt and RSA_sign into
a function called |private_transform|. It also allows this to be
overridden in a method, which is convenient for opaque keys that only
expose the raw RSA transform as it means that the padding code from
BoringSSL can be easily reimplemented.
One significant change here is that short RSA ciphertexts will no longer
be accepted. I think this is correct and OpenSSL has a comment about PGP
mistakenly stripping leading zeros. However, these is the possibility
that it could break something.
Change-Id: I258c5cbbf21314cc9b6e8d2a2b898fd9a440cd40
Reviewed-on: https://boringssl-review.googlesource.com/1554
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index 9827e69..bd0b0d1 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -386,6 +386,21 @@
int (*verify_raw)(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding);
+ /* private_transform takes a big-endian integer from |in|, calculates the
+ * d'th power of it, modulo the RSA modulus and writes the result as a
+ * big-endian integer to |out|. Both |in| and |out| are |len| bytes long and
+ * |len| is always equal to |RSA_size(rsa)|. If the result of the transform
+ * can be represented in fewer than |len| bytes, then |out| must be zero
+ * padded on the left.
+ *
+ * It returns one on success and zero otherwise.
+ *
+ * RSA decrypt and sign operations will call this, thus an ENGINE might wish
+ * to override it in order to avoid having to implement the padding
+ * functionality demanded by those, higher level, operations. */
+ int (*private_transform)(RSA *rsa, uint8_t *out, const uint8_t *in,
+ size_t len);
+
int (*mod_exp)(BIGNUM *r0, const BIGNUM *I, RSA *rsa,
BN_CTX *ctx); /* Can be null */
int (*bn_mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
@@ -470,8 +485,8 @@
#define RSA_F_RSA_padding_check_PKCS1_type_2 126
#define RSA_F_RSA_recover_crt_params 127
#define RSA_F_RSA_check_key 128
+#define RSA_F_private_transform 129
#define RSA_R_INVALID_MESSAGE_LENGTH 100
-#define RSA_R_DATA_GREATER_THAN_MOD_LEN 101
#define RSA_R_NO_PUBLIC_EXPONENT 102
#define RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE 103
#define RSA_R_BLOCK_TYPE_IS_NOT_01 104
@@ -514,5 +529,6 @@
#define RSA_R_CRT_VALUES_INCORRECT 141
#define RSA_R_INCONSISTENT_SET_OF_CRT_VALUES 142
#define RSA_R_ONLY_ONE_OF_P_Q_GIVEN 143
+#define RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN 144
#endif /* OPENSSL_HEADER_RSA_H */
