Split off private_transform function in RSA.
This change extracts two, common parts of RSA_decrypt and RSA_sign into
a function called |private_transform|. It also allows this to be
overridden in a method, which is convenient for opaque keys that only
expose the raw RSA transform as it means that the padding code from
BoringSSL can be easily reimplemented.
One significant change here is that short RSA ciphertexts will no longer
be accepted. I think this is correct and OpenSSL has a comment about PGP
mistakenly stripping leading zeros. However, these is the possibility
that it could break something.
Change-Id: I258c5cbbf21314cc9b6e8d2a2b898fd9a440cd40
Reviewed-on: https://boringssl-review.googlesource.com/1554
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/rsa/internal.h b/crypto/rsa/internal.h
index 410e72f..aa3a982 100644
--- a/crypto/rsa/internal.h
+++ b/crypto/rsa/internal.h
@@ -122,6 +122,12 @@
const EVP_MD *Hash, const EVP_MD *mgf1Hash,
int sLen);
+/* RSA_private_transform calls either the method-specific |private_transform|
+ * function (if given) or the generic one. See the comment for
+ * |private_transform| in |rsa_meth_st|. */
+int RSA_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
+ size_t len);
+
typedef struct rsa_pss_params_st {
X509_ALGOR *hashAlgorithm;
X509_ALGOR *maskGenAlgorithm;
diff --git a/crypto/rsa/rsa.c b/crypto/rsa/rsa.c
index 54187bb..147de19 100644
--- a/crypto/rsa/rsa.c
+++ b/crypto/rsa/rsa.c
@@ -741,3 +741,12 @@
}
return ok;
}
+
+int RSA_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
+ size_t len) {
+ if (rsa->meth->private_transform) {
+ return rsa->meth->private_transform(rsa, out, in, len);
+ }
+
+ return RSA_default_method.private_transform(rsa, out, in, len);
+}
diff --git a/crypto/rsa/rsa_error.c b/crypto/rsa/rsa_error.c
index e76f32a..3c8ebe4 100644
--- a/crypto/rsa/rsa_error.c
+++ b/crypto/rsa/rsa_error.c
@@ -43,6 +43,7 @@
{ERR_PACK(ERR_LIB_RSA, RSA_F_encrypt, 0), "encrypt"},
{ERR_PACK(ERR_LIB_RSA, RSA_F_keygen, 0), "keygen"},
{ERR_PACK(ERR_LIB_RSA, RSA_F_pkcs1_prefixed_msg, 0), "pkcs1_prefixed_msg"},
+ {ERR_PACK(ERR_LIB_RSA, RSA_F_private_transform, 0), "private_transform"},
{ERR_PACK(ERR_LIB_RSA, RSA_F_rsa_setup_blinding, 0), "rsa_setup_blinding"},
{ERR_PACK(ERR_LIB_RSA, RSA_F_sign_raw, 0), "sign_raw"},
{ERR_PACK(ERR_LIB_RSA, RSA_F_verify_raw, 0), "verify_raw"},
@@ -56,7 +57,7 @@
{ERR_PACK(ERR_LIB_RSA, 0, RSA_R_BN_NOT_INITIALIZED), "BN_NOT_INITIALIZED"},
{ERR_PACK(ERR_LIB_RSA, 0, RSA_R_CRT_PARAMS_ALREADY_GIVEN), "CRT_PARAMS_ALREADY_GIVEN"},
{ERR_PACK(ERR_LIB_RSA, 0, RSA_R_CRT_VALUES_INCORRECT), "CRT_VALUES_INCORRECT"},
- {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_DATA_GREATER_THAN_MOD_LEN), "DATA_GREATER_THAN_MOD_LEN"},
+ {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN), "DATA_LEN_NOT_EQUAL_TO_MOD_LEN"},
{ERR_PACK(ERR_LIB_RSA, 0, RSA_R_DATA_TOO_LARGE), "DATA_TOO_LARGE"},
{ERR_PACK(ERR_LIB_RSA, 0, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE), "DATA_TOO_LARGE_FOR_KEY_SIZE"},
{ERR_PACK(ERR_LIB_RSA, 0, RSA_R_DATA_TOO_LARGE_FOR_MODULUS), "DATA_TOO_LARGE_FOR_MODULUS"},
diff --git a/crypto/rsa/rsa_impl.c b/crypto/rsa/rsa_impl.c
index 926f48f..7eb1bc5 100644
--- a/crypto/rsa/rsa_impl.c
+++ b/crypto/rsa/rsa_impl.c
@@ -327,11 +327,7 @@
static int sign_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
- BIGNUM *f, *result;
uint8_t *buf = NULL;
- BN_CTX *ctx = NULL;
- unsigned blinding_index = 0;
- BN_BLINDING *blinding = NULL;
int i, ret = 0;
if (max_out < rsa_size) {
@@ -339,15 +335,8 @@
return 0;
}
- ctx = BN_CTX_new();
- if (ctx == NULL) {
- goto err;
- }
- BN_CTX_start(ctx);
- f = BN_CTX_get(ctx);
- result = BN_CTX_get(ctx);
buf = OPENSSL_malloc(rsa_size);
- if (!f || !result || !buf) {
+ if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, sign_raw, ERR_R_MALLOC_FAILURE);
goto err;
}
@@ -363,83 +352,24 @@
OPENSSL_PUT_ERROR(RSA, sign_raw, RSA_R_UNKNOWN_PADDING_TYPE);
goto err;
}
+
if (i <= 0) {
goto err;
}
- if (BN_bin2bn(buf, rsa_size, f) == NULL) {
- goto err;
- }
-
- if (BN_ucmp(f, rsa->n) >= 0) {
- /* usually the padding functions would catch this */
- OPENSSL_PUT_ERROR(RSA, sign_raw, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
- goto err;
- }
-
- if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) {
- blinding = rsa_blinding_get(rsa, &blinding_index, ctx);
- if (blinding == NULL) {
+ if (!RSA_private_transform(rsa, out, buf, rsa_size)) {
OPENSSL_PUT_ERROR(RSA, sign_raw, ERR_R_INTERNAL_ERROR);
goto err;
- }
- if (!BN_BLINDING_convert_ex(f, NULL, blinding, ctx)) {
- goto err;
- }
- }
-
- if ((rsa->flags & RSA_FLAG_EXT_PKEY) ||
- ((rsa->p != NULL) && (rsa->q != NULL) && (rsa->dmp1 != NULL) &&
- (rsa->dmq1 != NULL) && (rsa->iqmp != NULL))) {
- if (!rsa->meth->mod_exp(result, f, rsa, ctx)) {
- goto err;
- }
- } else {
- BIGNUM local_d;
- BIGNUM *d = NULL;
-
- BN_init(&local_d);
- d = &local_d;
- BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
-
- if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) {
- if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n,
- ctx)) {
- goto err;
- }
- }
-
- if (!rsa->meth->bn_mod_exp(result, f, d, rsa->n, ctx, rsa->_method_mod_n)) {
- goto err;
- }
- }
-
- if (blinding) {
- if (!BN_BLINDING_invert_ex(result, NULL, blinding, ctx)) {
- goto err;
- }
- }
-
- if (!BN_bn2bin_padded(out, rsa_size, result)) {
- OPENSSL_PUT_ERROR(RSA, sign_raw, ERR_R_INTERNAL_ERROR);
- goto err;
}
*out_len = rsa_size;
ret = 1;
err:
- if (ctx != NULL) {
- BN_CTX_end(ctx);
- BN_CTX_free(ctx);
- }
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
- if (blinding != NULL) {
- rsa_blinding_release(rsa, blinding, blinding_index);
- }
return ret;
}
@@ -447,12 +377,8 @@
static int decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding) {
const unsigned rsa_size = RSA_size(rsa);
- BIGNUM *f, *result;
int r = -1;
uint8_t *buf = NULL;
- BN_CTX *ctx = NULL;
- unsigned blinding_index;
- BN_BLINDING *blinding = NULL;
int ret = 0;
if (max_out < rsa_size) {
@@ -460,80 +386,18 @@
return 0;
}
- ctx = BN_CTX_new();
- if (ctx == NULL) {
- goto err;
- }
- BN_CTX_start(ctx);
- f = BN_CTX_get(ctx);
- result = BN_CTX_get(ctx);
buf = OPENSSL_malloc(rsa_size);
- if (!f || !result || !buf) {
+ if (buf == NULL) {
OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_MALLOC_FAILURE);
goto err;
}
- /* This check was for equality but PGP does evil things
- * and chops off the top '0' bytes.
- * TODO(fork): investigate this. */
- if (in_len > rsa_size) {
- OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_DATA_GREATER_THAN_MOD_LEN);
+ if (in_len != rsa_size) {
+ OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN);
goto err;
}
- /* make data into a big number */
- if (BN_bin2bn(in, (int)in_len, f) == NULL) {
- goto err;
- }
-
- if (BN_ucmp(f, rsa->n) >= 0) {
- OPENSSL_PUT_ERROR(RSA, decrypt, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
- goto err;
- }
-
- if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) {
- blinding = rsa_blinding_get(rsa, &blinding_index, ctx);
- if (blinding == NULL) {
- OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_INTERNAL_ERROR);
- goto err;
- }
- if (!BN_BLINDING_convert_ex(f, NULL, blinding, ctx)) {
- goto err;
- }
- }
-
- /* do the decrypt */
- if ((rsa->flags & RSA_FLAG_EXT_PKEY) ||
- ((rsa->p != NULL) && (rsa->q != NULL) && (rsa->dmp1 != NULL) &&
- (rsa->dmq1 != NULL) && (rsa->iqmp != NULL))) {
- if (!rsa->meth->mod_exp(result, f, rsa, ctx)) {
- goto err;
- }
- } else {
- BIGNUM local_d;
- BIGNUM *d = NULL;
-
- d = &local_d;
- BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
-
- if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) {
- if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n,
- ctx)) {
- goto err;
- }
- }
- if (!rsa->meth->bn_mod_exp(result, f, d, rsa->n, ctx, rsa->_method_mod_n)) {
- goto err;
- }
- }
-
- if (blinding) {
- if (!BN_BLINDING_invert_ex(result, NULL, blinding, ctx)) {
- goto err;
- }
- }
-
- if (!BN_bn2bin_padded(buf, rsa_size, result)) {
+ if (!RSA_private_transform(rsa, buf, in, rsa_size)) {
OPENSSL_PUT_ERROR(RSA, decrypt, ERR_R_INTERNAL_ERROR);
goto err;
}
@@ -563,17 +427,10 @@
}
err:
- if (ctx != NULL) {
- BN_CTX_end(ctx);
- BN_CTX_free(ctx);
- }
if (buf != NULL) {
OPENSSL_cleanse(buf, rsa_size);
OPENSSL_free(buf);
}
- if (blinding != NULL) {
- rsa_blinding_release(rsa, blinding, blinding_index);
- }
return ret;
}
@@ -623,11 +480,8 @@
goto err;
}
- /* This check was for equality but PGP does evil things
- * and chops off the top '0' bytes.
- * TODO(fork): investigate */
- if (in_len > rsa_size) {
- OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_DATA_GREATER_THAN_MOD_LEN);
+ if (in_len != rsa_size) {
+ OPENSSL_PUT_ERROR(RSA, verify_raw, RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN);
goto err;
}
@@ -688,6 +542,99 @@
return ret;
}
+static int private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
+ size_t len) {
+ BIGNUM *f, *result;
+ BN_CTX *ctx = NULL;
+ unsigned blinding_index = 0;
+ BN_BLINDING *blinding = NULL;
+ int ret = 0;
+
+ ctx = BN_CTX_new();
+ if (ctx == NULL) {
+ goto err;
+ }
+ BN_CTX_start(ctx);
+ f = BN_CTX_get(ctx);
+ result = BN_CTX_get(ctx);
+
+ if (f == NULL || result == NULL) {
+ OPENSSL_PUT_ERROR(RSA, private_transform, ERR_R_MALLOC_FAILURE);
+ goto err;
+ }
+
+ if (BN_bin2bn(in, len, f) == NULL) {
+ goto err;
+ }
+
+ if (BN_ucmp(f, rsa->n) >= 0) {
+ /* Usually the padding functions would catch this. */
+ OPENSSL_PUT_ERROR(RSA, private_transform, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
+ goto err;
+ }
+
+ if (!(rsa->flags & RSA_FLAG_NO_BLINDING)) {
+ blinding = rsa_blinding_get(rsa, &blinding_index, ctx);
+ if (blinding == NULL) {
+ OPENSSL_PUT_ERROR(RSA, private_transform, ERR_R_INTERNAL_ERROR);
+ goto err;
+ }
+ if (!BN_BLINDING_convert_ex(f, NULL, blinding, ctx)) {
+ goto err;
+ }
+ }
+
+ if ((rsa->flags & RSA_FLAG_EXT_PKEY) ||
+ ((rsa->p != NULL) && (rsa->q != NULL) && (rsa->dmp1 != NULL) &&
+ (rsa->dmq1 != NULL) && (rsa->iqmp != NULL))) {
+ if (!rsa->meth->mod_exp(result, f, rsa, ctx)) {
+ goto err;
+ }
+ } else {
+ BIGNUM local_d;
+ BIGNUM *d = NULL;
+
+ BN_init(&local_d);
+ d = &local_d;
+ BN_with_flags(d, rsa->d, BN_FLG_CONSTTIME);
+
+ if (rsa->flags & RSA_FLAG_CACHE_PUBLIC) {
+ if (!BN_MONT_CTX_set_locked(&rsa->_method_mod_n, CRYPTO_LOCK_RSA, rsa->n,
+ ctx)) {
+ goto err;
+ }
+ }
+
+ if (!rsa->meth->bn_mod_exp(result, f, d, rsa->n, ctx, rsa->_method_mod_n)) {
+ goto err;
+ }
+ }
+
+ if (blinding) {
+ if (!BN_BLINDING_invert_ex(result, NULL, blinding, ctx)) {
+ goto err;
+ }
+ }
+
+ if (!BN_bn2bin_padded(out, len, result)) {
+ OPENSSL_PUT_ERROR(RSA, private_transform, ERR_R_INTERNAL_ERROR);
+ goto err;
+ }
+
+ ret = 1;
+
+err:
+ if (ctx != NULL) {
+ BN_CTX_end(ctx);
+ BN_CTX_free(ctx);
+ }
+ if (blinding != NULL) {
+ rsa_blinding_release(rsa, blinding, blinding_index);
+ }
+
+ return ret;
+}
+
static int mod_exp(BIGNUM *r0, const BIGNUM *I, RSA *rsa, BN_CTX *ctx) {
BIGNUM *r1, *m1, *vrfy;
BIGNUM local_dmp1, local_dmq1, local_c, local_r1;
@@ -1000,6 +947,8 @@
decrypt,
verify_raw,
+ private_transform,
+
mod_exp /* mod_exp */,
BN_mod_exp_mont /* bn_mod_exp */,
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index 9827e69..bd0b0d1 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -386,6 +386,21 @@
int (*verify_raw)(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
const uint8_t *in, size_t in_len, int padding);
+ /* private_transform takes a big-endian integer from |in|, calculates the
+ * d'th power of it, modulo the RSA modulus and writes the result as a
+ * big-endian integer to |out|. Both |in| and |out| are |len| bytes long and
+ * |len| is always equal to |RSA_size(rsa)|. If the result of the transform
+ * can be represented in fewer than |len| bytes, then |out| must be zero
+ * padded on the left.
+ *
+ * It returns one on success and zero otherwise.
+ *
+ * RSA decrypt and sign operations will call this, thus an ENGINE might wish
+ * to override it in order to avoid having to implement the padding
+ * functionality demanded by those, higher level, operations. */
+ int (*private_transform)(RSA *rsa, uint8_t *out, const uint8_t *in,
+ size_t len);
+
int (*mod_exp)(BIGNUM *r0, const BIGNUM *I, RSA *rsa,
BN_CTX *ctx); /* Can be null */
int (*bn_mod_exp)(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
@@ -470,8 +485,8 @@
#define RSA_F_RSA_padding_check_PKCS1_type_2 126
#define RSA_F_RSA_recover_crt_params 127
#define RSA_F_RSA_check_key 128
+#define RSA_F_private_transform 129
#define RSA_R_INVALID_MESSAGE_LENGTH 100
-#define RSA_R_DATA_GREATER_THAN_MOD_LEN 101
#define RSA_R_NO_PUBLIC_EXPONENT 102
#define RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE 103
#define RSA_R_BLOCK_TYPE_IS_NOT_01 104
@@ -514,5 +529,6 @@
#define RSA_R_CRT_VALUES_INCORRECT 141
#define RSA_R_INCONSISTENT_SET_OF_CRT_VALUES 142
#define RSA_R_ONLY_ONE_OF_P_Q_GIVEN 143
+#define RSA_R_DATA_LEN_NOT_EQUAL_TO_MOD_LEN 144
#endif /* OPENSSL_HEADER_RSA_H */
