Test some obscure modes of EVP sign/verify with RSA keys

When the digest is unset, padding may be either RSA_PADDING_NONE or
RSA_PADDING_PKCS1.

If RSA_PADDING_NONE, this becomes raw RSA public and private key
operations, with signature verify comparing the "digest" against the
output of the raw public key operation.

If RSA_PADDING_PKCS1, this treats the "digest" as the raw DigestInfo
structure.

Test both of these, so we don't break them as we move code around. In
doing so, this revealed that verify in these modes, when the "digest"
doesn't match, forgot to add to the error queue. Fix that up.

Bug: 42290606
Change-Id: I3412a633124a12bda6dfebc08896f616b2d268aa
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/75228
Reviewed-by: Bob Beck <bbe@google.com>
Auto-Submit: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>

crypto/evp/evp_tests.txt

diff --git a/crypto/evp/evp_tests.txt b/crypto/evp/evp_tests.txt
index e8e9f27..e1ec21d 100644
--- a/crypto/evp/evp_tests.txt
+++ b/crypto/evp/evp_tests.txt
@@ -1551,6 +1551,62 @@
 Output = eaf1a73a1b0c4609537de69cd9228bbcfb9a8ca8c6c3efaf056fe4a7f4634ed00b7c39ec6922d7b8ea2c04ebac
 
 
+# RSA with no padding implements raw public and private transforms. This is not
+# a real signature scheme, but might be used to construct one.
+
+Sign = RSA-2048
+RSAPadding = None
+Input = 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+
+Verify = RSA-2048
+RSAPadding = None
+Input = 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+
+Verify = RSA-2048
+RSAPadding = None
+Input = 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+Error = BAD_SIGNATURE
+
+Sign = RSA-2048
+RSAPadding = None
+Input = "Too short"
+Error = DATA_TOO_SMALL
+
+Verify = RSA-2048
+RSAPadding = None
+Input = "Too short"
+Output = 93d0bae8ad0d94de400eb078dd10edd7418ef1bf11b8e8b5d2b86b142e77d603e108fbcca2b976aa7b5326e5369db3bb73bf74f8d47c36a6318e913888c873502a561fc69329e7c24a0a016d81310449a52b29e49a6a41bdfe6c10a8d90072d64b4486756fd007c0071da2a8c7107a904621c11f0d81aa80b655a713c28170594ece28133dfbfddd61d4e4dad0d6781f6145a351a994054993fd57cd1330966ce97d7ac259b15616fd7235e2cac29fdc1c05f1612c61785614b80e7b650c03ef77d64163d75fa637cc2a9a7e570b3176fdcfb6ad6d25e8515f6ced02cfb3a441c87220044110fd27dcb53888f0377e1797bf297b7da27d3f033cd8b5d60ececc
+Error = BAD_SIGNATURE
+
+Sign = RSA-2048
+Digest = SHA256
+RSAPadding = None
+Input = 00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002
+Error = INVALID_PADDING_MODE
+
+
+# RSASSA-PKCS1-v1_5 with no digest. This is not actually a defined mode in
+# PKCS #1, but OpenSSL's API interprets this to mean accepting an arbitrary
+# string instead of the serialized DigestInfo. This is not a real signature
+# scheme, but might be used to construct one.
+
+Sign = RSA-2048
+Input = 01234567
+Output = 9925a048eb9e225819dea311be206ea1287ad90bdf77218bb1348829ef7072e7b4a772b6cb1df1480d4ede4140feb84e03c4c99c851f91cf293ed3259a6ef8f567130164465a39e83306b69a22b74ec26adca77da8e1cc625c3721269b410819298f1c6399eb8c68bcc8df16ab149eae0959947a5918b4fd49396af80b557124712f27f760cc69d09d6d1db0883decf05ee072e98f15b153bd74e2ec98a00ea514a4c9c2a98a0813cb5b53381e030779612e3de972f8029cf363bec7098ecda568eee6371ecb10f6448e707e08b47f557a9f7e87024a2a1249d1f8dc540b35a79f9e6c0188cbc406cc434cefd83538c3cffb7663f9e46a7787a6a630c5a1c675
+
+Verify = RSA-2048
+Input = 01234567
+Output = 9925a048eb9e225819dea311be206ea1287ad90bdf77218bb1348829ef7072e7b4a772b6cb1df1480d4ede4140feb84e03c4c99c851f91cf293ed3259a6ef8f567130164465a39e83306b69a22b74ec26adca77da8e1cc625c3721269b410819298f1c6399eb8c68bcc8df16ab149eae0959947a5918b4fd49396af80b557124712f27f760cc69d09d6d1db0883decf05ee072e98f15b153bd74e2ec98a00ea514a4c9c2a98a0813cb5b53381e030779612e3de972f8029cf363bec7098ecda568eee6371ecb10f6448e707e08b47f557a9f7e87024a2a1249d1f8dc540b35a79f9e6c0188cbc406cc434cefd83538c3cffb7663f9e46a7787a6a630c5a1c675
+
+Verify = RSA-2048
+Input = 01234568
+Output = 9925a048eb9e225819dea311be206ea1287ad90bdf77218bb1348829ef7072e7b4a772b6cb1df1480d4ede4140feb84e03c4c99c851f91cf293ed3259a6ef8f567130164465a39e83306b69a22b74ec26adca77da8e1cc625c3721269b410819298f1c6399eb8c68bcc8df16ab149eae0959947a5918b4fd49396af80b557124712f27f760cc69d09d6d1db0883decf05ee072e98f15b153bd74e2ec98a00ea514a4c9c2a98a0813cb5b53381e030779612e3de972f8029cf363bec7098ecda568eee6371ecb10f6448e707e08b47f557a9f7e87024a2a1249d1f8dc540b35a79f9e6c0188cbc406cc434cefd83538c3cffb7663f9e46a7787a6a630c5a1c675
+Error = BAD_SIGNATURE
+
+
 # Single-shot signing tests.
 
 SignMessage = RSA-2048

crypto/evp/p_rsa.cc

diff --git a/crypto/evp/p_rsa.cc b/crypto/evp/p_rsa.cc
index d7504f3..c4b0fbb 100644
--- a/crypto/evp/p_rsa.cc
+++ b/crypto/evp/p_rsa.cc
@@ -184,8 +184,11 @@
   const size_t key_len = EVP_PKEY_size(ctx->pkey);
   if (!setup_tbuf(rctx, ctx) ||
       !RSA_verify_raw(rsa, &rslen, rctx->tbuf, key_len, sig, siglen,
-                      rctx->pad_mode) ||
-      rslen != tbslen || CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
+                      rctx->pad_mode)) {
+    return 0;
+  }
+  if (rslen != tbslen || CRYPTO_memcmp(tbs, rctx->tbuf, rslen) != 0) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_BAD_SIGNATURE);
     return 0;
   }