acvp: RSA signature verification tests.

Change-Id: I8697230d4feb3bc5308905aa8981087b0f080555
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/43626
Commit-Queue: Adam Langley <agl@google.com>
Reviewed-by: David Benjamin <davidben@google.com>
diff --git a/util/fipstools/acvp/acvptool/subprocess/rsa.go b/util/fipstools/acvp/acvptool/subprocess/rsa.go
index 0825c04..d2de73d 100644
--- a/util/fipstools/acvp/acvptool/subprocess/rsa.go
+++ b/util/fipstools/acvp/acvptool/subprocess/rsa.go
@@ -87,6 +87,36 @@
 	Sig string `json:"signature"`
 }
 
+type rsaSigVerTestVectorSet struct {
+	Groups []rsaSigVerGroup `json:"testGroups"`
+}
+
+type rsaSigVerGroup struct {
+	ID      uint64          `json:"tgId"`
+	Type    string          `json:"testType"`
+	SigType string          `json:"sigType"`
+	Hash    string          `json:"hashAlg"`
+	N       string          `json:"n"`
+	E       string          `json:"e"`
+	Tests   []rsaSigVerTest `json:"tests"`
+}
+
+type rsaSigVerTest struct {
+	ID           uint64 `json:"tcId"`
+	MessageHex   string `json:"message"`
+	SignatureHex string `json:"signature"`
+}
+
+type rsaSigVerTestGroupResponse struct {
+	ID    uint64                  `json:"tgId"`
+	Tests []rsaSigVerTestResponse `json:"tests"`
+}
+
+type rsaSigVerTestResponse struct {
+	ID     uint64 `json:"tcId"`
+	Passed bool   `json:"testPassed"`
+}
+
 func processKeyGen(vectorSet []byte, m Transactable) (interface{}, error) {
 	var parsed rsaKeyGenTestVectorSet
 	if err := json.Unmarshal(vectorSet, &parsed); err != nil {
@@ -179,6 +209,63 @@
 	return ret, nil
 }
 
+func processSigVer(vectorSet []byte, m Transactable) (interface{}, error) {
+	var parsed rsaSigVerTestVectorSet
+	if err := json.Unmarshal(vectorSet, &parsed); err != nil {
+		return nil, err
+	}
+
+	var ret []rsaSigVerTestGroupResponse
+
+	for _, group := range parsed.Groups {
+		// GDT means "Generated data test", which makes no sense in this context.
+		const expectedType = "GDT"
+		if group.Type != expectedType {
+			return nil, fmt.Errorf("RSA SigVer test group has type %q, but only 'generation' tests (%q) are supported", group.Type, expectedType)
+		}
+
+		n, err := hex.DecodeString(group.N)
+		if err != nil {
+			return nil, fmt.Errorf("test group %d contains invalid hex: %s", group.ID, err)
+		}
+		e, err := hex.DecodeString(group.E)
+		if err != nil {
+			return nil, fmt.Errorf("test group %d contains invalid hex: %s", group.ID, err)
+		}
+
+		response := rsaSigVerTestGroupResponse{
+			ID: group.ID,
+		}
+
+		operation := "RSA/sigVer/" + group.Hash + "/" + group.SigType
+
+		for _, test := range group.Tests {
+			msg, err := hex.DecodeString(test.MessageHex)
+			if err != nil {
+				return nil, fmt.Errorf("test case %d/%d contains invalid hex: %s", group.ID, test.ID, err)
+			}
+			sig, err := hex.DecodeString(test.SignatureHex)
+			if err != nil {
+				return nil, fmt.Errorf("test case %d/%d contains invalid hex: %s", group.ID, test.ID, err)
+			}
+
+			results, err := m.Transact(operation, 1, n, e, msg, sig)
+			if err != nil {
+				return nil, err
+			}
+
+			response.Tests = append(response.Tests, rsaSigVerTestResponse{
+				ID:     test.ID,
+				Passed: len(results[0]) == 1 && results[0][0] == 1,
+			})
+		}
+
+		ret = append(ret, response)
+	}
+
+	return ret, nil
+}
+
 type rsa struct{}
 
 func (*rsa) Process(vectorSet []byte, m Transactable) (interface{}, error) {
@@ -192,6 +279,8 @@
 		return processKeyGen(vectorSet, m)
 	case "sigGen":
 		return processSigGen(vectorSet, m)
+	case "sigVer":
+		return processSigVer(vectorSet, m)
 	default:
 		return nil, fmt.Errorf("Unknown RSA mode %q", parsed.Mode)
 	}
diff --git a/util/fipstools/acvp/modulewrapper/modulewrapper.cc b/util/fipstools/acvp/modulewrapper/modulewrapper.cc
index aad2a3c..8043497 100644
--- a/util/fipstools/acvp/modulewrapper/modulewrapper.cc
+++ b/util/fipstools/acvp/modulewrapper/modulewrapper.cc
@@ -33,6 +33,7 @@
 #include <openssl/ec.h>
 #include <openssl/ec_key.h>
 #include <openssl/ecdsa.h>
+#include <openssl/err.h>
 #include <openssl/hmac.h>
 #include <openssl/obj.h>
 #include <openssl/rsa.h>
@@ -543,6 +544,141 @@
         }]
       },
       {
+        "algorithm": "RSA",
+        "mode": "sigVer",
+        "revision": "FIPS186-4",
+        "pubExpMode": "fixed",
+        "fixedPubExp": "010001",
+        "capabilities": [{
+          "sigType": "pkcs1v1.5",
+          "properties": [{
+            "modulo": 1024,
+            "hashPair": [{
+              "hashAlg": "SHA2-224"
+            }, {
+              "hashAlg": "SHA2-256"
+            }, {
+              "hashAlg": "SHA2-384"
+            }, {
+              "hashAlg": "SHA2-512"
+            }, {
+              "hashAlg": "SHA-1"
+            }]
+          }]
+        },{
+          "sigType": "pkcs1v1.5",
+          "properties": [{
+            "modulo": 2048,
+            "hashPair": [{
+              "hashAlg": "SHA2-224"
+            }, {
+              "hashAlg": "SHA2-256"
+            }, {
+              "hashAlg": "SHA2-384"
+            }, {
+              "hashAlg": "SHA2-512"
+            }, {
+              "hashAlg": "SHA-1"
+            }]
+          }]
+        },{
+          "sigType": "pkcs1v1.5",
+          "properties": [{
+            "modulo": 3072,
+            "hashPair": [{
+              "hashAlg": "SHA2-224"
+            }, {
+              "hashAlg": "SHA2-256"
+            }, {
+              "hashAlg": "SHA2-384"
+            }, {
+              "hashAlg": "SHA2-512"
+            }, {
+              "hashAlg": "SHA-1"
+            }]
+          }]
+        },{
+          "sigType": "pkcs1v1.5",
+          "properties": [{
+            "modulo": 4096,
+            "hashPair": [{
+              "hashAlg": "SHA2-224"
+            }, {
+              "hashAlg": "SHA2-256"
+            }, {
+              "hashAlg": "SHA2-384"
+            }, {
+              "hashAlg": "SHA2-512"
+            }, {
+              "hashAlg": "SHA-1"
+            }]
+          }]
+        },{
+          "sigType": "pss",
+          "properties": [{
+            "modulo": 2048,
+            "hashPair": [{
+              "hashAlg": "SHA2-224",
+              "saltLen": 28
+            }, {
+              "hashAlg": "SHA2-256",
+              "saltLen": 32
+            }, {
+              "hashAlg": "SHA2-384",
+              "saltLen": 48
+            }, {
+              "hashAlg": "SHA2-512",
+              "saltLen": 64
+            }, {
+              "hashAlg": "SHA-1",
+              "saltLen": 20
+            }]
+          }]
+        },{
+          "sigType": "pss",
+          "properties": [{
+            "modulo": 3072,
+            "hashPair": [{
+              "hashAlg": "SHA2-224",
+              "saltLen": 28
+            }, {
+              "hashAlg": "SHA2-256",
+              "saltLen": 32
+            }, {
+              "hashAlg": "SHA2-384",
+              "saltLen": 48
+            }, {
+              "hashAlg": "SHA2-512",
+              "saltLen": 64
+            }, {
+              "hashAlg": "SHA-1",
+              "saltLen": 20
+            }]
+          }]
+        },{
+          "sigType": "pss",
+          "properties": [{
+            "modulo": 4096,
+            "hashPair": [{
+              "hashAlg": "SHA2-224",
+              "saltLen": 28
+            }, {
+              "hashAlg": "SHA2-256",
+              "saltLen": 32
+            }, {
+              "hashAlg": "SHA2-384",
+              "saltLen": 48
+            }, {
+              "hashAlg": "SHA2-512",
+              "saltLen": 64
+            }, {
+              "hashAlg": "SHA-1",
+              "saltLen": 20
+            }]
+          }]
+        }]
+      },
+      {
         "algorithm": "CMAC-AES",
         "revision": "1.0",
         "capabilities": [{
@@ -1238,6 +1374,42 @@
                     BIGNUMBytes(RSA_get0_e(key)), sig);
 }
 
+template<const EVP_MD *(MDFunc)(), bool UsePSS>
+static bool RSASigVer(const Span<const uint8_t> args[]) {
+  const Span<const uint8_t> n_bytes = args[0];
+  const Span<const uint8_t> e_bytes = args[1];
+  const Span<const uint8_t> msg = args[2];
+  const Span<const uint8_t> sig = args[3];
+
+  BIGNUM *n = BN_new();
+  BIGNUM *e = BN_new();
+  bssl::UniquePtr<RSA> key(RSA_new());
+  if (!BN_bin2bn(n_bytes.data(), n_bytes.size(), n) ||
+      !BN_bin2bn(e_bytes.data(), e_bytes.size(), e) ||
+      !RSA_set0_key(key.get(), n, e, /*d=*/nullptr)) {
+    return false;
+  }
+
+  const EVP_MD *const md = MDFunc();
+  uint8_t digest_buf[EVP_MAX_MD_SIZE];
+  unsigned digest_len;
+  if (!EVP_Digest(msg.data(), msg.size(), digest_buf, &digest_len, md, NULL)) {
+    return false;
+  }
+
+  uint8_t ok;
+  if (UsePSS) {
+    ok = RSA_verify_pss_mgf1(key.get(), digest_buf, digest_len, md, md, -1,
+                             sig.data(), sig.size());
+  } else {
+    ok = RSA_verify(EVP_MD_type(md), digest_buf, digest_len, sig.data(),
+                    sig.size(), key.get());
+  }
+  ERR_clear_error();
+
+  return WriteReply(STDOUT_FILENO, Span<const uint8_t>(&ok, 1));
+}
+
 static constexpr struct {
   const char name[kMaxNameLength + 1];
   uint8_t expected_args;
@@ -1289,6 +1461,16 @@
     {"RSA/sigGen/SHA2-384/pss", 2, RSASigGen<EVP_sha384, true>},
     {"RSA/sigGen/SHA2-512/pss", 2, RSASigGen<EVP_sha512, true>},
     {"RSA/sigGen/SHA-1/pss", 2, RSASigGen<EVP_sha1, true>},
+    {"RSA/sigVer/SHA2-224/pkcs1v1.5", 4, RSASigVer<EVP_sha224, false>},
+    {"RSA/sigVer/SHA2-256/pkcs1v1.5", 4, RSASigVer<EVP_sha256, false>},
+    {"RSA/sigVer/SHA2-384/pkcs1v1.5", 4, RSASigVer<EVP_sha384, false>},
+    {"RSA/sigVer/SHA2-512/pkcs1v1.5", 4, RSASigVer<EVP_sha512, false>},
+    {"RSA/sigVer/SHA-1/pkcs1v1.5", 4, RSASigVer<EVP_sha1, false>},
+    {"RSA/sigVer/SHA2-224/pss", 4, RSASigVer<EVP_sha224, true>},
+    {"RSA/sigVer/SHA2-256/pss", 4, RSASigVer<EVP_sha256, true>},
+    {"RSA/sigVer/SHA2-384/pss", 4, RSASigVer<EVP_sha384, true>},
+    {"RSA/sigVer/SHA2-512/pss", 4, RSASigVer<EVP_sha512, true>},
+    {"RSA/sigVer/SHA-1/pss", 4, RSASigVer<EVP_sha1, true>},
 };
 
 int main() {