Make X509_verify X509_sign_ctx work with EVP_PKEY_RSA_PSS

Also test that putting an EVP_PKEY_RSA_PSS key in a certificate works...
mostly. X509_get0_pubkey on the resulting object should work but does
not yet. Later work will fix this.

This code also should be reworked to not depend on the big OID table,
but I've left that as a TODO for now.

Bug: 42290364, 384818542
Change-Id: Ifd727cb06ed6c9d1501ef2ef155f4b3c66c8d488
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/81782
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/build.json b/build.json
index 39ba40e..757b6cb 100644
--- a/build.json
+++ b/build.json
@@ -947,6 +947,7 @@
             "crypto/slhdsa/slhdsa_siggen.txt",
             "crypto/slhdsa/slhdsa_sigver.txt",
             "crypto/x509/test/*.pem",
+            "crypto/x509/test/*.pk8",
             "third_party/wycheproof_testvectors/*.txt"
         ]
     },
diff --git a/crypto/x509/algorithm.cc b/crypto/x509/algorithm.cc
index 5e86014..55c3eaf 100644
--- a/crypto/x509/algorithm.cc
+++ b/crypto/x509/algorithm.cc
@@ -22,6 +22,12 @@
 
 #include "internal.h"
 
+
+// TODO(crbug.com/42290422): Rewrite this logic to recognize signature
+// algorithms without pulling in the OID table. We can enumerate every supported
+// signature algorithm into a small enum and convert them to/from |EVP_PKEY_CTX|
+// and |X509_ALGOR|.
+
 // Restrict the digests that are allowed in X509 certificates
 static int x509_digest_nid_ok(const int digest_nid) {
   switch (digest_nid) {
@@ -39,7 +45,8 @@
     return 0;
   }
 
-  if (EVP_PKEY_id(pkey) == EVP_PKEY_RSA) {
+  if (EVP_PKEY_id(pkey) == EVP_PKEY_RSA ||
+      EVP_PKEY_id(pkey) == EVP_PKEY_RSA_PSS) {
     int pad_mode;
     if (!EVP_PKEY_CTX_get_rsa_padding(ctx->pctx, &pad_mode)) {
       return 0;
@@ -88,7 +95,10 @@
   }
 
   // Check the public key OID matches the public key type.
-  if (pkey_nid != EVP_PKEY_id(pkey)) {
+  const bool pkey_matches =
+      pkey_nid == EVP_PKEY_id(pkey) ||
+      (sigalg_nid == NID_rsassaPss && EVP_PKEY_id(pkey) == EVP_PKEY_RSA_PSS);
+  if (!pkey_matches) {
     OPENSSL_PUT_ERROR(ASN1, ASN1_R_WRONG_PUBLIC_KEY_TYPE);
     return 0;
   }
diff --git a/crypto/x509/test/rsa_pss_sha256_key.pk8 b/crypto/x509/test/rsa_pss_sha256_key.pk8
new file mode 100644
index 0000000..9bb598d
--- /dev/null
+++ b/crypto/x509/test/rsa_pss_sha256_key.pk8
Binary files differ
diff --git a/crypto/x509/x509_test.cc b/crypto/x509/x509_test.cc
index c2eb764..4e3a72b 100644
--- a/crypto/x509/x509_test.cc
+++ b/crypto/x509/x509_test.cc
@@ -8894,4 +8894,49 @@
   EXPECT_TRUE(ok);
 }
 
+TEST(X509Test, NonDefaultKeyType) {
+  // Parse an RSA-PSS key. This key type is not enabled by default.
+  std::string pkcs8_str =
+      GetTestData("crypto/x509/test/rsa_pss_sha256_key.pk8");
+  auto pkcs8 = bssl::StringAsBytes(pkcs8_str);
+  const EVP_PKEY_ALG *const alg = EVP_pkey_rsa_pss_sha256();
+  bssl::UniquePtr<EVP_PKEY> pkey(
+      EVP_PKEY_from_private_key_info(pkcs8.data(), pkcs8.size(), &alg, 1));
+  ASSERT_TRUE(pkey);
+  EXPECT_EQ(EVP_PKEY_id(pkey.get()), EVP_PKEY_RSA_PSS);
+
+  // It should be possible to use |pkey| to make a certificate.
+  bssl::UniquePtr<X509> cert =
+      MakeTestCert("Test Issuer", "Test Subject", pkey.get(), /*is_ca=*/false);
+  ASSERT_TRUE(cert);
+  ASSERT_TRUE(X509_sign(cert.get(), pkey.get(), EVP_sha256()));
+
+  // Verify the signature with |pkey|.
+  EXPECT_TRUE(X509_verify(cert.get(), pkey.get()));
+
+#if 1
+  // TODO(crbug.com/42290364): This does not currently work, but it should.
+  EXPECT_FALSE(X509_get0_pubkey(cert.get()));
+#else
+  // The public key can be extracted from |cert|.
+  const EVP_PKEY *cert_pkey = X509_get0_pubkey(cert.get());
+  ASSERT_TRUE(cert_pkey);
+  EXPECT_EQ(EVP_PKEY_cmp(pkey.get(), cert_pkey), 1);
+  // |X509_check_private_key| should work.
+  EXPECT_EQ(X509_check_private_key(cert.get(), pkey.get()), 1);
+#endif
+
+  // The resulting certificate can be serialized and re-parsed.
+  bssl::UniquePtr<X509> reparsed = ReencodeCertificate(cert.get());
+  ASSERT_TRUE(reparsed);
+
+  // RSA-PSS is off by default, so parsing certificates anew with |d2i_X509|
+  // will not enable off-by-default algorithms.
+  EXPECT_FALSE(X509_get0_pubkey(reparsed.get()));
+  EXPECT_EQ(X509_check_private_key(reparsed.get(), pkey.get()), 0);
+
+  // TODO(crbug.com/42290364): Add an API to parse certificates with a custom
+  // algorithm list. That should be able to extract the key.
+}
+
 }  // namespace
diff --git a/gen/sources.bzl b/gen/sources.bzl
index 0bfbe31..edc4dca 100644
--- a/gen/sources.bzl
+++ b/gen/sources.bzl
@@ -994,6 +994,7 @@
     "crypto/x509/test/pss_sha256_wrong_trailer.pem",
     "crypto/x509/test/pss_sha384.pem",
     "crypto/x509/test/pss_sha512.pem",
+    "crypto/x509/test/rsa_pss_sha256_key.pk8",
     "crypto/x509/test/some_names1.pem",
     "crypto/x509/test/some_names2.pem",
     "crypto/x509/test/some_names3.pem",
diff --git a/gen/sources.cmake b/gen/sources.cmake
index 6f24612..0a5e083 100644
--- a/gen/sources.cmake
+++ b/gen/sources.cmake
@@ -1020,6 +1020,7 @@
   crypto/x509/test/pss_sha256_wrong_trailer.pem
   crypto/x509/test/pss_sha384.pem
   crypto/x509/test/pss_sha512.pem
+  crypto/x509/test/rsa_pss_sha256_key.pk8
   crypto/x509/test/some_names1.pem
   crypto/x509/test/some_names2.pem
   crypto/x509/test/some_names3.pem
diff --git a/gen/sources.gni b/gen/sources.gni
index 14dce85..4807120 100644
--- a/gen/sources.gni
+++ b/gen/sources.gni
@@ -994,6 +994,7 @@
   "crypto/x509/test/pss_sha256_wrong_trailer.pem",
   "crypto/x509/test/pss_sha384.pem",
   "crypto/x509/test/pss_sha512.pem",
+  "crypto/x509/test/rsa_pss_sha256_key.pk8",
   "crypto/x509/test/some_names1.pem",
   "crypto/x509/test/some_names2.pem",
   "crypto/x509/test/some_names3.pem",
diff --git a/gen/sources.json b/gen/sources.json
index a3a0569..29be1ce 100644
--- a/gen/sources.json
+++ b/gen/sources.json
@@ -974,6 +974,7 @@
       "crypto/x509/test/pss_sha256_wrong_trailer.pem",
       "crypto/x509/test/pss_sha384.pem",
       "crypto/x509/test/pss_sha512.pem",
+      "crypto/x509/test/rsa_pss_sha256_key.pk8",
       "crypto/x509/test/some_names1.pem",
       "crypto/x509/test/some_names2.pem",
       "crypto/x509/test/some_names3.pem",
diff --git a/gen/sources.mk b/gen/sources.mk
index 5c2c045..de380cf 100644
--- a/gen/sources.mk
+++ b/gen/sources.mk
@@ -982,6 +982,7 @@
   crypto/x509/test/pss_sha256_wrong_trailer.pem \
   crypto/x509/test/pss_sha384.pem \
   crypto/x509/test/pss_sha512.pem \
+  crypto/x509/test/rsa_pss_sha256_key.pk8 \
   crypto/x509/test/some_names1.pem \
   crypto/x509/test/some_names2.pem \
   crypto/x509/test/some_names3.pem \