EVP_KEM: Add an optional check_key hook EVP_KEM matches keys on pkey_id, but that does not always determine whether a key is usable with a KEM, such as for EC curves which all share the same pkey_id. Add an optional check_key hook to EVP_KEM. No EVP_KEM sets it yet. Bug: 535883377 Change-Id: Idf4b8a362ed6d9f05d146069b4ca05a16a6a6964 Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/103467 Commit-Queue: Lily Chen <chlily@google.com> Reviewed-by: David Benjamin <davidben@google.com>

diff --git a/crypto/evp/evp_kem.cc b/crypto/evp/evp_kem.cc
index 328a998..e718caf 100644
--- a/crypto/evp/evp_kem.cc
+++ b/crypto/evp/evp_kem.cc

@@ -36,7 +36,8 @@
     OPENSSL_PUT_ERROR(EVP, ERR_R_PASSED_NULL_PARAMETER);
     return false;
   }
-  if (kem->pkey_id != EVP_PKEY_id(pkey_impl)) {
+  if (kem->pkey_id != EVP_PKEY_id(pkey_impl) ||
+      (kem->check_key != nullptr && !kem->check_key(kem, pkey_impl))) {
     OPENSSL_PUT_ERROR(EVP, EVP_R_UNSUPPORTED_ALGORITHM);
     return false;
   }

diff --git a/crypto/evp/internal.h b/crypto/evp/internal.h
index e498fad..d4e57dc 100644
--- a/crypto/evp/internal.h
+++ b/crypto/evp/internal.h

@@ -317,6 +317,10 @@
   // Fixed length of external entropy for testing.
   size_t entropy_len;
 
+  // check_key, if non-null, returns whether `key`, whose type matches
+  // `pkey_id`, is usable with this KEM.
+  bool (*check_key)(const EVP_KEM *kem, const EVP_PKEY *key);
+
   int (*encap)(const EVP_KEM *kem, bssl::Span<uint8_t> out_ciphertext,
                bssl::Span<uint8_t> out_secret, const EVP_PKEY *peer_key);
   int (*encap_external_entropy)(const EVP_KEM *kem,

diff --git a/crypto/evp/p_mlkem.cc b/crypto/evp/p_mlkem.cc
index 6881109..dd8710c 100644
--- a/crypto/evp/p_mlkem.cc
+++ b/crypto/evp/p_mlkem.cc

@@ -414,6 +414,7 @@
       /*ciphertext_len=*/Traits::kCiphertextBytes,
       /*secret_len=*/MLKEM_SHARED_SECRET_BYTES,
       /*entropy_len=*/BCM_MLKEM_ENCAP_ENTROPY,
+      /*check_key=*/nullptr,
       &KemEncap,
       &KemEncapExternalEntropy,
       &KemDecap,

diff --git a/crypto/evp/p_xwing.cc b/crypto/evp/p_xwing.cc
index 750c94c..7d9362c 100644
--- a/crypto/evp/p_xwing.cc
+++ b/crypto/evp/p_xwing.cc

@@ -224,6 +224,7 @@
     XWING_CIPHERTEXT_BYTES,             //
     XWING_SHARED_SECRET_BYTES,          //
     kXwingEncapEntropyBytes,            //
+    /*check_key=*/nullptr,              //
     &xwing_kem_encap,                   //
     &xwing_kem_encap_external_entropy,  //
     &xwing_kem_decap,                   //