commit 0c0b7e1e1f35c23ecaed10fb262403f1b3cf5882
author David Benjamin <davidben@google.com> Thu Jul 14 13:47:55 2016 -0400
committer David Benjamin <davidben@google.com> Fri Jul 15 18:17:32 2016 +0000
tree 3828c8d92630b11d94e6f35498b368afc997e8c8
parent ca3d545d7f34d67c8fb05265a0f1cf7f88076776

Widen SSL_PRIVATE_KEY_METHOD types to include the curve name.

This makes custom private keys and EVP_PKEYs symmetric again. There is
no longer a requirement that the caller pre-filter the configured
signing prefs.

Also switch EVP_PKEY_RSA to NID_rsaEncryption. These are identical, but
if some key types are to be NIDs, we should make them all NIDs.

Change-Id: I82ea41c27a3c57f4c4401ffe1ccad406783e4c64
Reviewed-on: https://boringssl-review.googlesource.com/8785
Reviewed-by: David Benjamin <davidben@google.com>

ssl/t1_lib.c

diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index 4594649..9761dc9 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -2596,12 +2596,17 @@
   /* Before TLS 1.2, the signature algorithm isn't negotiated as part of the
    * handshake. It is fixed at MD5-SHA1 for RSA and SHA1 for ECDSA. */
   if (ssl3_protocol_version(ssl) < TLS1_2_VERSION) {
-    if (ssl_private_key_type(ssl) == EVP_PKEY_RSA) {
+    int type = ssl_private_key_type(ssl);
+    if (type == NID_rsaEncryption) {
       *out = SSL_SIGN_RSA_PKCS1_MD5_SHA1;
-    } else {
-      *out = SSL_SIGN_ECDSA_SHA1;
+      return 1;
     }
-    return 1;
+    if (ssl_is_ecdsa_key_type(type)) {
+      *out = SSL_SIGN_ECDSA_SHA1;
+      return 1;
+    }
+    OPENSSL_PUT_ERROR(SSL, SSL_R_NO_COMMON_SIGNATURE_ALGORITHMS);
+    return 0;
   }
 
   const uint16_t *sigalgs;