Don't decompose sigalgs in key preferences.

Instead, in SSL_set_private_key_digest_prefs, convert the NID list to a
sigalgs list. We'll need to add a new API later when custom key callers
are ready to start advertising RSA-PSS.

This removes all callers of tls12_get_hash except inside the signing and
verifying functions.

Change-Id: Ie534f3b736c6ac6ebeb0d7770d489f72e3321865
Reviewed-on: https://boringssl-review.googlesource.com/8693
Reviewed-by: David Benjamin <davidben@google.com>
diff --git a/ssl/internal.h b/ssl/internal.h
index 5281504..03a1320 100644
--- a/ssl/internal.h
+++ b/ssl/internal.h
@@ -778,10 +778,10 @@
   /* peer_sigalgslen is the number of entries in |peer_sigalgs|. */
   size_t peer_sigalgslen;
 
-  /* digest_nids, if non-NULL, is the set of digests supported by |privatekey|
-   * in decreasing order of preference. */
-  int *digest_nids;
-  size_t num_digest_nids;
+  /* sigalgs, if non-NULL, is the set of digests supported by |privatekey| in
+   * decreasing order of preference. */
+  uint16_t *sigalgs;
+  size_t sigalgs_len;
 
   /* Certificate setup callback: if set is called whenever a
    * certificate may be required (client or server). the callback
@@ -1221,10 +1221,6 @@
 
 size_t tls12_get_psigalgs(SSL *ssl, const uint16_t **psigs);
 
-/* tls12_get_hash returns the EVP_MD corresponding to the TLS signature
- * algorithm |sigalg|. It returns NULL if the type is unknown. */
-const EVP_MD *tls12_get_hash(uint16_t sigalg);
-
 /* tls12_check_peer_sigalg checks that |signature_algorithm| is consistent with
  * the |pkey| and |ssl|'s sent, supported signature algorithms and returns 1.
  * Otherwise it returns 0 and writes an alert into |*out_alert|. */
diff --git a/ssl/ssl_cert.c b/ssl/ssl_cert.c
index a361af2..d61f9f5 100644
--- a/ssl/ssl_cert.c
+++ b/ssl/ssl_cert.c
@@ -226,7 +226,7 @@
 
   ssl_cert_clear_certs(c);
   OPENSSL_free(c->peer_sigalgs);
-  OPENSSL_free(c->digest_nids);
+  OPENSSL_free(c->sigalgs);
   X509_STORE_free(c->verify_store);
 
   OPENSSL_free(c);
diff --git a/ssl/ssl_rsa.c b/ssl/ssl_rsa.c
index a65f6f4..cfa4cda 100644
--- a/ssl/ssl_rsa.c
+++ b/ssl/ssl_rsa.c
@@ -61,6 +61,7 @@
 #include <openssl/err.h>
 #include <openssl/evp.h>
 #include <openssl/mem.h>
+#include <openssl/type_check.h>
 #include <openssl/x509.h>
 
 #include "internal.h"
@@ -331,18 +332,51 @@
   ctx->cert->key_method = key_method;
 }
 
+OPENSSL_COMPILE_ASSERT(sizeof(int) >= 2 * sizeof(uint16_t),
+                       digest_list_conversion_cannot_overflow);
+
 int SSL_set_private_key_digest_prefs(SSL *ssl, const int *digest_nids,
                                      size_t num_digests) {
-  OPENSSL_free(ssl->cert->digest_nids);
+  OPENSSL_free(ssl->cert->sigalgs);
 
-  ssl->cert->num_digest_nids = 0;
-  ssl->cert->digest_nids = BUF_memdup(digest_nids, num_digests*sizeof(int));
-  if (ssl->cert->digest_nids == NULL) {
+  ssl->cert->sigalgs_len = 0;
+  ssl->cert->sigalgs = OPENSSL_malloc(sizeof(uint16_t) * 2 * num_digests);
+  if (ssl->cert->sigalgs == NULL) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_MALLOC_FAILURE);
     return 0;
   }
 
-  ssl->cert->num_digest_nids = num_digests;
+  /* Convert the digest list to a signature algorithms list.
+   *
+   * TODO(davidben): Replace this API with one that can express RSA-PSS, etc. */
+  for (size_t i = 0; i < num_digests; i++) {
+    switch (digest_nids[i]) {
+      case NID_sha1:
+        ssl->cert->sigalgs[ssl->cert->sigalgs_len] = SSL_SIGN_RSA_PKCS1_SHA1;
+        ssl->cert->sigalgs[ssl->cert->sigalgs_len + 1] = SSL_SIGN_ECDSA_SHA1;
+        ssl->cert->sigalgs_len += 2;
+        break;
+      case NID_sha256:
+        ssl->cert->sigalgs[ssl->cert->sigalgs_len] = SSL_SIGN_RSA_PKCS1_SHA256;
+        ssl->cert->sigalgs[ssl->cert->sigalgs_len + 1] =
+            SSL_SIGN_ECDSA_SECP256R1_SHA256;
+        ssl->cert->sigalgs_len += 2;
+        break;
+      case NID_sha384:
+        ssl->cert->sigalgs[ssl->cert->sigalgs_len] = SSL_SIGN_RSA_PKCS1_SHA384;
+        ssl->cert->sigalgs[ssl->cert->sigalgs_len + 1] =
+            SSL_SIGN_ECDSA_SECP384R1_SHA384;
+        ssl->cert->sigalgs_len += 2;
+        break;
+      case NID_sha512:
+        ssl->cert->sigalgs[ssl->cert->sigalgs_len] = SSL_SIGN_RSA_PKCS1_SHA512;
+        ssl->cert->sigalgs[ssl->cert->sigalgs_len + 1] =
+            SSL_SIGN_ECDSA_SECP521R1_SHA512;
+        ssl->cert->sigalgs_len += 2;
+        break;
+    }
+  }
+
   return 1;
 }
 
@@ -364,6 +398,31 @@
   return EVP_PKEY_size(ssl->cert->privatekey);
 }
 
+/* tls12_get_hash returns the EVP_MD corresponding to the TLS signature
+ * algorithm |sigalg|. It returns NULL if the type is unknown. */
+static const EVP_MD *tls12_get_hash(uint16_t sigalg) {
+  if (sigalg == SSL_SIGN_RSA_PKCS1_MD5_SHA1) {
+    return EVP_md5_sha1();
+  }
+
+  switch (sigalg >> 8) {
+    case TLSEXT_hash_sha1:
+      return EVP_sha1();
+
+    case TLSEXT_hash_sha256:
+      return EVP_sha256();
+
+    case TLSEXT_hash_sha384:
+      return EVP_sha384();
+
+    case TLSEXT_hash_sha512:
+      return EVP_sha512();
+
+    default:
+      return NULL;
+  }
+}
+
 enum ssl_private_key_result_t ssl_private_key_sign(
     SSL *ssl, uint8_t *out, size_t *out_len, size_t max_out,
     uint16_t signature_algorithm, const uint8_t *in, size_t in_len) {
diff --git a/ssl/t1_lib.c b/ssl/t1_lib.c
index 9be9add..f1551c8 100644
--- a/ssl/t1_lib.c
+++ b/ssl/t1_lib.c
@@ -2520,37 +2520,20 @@
   return ret;
 }
 
-const EVP_MD *tls12_get_hash(uint16_t sigalg) {
-  if (sigalg == SSL_SIGN_RSA_PKCS1_MD5_SHA1) {
-    return EVP_md5_sha1();
-  }
-
-  switch (sigalg >> 8) {
-    case TLSEXT_hash_sha1:
-      return EVP_sha1();
-
-    case TLSEXT_hash_sha256:
-      return EVP_sha256();
-
-    case TLSEXT_hash_sha384:
-      return EVP_sha384();
-
-    case TLSEXT_hash_sha512:
-      return EVP_sha512();
-
-    default:
-      return NULL;
-  }
-}
-
 /* tls12_get_pkey_type returns the EVP_PKEY type corresponding to TLS signature
  * algorithm |sigalg|. It returns -1 if the type is unknown. */
 static int tls12_get_pkey_type(uint16_t sigalg) {
-  switch (sigalg & 0xff) {
-    case TLSEXT_signature_rsa:
+  switch (sigalg) {
+    case SSL_SIGN_RSA_PKCS1_SHA1:
+    case SSL_SIGN_RSA_PKCS1_SHA256:
+    case SSL_SIGN_RSA_PKCS1_SHA384:
+    case SSL_SIGN_RSA_PKCS1_SHA512:
       return EVP_PKEY_RSA;
 
-    case TLSEXT_signature_ecdsa:
+    case SSL_SIGN_ECDSA_SHA1:
+    case SSL_SIGN_ECDSA_SECP256R1_SHA256:
+    case SSL_SIGN_ECDSA_SECP384R1_SHA384:
+    case SSL_SIGN_ECDSA_SECP521R1_SHA512:
       return EVP_PKEY_EC;
 
     default:
@@ -2617,28 +2600,21 @@
     return SSL_SIGN_ECDSA_SHA1;
   }
 
-  static const int kDefaultDigestList[] = {NID_sha256, NID_sha384, NID_sha512,
-                                           NID_sha1};
-
-  const int *digest_nids = kDefaultDigestList;
-  size_t num_digest_nids =
-      sizeof(kDefaultDigestList) / sizeof(kDefaultDigestList[0]);
-  if (cert->digest_nids != NULL) {
-    digest_nids = cert->digest_nids;
-    num_digest_nids = cert->num_digest_nids;
+  const uint16_t *sigalgs = kDefaultSignatureAlgorithms;
+  size_t sigalgs_len = sizeof(kDefaultSignatureAlgorithms) /
+                       sizeof(kDefaultSignatureAlgorithms[0]);
+  if (cert->sigalgs != NULL) {
+    sigalgs = cert->sigalgs;
+    sigalgs_len = cert->sigalgs_len;
   }
 
-  for (i = 0; i < num_digest_nids; i++) {
-    const int digest_nid = digest_nids[i];
+  for (i = 0; i < sigalgs_len; i++) {
     for (j = 0; j < cert->peer_sigalgslen; j++) {
       uint16_t signature_algorithm = cert->peer_sigalgs[j];
       /* SSL_SIGN_RSA_PKCS1_MD5_SHA1 is an internal value and should never be
        * negotiated. */
-      if (signature_algorithm == SSL_SIGN_RSA_PKCS1_MD5_SHA1) {
-        continue;
-      }
-      const EVP_MD *md = tls12_get_hash(signature_algorithm);
-      if (md != NULL && EVP_MD_type(md) == digest_nid &&
+      if (signature_algorithm != SSL_SIGN_RSA_PKCS1_MD5_SHA1 &&
+          signature_algorithm == sigalgs[i] &&
           tls12_get_pkey_type(signature_algorithm) == type) {
         return signature_algorithm;
       }