Align with upstream's error strings, take two.

I messed up a few of these.

ASN1_R_UNSUPPORTED_ALGORITHM doesn't exist. X509_R_UNSUPPORTED_ALGORITHM does
exist as part of X509_PUBKEY_set, but the SPKI parser doesn't emit this. (I
don't mind the legacy code having really weird errors, but since EVP is now
limited to things we like, let's try to keep that clean.) To avoid churn in
Conscrypt, we'll keep defining X509_R_UNSUPPORTED_ALGORITHM, but not actually
do anything with it anymore.  Conscrypt was already aware of
EVP_R_UNSUPPORTED_ALGORITHM, so this should be fine. (I don't expect
EVP_R_UNSUPPORTED_ALGORITHM to go away. The SPKI parsers we like live in EVP
now.)

A few other ASN1_R_* values didn't quite match upstream, so make those match
again. Finally, I got some of the rsa_pss.c values wrong. Each of those
corresponds to an (overly specific) RSA_R_* value in upstream. However, those
were gone in BoringSSL since even the initial commit. We placed the RSA <-> EVP
glue in crypto/evp (so crypto/rsa wouldn't depend on crypto/evp) while upstream
placed them in crypto/rsa.

Since no one seemed to notice the loss of RSA_R_INVALID_SALT_LENGTH, let's undo
all the cross-module errors inserted in crypto/rsa. Instead, since that kind of
specificity is not useful, funnel it all into X509_R_INVALID_PSS_PARAMETERS
(formerly EVP_R_INVALID_PSS_PARAMETERS, formerly RSA_R_INVALID_PSS_PARAMETERS).

Reset the error codes for all affected modules.

(That our error code story means error codes are not stable across this kind of
refactoring is kind of a problem. Hopefully this will be the last of it.)

Change-Id: Ibfb3a0ac340bfc777bc7de6980ef3ddf0a8c84bc
Reviewed-on: https://boringssl-review.googlesource.com/7458
Reviewed-by: Emily Stark (Dunn) <estark@google.com>
Reviewed-by: David Benjamin <davidben@google.com>
diff --git a/crypto/x509/algorithm.c b/crypto/x509/algorithm.c
index 29bfc8d..78ae882 100644
--- a/crypto/x509/algorithm.c
+++ b/crypto/x509/algorithm.c
@@ -89,7 +89,7 @@
   int sign_nid;
   if (!OBJ_find_sigid_by_algs(&sign_nid, EVP_MD_type(digest),
                               EVP_PKEY_id(pkey))) {
-    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_ALGORITHM);
+    OPENSSL_PUT_ERROR(ASN1, ASN1_R_DIGEST_AND_KEY_TYPE_NOT_SUPPORTED);
     return 0;
   }
 
@@ -107,7 +107,7 @@
   int sigalg_nid = OBJ_obj2nid(sigalg->algorithm);
   int digest_nid, pkey_nid;
   if (!OBJ_find_sigid_algs(sigalg_nid, &digest_nid, &pkey_nid)) {
-    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_ALGORITHM);
+    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNKNOWN_SIGNATURE_ALGORITHM);
     return 0;
   }
 
@@ -120,7 +120,7 @@
   /* NID_undef signals that there are custom parameters to set. */
   if (digest_nid == NID_undef) {
     if (sigalg_nid != NID_rsassaPss) {
-      OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_ALGORITHM);
+      OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNKNOWN_SIGNATURE_ALGORITHM);
       return 0;
     }
     return x509_rsa_pss_to_ctx(ctx, sigalg, pkey);
@@ -129,7 +129,7 @@
   /* Otherwise, initialize with the digest from the OID. */
   const EVP_MD *digest = EVP_get_digestbynid(digest_nid);
   if (digest == NULL) {
-    OPENSSL_PUT_ERROR(X509, X509_R_UNKNOWN_NID);
+    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM);
     return 0;
   }
 
diff --git a/crypto/x509/rsa_pss.c b/crypto/x509/rsa_pss.c
index 887f372..1ae01a3 100644
--- a/crypto/x509/rsa_pss.c
+++ b/crypto/x509/rsa_pss.c
@@ -161,7 +161,7 @@
   }
   md = EVP_get_digestbyobj(alg->algorithm);
   if (md == NULL) {
-    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_ALGORITHM);
+    OPENSSL_PUT_ERROR(X509, X509_R_INVALID_PSS_PARAMETERS);
   }
   return md;
 }
@@ -173,17 +173,14 @@
     return EVP_sha1();
   }
   /* Check mask and lookup mask hash algorithm */
-  if (OBJ_obj2nid(alg->algorithm) != NID_mgf1) {
-    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_ALGORITHM);
-    return NULL;
-  }
-  if (!maskHash) {
-    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_ALGORITHM);
+  if (OBJ_obj2nid(alg->algorithm) != NID_mgf1 ||
+      maskHash == NULL) {
+    OPENSSL_PUT_ERROR(X509, X509_R_INVALID_PSS_PARAMETERS);
     return NULL;
   }
   md = EVP_get_digestbyobj(maskHash->algorithm);
   if (md == NULL) {
-    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_ALGORITHM);
+    OPENSSL_PUT_ERROR(X509, X509_R_INVALID_PSS_PARAMETERS);
     return NULL;
   }
   return md;
@@ -253,7 +250,7 @@
   X509_ALGOR *maskHash;
   RSA_PSS_PARAMS *pss = rsa_pss_decode(sigalg, &maskHash);
   if (pss == NULL) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_INVALID_PSS_PARAMETERS);
+    OPENSSL_PUT_ERROR(X509, X509_R_INVALID_PSS_PARAMETERS);
     goto err;
   }
 
@@ -270,7 +267,7 @@
     /* Could perform more salt length sanity checks but the main
      * RSA routines will trap other invalid values anyway. */
     if (saltlen < 0) {
-      OPENSSL_PUT_ERROR(RSA, RSA_R_INVALID_SALT_LENGTH);
+      OPENSSL_PUT_ERROR(X509, X509_R_INVALID_PSS_PARAMETERS);
       goto err;
     }
   }
@@ -278,7 +275,7 @@
   /* low-level routines support only trailer field 0xbc (value 1)
    * and PKCS#1 says we should reject any other value anyway. */
   if (pss->trailerField != NULL && ASN1_INTEGER_get(pss->trailerField) != 1) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_INVALID_TRAILER);
+    OPENSSL_PUT_ERROR(X509, X509_R_INVALID_PSS_PARAMETERS);
     goto err;
   }
 
diff --git a/crypto/x509/x509.c b/crypto/x509/x509.c
index 31f9e1e..188fd49 100644
--- a/crypto/x509/x509.c
+++ b/crypto/x509/x509.c
@@ -57,9 +57,14 @@
 #include <openssl/x509.h>
 
 #include <openssl/bio.h>
+#include <openssl/err.h>
 #include <openssl/mem.h>
 
 
+/* |X509_R_UNSUPPORTED_ALGORITHM| is no longer emitted, but continue to define
+ * it to avoid downstream churn. */
+OPENSSL_DECLARE_ERROR_REASON(X509, UNSUPPORTED_ALGORITHM)
+
 int PKCS8_pkey_set0(PKCS8_PRIV_KEY_INFO *priv, ASN1_OBJECT *aobj, int version,
                     int ptype, void *pval, uint8_t *penc, int penclen) {
   uint8_t **ppenc = NULL;