Change the RSA-PSS salt length default to RSA_PSS_SALTLEN_DIGEST

Update-Note: Signing RSA-PSS with the EVP APIs will now default to a
salt length of RSA_PSS_SALTLEN_DIGEST (-1) instead of
RSA_PSS_SALTLEN_AUTO (-2). Applications that use
EVP_PKEY_CTX_set_rsa_padding(RSA_PKCS1_PSS_PADDING) without calling
EVP_PKEY_CTX_set_rsa_pss_saltlen will see slightly different behavior.
Call EVP_PKEY_CTX_set_rsa_pss_saltlen(RSA_PSS_SALTLEN_AUTO) to restore
the old behavior.

The new behavior matches that protocols do in practice (TLS, our only
supported X.509 modes), and also matches FIPS 186-5 requirements. The
RSA_PSS_SALTLEN_AUTO behavior caused signing to maximize the salt
length and caused verifying to automatically recover the salt length and
accept all values. Both behaviors are forbidden by FIPS 186-5, and the
verification procedure in RFC 8017 does not admit this auto-recovery
behavior.

Change-Id: I1d5666d3401c335840d8736207143bc673d5c789
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/79987
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
Commit-Queue: Adam Langley <agl@google.com>
Auto-Submit: David Benjamin <davidben@google.com>

crypto/evp/evp_extra_test.cc

diff --git a/crypto/evp/evp_extra_test.cc b/crypto/evp/evp_extra_test.cc
index d533733..2d2dfc6 100644
--- a/crypto/evp/evp_extra_test.cc
+++ b/crypto/evp/evp_extra_test.cc
@@ -1307,3 +1307,17 @@
   EXPECT_FALSE(
       EVP_PKEY_new_raw_private_key(EVP_PKEY_RSA, nullptr, kKey, sizeof(kKey)));
 }
+
+// The default salt length for PSS should be |RSA_PSS_SALTLEN_DIGEST|.
+TEST(EVPExtraTest, PSSDefaultSaltLen) {
+  bssl::UniquePtr<EVP_PKEY> key = LoadExampleRSAKey();
+  ASSERT_TRUE(key);
+  bssl::ScopedEVP_MD_CTX ctx;
+  EVP_PKEY_CTX *pctx;
+  ASSERT_TRUE(
+      EVP_DigestSignInit(ctx.get(), &pctx, EVP_sha256(), nullptr, key.get()));
+  ASSERT_TRUE(EVP_PKEY_CTX_set_rsa_padding(pctx, RSA_PKCS1_PSS_PADDING));
+  int salt_len;
+  ASSERT_TRUE(EVP_PKEY_CTX_get_rsa_pss_saltlen(pctx, &salt_len));
+  EXPECT_EQ(salt_len, RSA_PSS_SALTLEN_DIGEST);
+}

crypto/evp/p_rsa.cc

diff --git a/crypto/evp/p_rsa.cc b/crypto/evp/p_rsa.cc
index c996f25..2f15f70 100644
--- a/crypto/evp/p_rsa.cc
+++ b/crypto/evp/p_rsa.cc
@@ -45,7 +45,7 @@
   // message digest for MGF1
   const EVP_MD *mgf1md = nullptr;
   // PSS salt length
-  int saltlen = RSA_PSS_SALTLEN_AUTO;
+  int saltlen = RSA_PSS_SALTLEN_DIGEST;
   bssl::Array<uint8_t> oaep_label;
 };
 

include/openssl/evp.h

diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 87e3ad2..c5c2391 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -692,12 +692,10 @@
 // recovered from the signature when verifying. Otherwise the value gives the
 // size of the salt in bytes.
 //
-// If unsure, use |RSA_PSS_SALTLEN_DIGEST|.
+// If unsure, use |RSA_PSS_SALTLEN_DIGEST|, which is the default. Note this
+// differs from OpenSSL, which defaults to |RSA_PSS_SALTLEN_AUTO|.
 //
 // Returns one on success or zero on error.
-//
-// TODO(davidben): The default is currently |RSA_PSS_SALTLEN_AUTO|. Switch it to
-// |RSA_PSS_SALTLEN_DIGEST|.
 OPENSSL_EXPORT int EVP_PKEY_CTX_set_rsa_pss_saltlen(EVP_PKEY_CTX *ctx,
                                                     int salt_len);
 

include/openssl/rsa.h

diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index c80aea8..94923e7 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -305,7 +305,9 @@
 // length. This is recommended.
 #define RSA_PSS_SALTLEN_DIGEST (-1)
 // RSA_PSS_SALTLEN_AUTO indicates a maximum possible PSS salt length when
-// signing, and automatically detecting the salt length when verifying.
+// signing, and automatically detecting the salt length when verifying. This is
+// not recommended. Neither the signing nor verifying behaviors are compliant
+// with FIPS 186-5.
 #define RSA_PSS_SALTLEN_AUTO (-2)
 
 // RSA_sign_pss_mgf1 signs |digest_len| bytes from |digest| with the public key