Avoid making assumptions about RSA_PSS_* constants

Follow-up to Ibd82361e81fe58f4a1006fc9f4c605c59b66ab12.

Change-Id: Iaa1ffda4b4316b43d03d80a4e0d32c9f06670632
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/79907
Commit-Queue: David Benjamin <davidben@google.com>
Auto-Submit: David Benjamin <davidben@google.com>
Commit-Queue: Adam Langley <agl@google.com>
Reviewed-by: Adam Langley <agl@google.com>

crypto/evp/p_rsa.cc

diff --git a/crypto/evp/p_rsa.cc b/crypto/evp/p_rsa.cc
index 4b47ee3..4592c19 100644
--- a/crypto/evp/p_rsa.cc
+++ b/crypto/evp/p_rsa.cc
@@ -385,7 +385,9 @@
       if (type == EVP_PKEY_CTRL_GET_RSA_PSS_SALTLEN) {
         *(int *)p2 = rctx->saltlen;
       } else {
-        if (p1 < -2) {
+        // Negative salt lengths are special values.
+        if (p1 < 0 &&
+            (p1 != RSA_PSS_SALTLEN_DIGEST && p1 != RSA_PSS_SALTLEN_AUTO)) {
           return 0;
         }
         rctx->saltlen = p1;

crypto/fipsmodule/rsa/padding.cc.inc

diff --git a/crypto/fipsmodule/rsa/padding.cc.inc b/crypto/fipsmodule/rsa/padding.cc.inc
index 2b9e20d..726d23d 100644
--- a/crypto/fipsmodule/rsa/padding.cc.inc
+++ b/crypto/fipsmodule/rsa/padding.cc.inc
@@ -178,16 +178,14 @@
   size_t emLen, maskedDBLen, salt_start;
   FIPS_service_indicator_lock_state();
 
-  // Negative sLen has special meanings:
-  //   -1      sLen == hLen
-  //   -2      salt length is autorecovered from signature
-  //   -N      reserved
   size_t hLen = EVP_MD_size(Hash);
   if (sLen == RSA_PSS_SALTLEN_DIGEST) {
     sLen = (int)hLen;
   } else if (sLen == RSA_PSS_SALTLEN_AUTO) {
-    // Leave |sLen| negative.
+    // Leave |sLen| negative, which will trigger the logic below to recover and
+    // allow any salt length.
   } else if (sLen < 0) {
+    // Other negative values are reserved.
     OPENSSL_PUT_ERROR(RSA, RSA_R_SLEN_CHECK_FAILED);
     goto err;
   }
@@ -202,7 +200,7 @@
     EM++;
     emLen--;
   }
-  // |sLen| may be -2 for the non-standard salt length recovery mode.
+  // |sLen| may be negative for the non-standard salt length recovery mode.
   if (emLen < hLen + 2 || (sLen >= 0 && emLen < hLen + (size_t)sLen + 2)) {
     OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
     goto err;
@@ -299,16 +297,14 @@
     goto err;
   }
 
-  // Negative sLenRequested has special meanings:
-  //   -1  sLen == hLen
-  //   -2  salt length is maximized
-  //   -N  reserved
   size_t sLen;
   if (sLenRequested == RSA_PSS_SALTLEN_DIGEST) {
     sLen = hLen;
   } else if (sLenRequested == RSA_PSS_SALTLEN_AUTO) {
+    // Use the maximum possible salt length.
     sLen = emLen - hLen - 2;
   } else if (sLenRequested < 0) {
+    // Other negative values are reserved.
     OPENSSL_PUT_ERROR(RSA, RSA_R_SLEN_CHECK_FAILED);
     goto err;
   } else {