Add the proper limit to PKCS5_PBKDF2_HMAC.

RFC 8018 states one to avoid cycling - so let's do that.

Change-Id: I7abe0165e37cd35a94f02f889ad49be16a6a6964
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/95931
Commit-Queue: Rudolf Polzer <rpolzer@google.com>
Reviewed-by: David Benjamin <davidben@google.com>

crypto/evp/pbkdf.cc

diff --git a/crypto/evp/pbkdf.cc b/crypto/evp/pbkdf.cc
index 6388876..aad9348 100644
--- a/crypto/evp/pbkdf.cc
+++ b/crypto/evp/pbkdf.cc
@@ -16,6 +16,7 @@
 
 #include <string.h>
 
+#include <openssl/err.h>
 #include <openssl/hmac.h>
 
 #include "../internal.h"
@@ -34,6 +35,10 @@
 
   uint32_t i = 1;
   size_t md_len = EVP_MD_size(digest);
+  if (key_len > ((uint64_t{1} << 32) - 1) * md_len) {
+    OPENSSL_PUT_ERROR(EVP, EVP_R_INVALID_SECRET_LENGTH);
+    return 0;
+  }
   while (key_len > 0) {
     size_t todo = md_len;
     if (todo > key_len) {

crypto/evp/pbkdf_test.cc

diff --git a/crypto/evp/pbkdf_test.cc b/crypto/evp/pbkdf_test.cc
index d6a321f..5daffa7 100644
--- a/crypto/evp/pbkdf_test.cc
+++ b/crypto/evp/pbkdf_test.cc
@@ -15,6 +15,7 @@
 #include <gtest/gtest.h>
 
 #include <openssl/digest.h>
+#include <openssl/err.h>
 #include <openssl/evp.h>
 
 #include "../internal.h"
@@ -148,3 +149,18 @@
   // the out key.
   EXPECT_EQ(expected_first_byte, key[0]);
 }
+
+#if defined(OPENSSL_64_BIT)
+TEST(PBKDFTest, HugeKeyLen) {
+  static const char kPassword[] = "password";
+  static const uint8_t kSalt[] = {1, 2, 3, 4};
+  // Try a size chosen to clearly exceed the limit of 2^32-1 SHA-1 blocks by 1.
+  // In case the function does not reject this input, it will crash on writing
+  // to the nullptr.
+  EXPECT_FALSE(PKCS5_PBKDF2_HMAC(kPassword, strlen(kPassword), kSalt,
+                                 sizeof(kSalt), 1, EVP_sha1(),
+                                 size_t{SHA_DIGEST_LENGTH} << 32, nullptr));
+  EXPECT_TRUE(
+      ErrorEquals(ERR_get_error(), ERR_LIB_EVP, EVP_R_INVALID_SECRET_LENGTH));
+}
+#endif