Document error behavior of PKCS5_PBKDF2_HMAC and EVP_PBE_scrypt

Change-Id: I5ce176538a53136aff3eea4af04b762ac9a5a994
Reviewed-on: https://boringssl-review.googlesource.com/31044
Reviewed-by: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

crypto/evp/scrypt.c

diff --git a/crypto/evp/scrypt.c b/crypto/evp/scrypt.c
index ed186ee..53a4554 100644
--- a/crypto/evp/scrypt.c
+++ b/crypto/evp/scrypt.c
@@ -187,6 +187,10 @@
   int ret = 0;
   block_t *T = B + B_blocks;
   block_t *V = T + T_blocks;
+
+  // NOTE: PKCS5_PBKDF2_HMAC can only fail due to allocation failure
+  // or |iterations| of 0 (we pass 1 here). This is consistent with
+  // the documented failure conditions of EVP_PBE_scrypt.
   if (!PKCS5_PBKDF2_HMAC(password, password_len, salt, salt_len, 1,
                          EVP_sha256(), B_bytes, (uint8_t *)B)) {
     goto err;

include/openssl/evp.h

diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index eb4c727..3719d7f 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -417,7 +417,7 @@
 
 // PKCS5_PBKDF2_HMAC computes |iterations| iterations of PBKDF2 of |password|
 // and |salt|, using |digest|, and outputs |key_len| bytes to |out_key|. It
-// returns one on success and zero on error.
+// returns one on success and zero on allocation failure or if iterations is 0.
 OPENSSL_EXPORT int PKCS5_PBKDF2_HMAC(const char *password, size_t password_len,
                                      const uint8_t *salt, size_t salt_len,
                                      unsigned iterations, const EVP_MD *digest,
@@ -433,12 +433,20 @@
 
 // EVP_PBE_scrypt expands |password| into a secret key of length |key_len| using
 // scrypt, as described in RFC 7914, and writes the result to |out_key|. It
-// returns one on success and zero on error.
+// returns one on success and zero on allocation failure, if the memory required
+// for the operation exceeds |max_mem|, or if any of the parameters are invalid
+// as described below.
 //
 // |N|, |r|, and |p| are as described in RFC 7914 section 6. They determine the
-// cost of the operation. If the memory required exceeds |max_mem|, the
-// operation will fail instead. If |max_mem| is zero, a defult limit of 32MiB
-// will be used.
+// cost of the operation. If |max_mem| is zero, a defult limit of 32MiB will be
+// used.
+//
+// The parameters are considered invalid under any of the following conditions:
+// - |r| or |p| are zero
+// - |p| > (2^30 - 1) / |r|
+// - |N| is not a power of two
+// - |N| > 2^32
+// - |N| > 2^(128 * |r| / 8)
 OPENSSL_EXPORT int EVP_PBE_scrypt(const char *password, size_t password_len,
                                   const uint8_t *salt, size_t salt_len,
                                   uint64_t N, uint64_t r, uint64_t p,