Align EVP return values with BoringSSL convention.
Where possible, functions should return one for success and zero for
error. The use of additional negative values to indicate an error is,
itself, error prone.
This change fixes many EVP functions to remove the possibility of
negative return values. Existing code that is testing for <= 0 will
continue to function, although there is the possibility that some code
was differentiating between negative values (error) and zero (invalid
signature) for the verify functions and will now show the wrong error
message.
Change-Id: I982512596bb18a82df65861394dbd7487783bd3d
Reviewed-on: https://boringssl-review.googlesource.com/1333
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/include/openssl/ecdsa.h b/include/openssl/ecdsa.h
index 8a74ecb..a2436f1 100644
--- a/include/openssl/ecdsa.h
+++ b/include/openssl/ecdsa.h
@@ -78,12 +78,8 @@
/* ECDSA_verify verifies that |sig_len| bytes from |sig| constitute a valid
* signature by |key| of |digest|. (The |type| argument should be zero.) It
- * returns one on success, zero if the signature is invalid and -1 on error.
- *
- * DANGER: this function differs from the usual OpenSSL return value
- * convention.
- *
- * TODO(fork): change this to return the usual 0/1. */
+ * returns one on success or zero if the signature is invalid or an error
+ * occured. */
int ECDSA_verify(int type, const uint8_t *digest, size_t digest_len,
const uint8_t *sig, size_t sig_len, EC_KEY *key);
@@ -116,12 +112,9 @@
EC_KEY *key);
/* ECDSA_verify verifies that |sig| constitutes a valid signature by |key| of
- * |digest|. It returns one on success, zero if the signature is invalid and -1
+ * |digest|. It returns one on success or zero if the signature is invalid or
* on error.
*
- * DANGER: this function differs from the usual OpenSSL return value
- * convention.
- *
* TODO(fork): remove this function. */
int ECDSA_do_verify(const uint8_t *digest, size_t digest_len,
const ECDSA_SIG *sig, EC_KEY *key);
diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index b522e8f..d47d51f 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -222,8 +222,7 @@
* operation will be written to |*pctx|; this can be used to set alternative
* signing options.
*
- * It returns one on success, or <= 0 on error. WARNING: this differs from the
- * usual OpenSSL return convention. */
+ * It returns one on success, or zero on error. */
int EVP_DigestSignInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx, const EVP_MD *type,
ENGINE *e, EVP_PKEY *pkey);
@@ -239,8 +238,7 @@
* is successful, the signature is written to |out_sig| and |*out_sig_len| is
* set to its length.
*
- * It returns one on success and <= 0 on error. WARNING: this differs from the
- * usual, OpenSSL return value convention. */
+ * It returns one on success, or zero on error. */
int EVP_DigestSignFinal(EVP_MD_CTX *ctx, uint8_t *out_sig, size_t *out_sig_len);
@@ -252,8 +250,7 @@
* operation will be written to |*pctx|; this can be used to set alternative
* signing options.
*
- * It returns one on success, or <= 0 on error. WARNING: this differs from the
- * usual OpenSSL return convention. */
+ * It returns one on success, or zero on error. */
int EVP_DigestVerifyInit(EVP_MD_CTX *ctx, EVP_PKEY_CTX **pctx,
const EVP_MD *type, ENGINE *e, EVP_PKEY *pkey);
@@ -422,8 +419,7 @@
/* EVP_PKEY_sign_init initialises an |EVP_PKEY_CTX| for a signing operation. It
* should be called before |EVP_PKEY_sign|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_sign_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_sign signs |data_len| bytes from |data| using |ctx|. If |sig| is
@@ -443,23 +439,20 @@
/* EVP_PKEY_verify_init initialises an |EVP_PKEY_CTX| for a signature
* verification operation. It should be called before |EVP_PKEY_verify|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_verify_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_verify verifies that |sig_len| bytes from |sig| are a valid signature
* for |data|.
*
- * It returns one on success or zero on error. (Note: this differs from
- * OpenSSL, which can also return negative values to indicate an error. ) */
+ * It returns one on success or zero on error. */
int EVP_PKEY_verify(EVP_PKEY_CTX *ctx, const uint8_t *sig, size_t sig_len,
const uint8_t *data, size_t data_len);
/* EVP_PKEY_encrypt_init initialises an |EVP_PKEY_CTX| for an encryption
* operation. It should be called before |EVP_PKEY_encrypt|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_encrypt_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_encrypt encrypts |in_len| bytes from |in|. If |out| is NULL, the
@@ -471,16 +464,14 @@
* WARNING: Setting |out| to NULL only gives the maximum size of the
* ciphertext. The actual ciphertext may be smaller.
*
- * It returns one on success or <= 0 on error. (Note: this differs from
- * OpenSSL, which can also return negative values to indicate an error. ) */
+ * It returns one on success or zero on error. */
int EVP_PKEY_encrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len,
const uint8_t *in, size_t in_len);
/* EVP_PKEY_decrypt_init initialises an |EVP_PKEY_CTX| for a decryption
* operation. It should be called before |EVP_PKEY_decrypt|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_decrypt_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_decrypt decrypts |in_len| bytes from |in|. If |out| is NULL, the
@@ -492,8 +483,7 @@
* WARNING: Setting |out| to NULL only gives the maximum size of the
* plaintext. The actual plaintext may be smaller.
*
- * It returns one on success or <= 0 on error. (Note: this differs from
- * OpenSSL, which can also return negative values to indicate an error. ) */
+ * It returns one on success or zero on error. */
int EVP_PKEY_decrypt(EVP_PKEY_CTX *ctx, uint8_t *out, size_t *out_len,
const uint8_t *in, size_t in_len);
@@ -501,15 +491,13 @@
* operation. It should be called before |EVP_PKEY_derive_set_peer| and
* |EVP_PKEY_derive|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_derive_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_derive_set_peer sets the peer's key to be used for key derivation
* by |ctx| to |peer|. It should be called after |EVP_PKEY_derive_init|. (For
* example, this is used to set the peer's key in (EC)DH.) It returns one on
- * success and <= 0 on error. WARNING: this differs from the usual return value
- * convention. */
+ * success and zero on error. */
int EVP_PKEY_derive_set_peer(EVP_PKEY_CTX *ctx, EVP_PKEY *peer);
/* EVP_PKEY_derive derives a shared key between the two keys configured in
@@ -521,21 +509,18 @@
* WARNING: Setting |out| to NULL only gives the maximum size of the key. The
* actual key may be smaller.
*
- * It returns one on success and <= 0 on error. WARNING: this differs from the
- * usual return convention. */
+ * It returns one on success and zero on error. */
int EVP_PKEY_derive(EVP_PKEY_CTX *ctx, uint8_t *key, size_t *out_key_len);
/* EVP_PKEY_keygen_init initialises an |EVP_PKEY_CTX| for a key generation
* operation. It should be called before |EVP_PKEY_keygen|.
*
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * usual return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_keygen_init(EVP_PKEY_CTX *ctx);
/* EVP_PKEY_keygen performs a key generation operation using the values from
* |ctx| and sets |*ppkey| to a fresh |EVP_PKEY| containing the resulting key.
- * It returns one on success or <= 0 on error. WARNING: this differs from the
- * normal return value convention. */
+ * It returns one on success or zero on error. */
int EVP_PKEY_keygen(EVP_PKEY_CTX *ctx, EVP_PKEY **ppkey);
diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index 821bb1b..9cc465d 100644
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -93,7 +93,9 @@
* for |e|. If |cb| is not NULL then it is called during the key generation
* process. In addition to the calls documented for |BN_generate_prime_ex|, it
* is called with event=2 when the n'th prime is rejected as unsuitable and
- * with event=3 when a suitable value for |p| is found. */
+ * with event=3 when a suitable value for |p| is found.
+ *
+ * It returns one on success or zero on error. */
int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e, BN_GENCB *cb);
