Add XOF compilation compatibility flags

This CL adds compatibility flags for XOF digests in service of easing
compatibility between OpenSSL and BoringSSL. See this logic in Node:
https://github.com/nodejs/node/blob/master/src/node_crypto.cc#L4599-L4611

Change-Id: I7f12bed8fb1ea2d9e49dba14ed0c4c819596c70d
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/37564
Reviewed-by: Adam Langley <agl@google.com>
Commit-Queue: David Benjamin <davidben@google.com>

crypto/err/evp.errordata

diff --git a/crypto/err/evp.errordata b/crypto/err/evp.errordata
index 771cd6a..390dec0 100644
--- a/crypto/err/evp.errordata
+++ b/crypto/err/evp.errordata
@@ -22,6 +22,7 @@
 EVP,132,MEMORY_LIMIT_EXCEEDED
 EVP,118,MISSING_PARAMETERS
 EVP,130,NOT_A_PRIVATE_KEY
+EVP,135,NOT_XOF_OR_INVALID_LENGTH
 EVP,119,NO_DEFAULT_DIGEST
 EVP,120,NO_KEY_SET
 EVP,121,NO_MDC2_SUPPORT

crypto/evp/evp.c

diff --git a/crypto/evp/evp.c b/crypto/evp/evp.c
index 0e90b6f..60fdf64 100644
--- a/crypto/evp/evp.c
+++ b/crypto/evp/evp.c
@@ -71,6 +71,11 @@
 #include "../internal.h"
 
 
+// Node depends on |EVP_R_NOT_XOF_OR_INVALID_LENGTH|.
+//
+// TODO(davidben): Fix Node to not touch the error queue itself and remove this.
+OPENSSL_DECLARE_ERROR_REASON(EVP, NOT_XOF_OR_INVALID_LENGTH)
+
 EVP_PKEY *EVP_PKEY_new(void) {
   EVP_PKEY *ret;
 

crypto/fipsmodule/digest/digest.c

diff --git a/crypto/fipsmodule/digest/digest.c b/crypto/fipsmodule/digest/digest.c
index 68e81c4..a0b3bf5 100644
--- a/crypto/fipsmodule/digest/digest.c
+++ b/crypto/fipsmodule/digest/digest.c
@@ -120,6 +120,8 @@
   return 0;
 }
 
+uint32_t EVP_MD_meth_get_flags(const EVP_MD *md) { return EVP_MD_flags(md); }
+
 int EVP_MD_CTX_copy_ex(EVP_MD_CTX *out, const EVP_MD_CTX *in) {
   // |in->digest| may be NULL if this is a signing |EVP_MD_CTX| for, e.g.,
   // Ed25519 which does not hash with |EVP_MD_CTX|.

include/openssl/digest.h

diff --git a/include/openssl/digest.h b/include/openssl/digest.h
index c7c6797..c3ceb7f 100644
--- a/include/openssl/digest.h
+++ b/include/openssl/digest.h
@@ -195,7 +195,7 @@
 // EVP_MD_block_size returns the native block-size of |md|, in bytes.
 OPENSSL_EXPORT size_t EVP_MD_block_size(const EVP_MD *md);
 
-// EVP_MD_FLAG_PKEY_DIGEST indicates the the digest function is used with a
+// EVP_MD_FLAG_PKEY_DIGEST indicates that the digest function is used with a
 // specific public key in order to verify signatures. (For example,
 // EVP_dss1.)
 #define EVP_MD_FLAG_PKEY_DIGEST 1
@@ -205,6 +205,11 @@
 // undefined rather than NULL.
 #define EVP_MD_FLAG_DIGALGID_ABSENT 2
 
+// EVP_MD_FLAG_XOF indicates that the digest is an extensible-output function
+// (XOF). This flag is defined for compatibility and will never be set in any
+// |EVP_MD| in BoringSSL.
+#define EVP_MD_FLAG_XOF 4
+
 
 // Digest operation accessors.
 
@@ -274,6 +279,9 @@
 OPENSSL_EXPORT int EVP_DigestFinalXOF(EVP_MD_CTX *ctx, uint8_t *out,
                                       size_t len);
 
+// EVP_MD_meth_get_flags calls |EVP_MD_flags|.
+OPENSSL_EXPORT uint32_t EVP_MD_meth_get_flags(const EVP_MD *md);
+
 
 struct evp_md_pctx_ops;
 

include/openssl/evp.h

diff --git a/include/openssl/evp.h b/include/openssl/evp.h
index 999e19d..ad9c05e 100644
--- a/include/openssl/evp.h
+++ b/include/openssl/evp.h
@@ -942,6 +942,15 @@
 #endif
 
 
+// Nodejs compatibility section (hidden).
+//
+// These defines exist for node.js, with the hope that we can eliminate the
+// need for them over time.
+
+#define EVPerr(function, reason) \
+  ERR_put_error(ERR_LIB_EVP, 0, reason, __FILE__, __LINE__)
+
+
 // Private structures.
 
 struct evp_pkey_st {
@@ -1016,5 +1025,6 @@
 #define EVP_R_MEMORY_LIMIT_EXCEEDED 132
 #define EVP_R_INVALID_PARAMETERS 133
 #define EVP_R_INVALID_PEER_KEY 134
+#define EVP_R_NOT_XOF_OR_INVALID_LENGTH 135
 
 #endif  // OPENSSL_HEADER_EVP_H