Document filesystem-based X509_STORE APIs

This includes the somewhat odd "defaults" API, which I've currently left
kind of handwavy. We should eventually decide what to do with this, be
it remove it, decide /etc/ssl is a fine default, or do something else
entirely. But I'll leave that to future us.

(If nothing else, we really should make it return an error on Windows
and macOS. It's really just Linux where /etc/ssl is a plausible platform
API.)

Bug: 426
Change-Id: Iacd2bb903f452ffe236a7a0b97e3072b5dcd8516
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/66388
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: Bob Beck <bbe@google.com>
diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h
index 04f1873..f3aac2b 100644
--- a/include/openssl/ssl.h
+++ b/include/openssl/ssl.h
@@ -2678,19 +2678,17 @@
 // SSL_CTX_get_cert_store returns |ctx|'s certificate store.
 OPENSSL_EXPORT X509_STORE *SSL_CTX_get_cert_store(const SSL_CTX *ctx);
 
-// SSL_CTX_set_default_verify_paths loads the OpenSSL system-default trust
-// anchors into |ctx|'s store. It returns one on success and zero on failure.
+// SSL_CTX_set_default_verify_paths calls |X509_STORE_set_default_paths| on
+// |ctx|'s store. See that function for details.
+//
+// Using this function is not recommended. In OpenSSL, these defaults are
+// determined by OpenSSL's install prefix. There is no corresponding concept for
+// BoringSSL. Future versions of BoringSSL may change or remove this
+// functionality.
 OPENSSL_EXPORT int SSL_CTX_set_default_verify_paths(SSL_CTX *ctx);
 
-// SSL_CTX_load_verify_locations loads trust anchors into |ctx|'s store from
-// |ca_file| and |ca_dir|, either of which may be NULL. If |ca_file| is passed,
-// it is opened and PEM-encoded CA certificates are read. If |ca_dir| is passed,
-// it is treated as a directory in OpenSSL's hashed directory format. It returns
-// one on success and zero on failure.
-//
-// See
-// https://www.openssl.org/docs/man1.1.0/man3/SSL_CTX_load_verify_locations.html
-// for documentation on the directory format.
+// SSL_CTX_load_verify_locations calls |X509_STORE_load_locations| on |ctx|'s
+// store. See that function for details.
 OPENSSL_EXPORT int SSL_CTX_load_verify_locations(SSL_CTX *ctx,
                                                  const char *ca_file,
                                                  const char *ca_dir);
diff --git a/include/openssl/x509.h b/include/openssl/x509.h
index 0b5921e..9e66321 100644
--- a/include/openssl/x509.h
+++ b/include/openssl/x509.h
@@ -2532,7 +2532,7 @@
 //
 // The result will include all certificates and CRLs added via
 // |X509_STORE_add_cert| and |X509_STORE_add_crl|, as well as any cached objects
-// added by |X509_LOOKUP_hash_dir|. The last of these may change over time, as
+// added by |X509_LOOKUP_add_dir|. The last of these may change over time, as
 // different objects are loaded from the filesystem. Callers should not depend
 // on this caching behavior. The objects are returned in no particular order.
 OPENSSL_EXPORT STACK_OF(X509_OBJECT) *X509_STORE_get1_objects(
@@ -3141,6 +3141,201 @@
                                                int trust);
 
 
+// Filesystem-based certificate stores.
+//
+// An |X509_STORE| may be configured to get its contents from the filesystem.
+// This is done by adding |X509_LOOKUP| structures to the |X509_STORE| with
+// |X509_STORE_add_lookup| and then configuring the |X509_LOOKUP| with paths.
+//
+// Most cases can use |X509_STORE_load_locations|, which configures the same
+// thing but is simpler to use.
+
+// X509_STORE_load_locations configures |store| to load data from filepaths
+// |file| and |dir|. It returns one on success and zero on error. Either of
+// |file| or |dir| may be NULL, but at least one must be non-NULL.
+//
+// If |file| is non-NULL, it loads CRLs and trusted certificates in PEM format
+// from the file at |file|, and them to |store|, as in |X509_load_cert_crl_file|
+// with |X509_FILETYPE_PEM|.
+//
+// If |dir| is non-NULL, it configures |store| to load CRLs and trusted
+// certificates from the directory at |dir| in PEM format, as in
+// |X509_LOOKUP_add_dir| with |X509_FILETYPE_PEM|.
+OPENSSL_EXPORT int X509_STORE_load_locations(X509_STORE *store,
+                                             const char *file, const char *dir);
+
+// X509_STORE_add_lookup returns an |X509_LOOKUP| associated with |store| with
+// type |method|, or NULL on error. The result is owned by |store|, so callers
+// are not expected to free it. This may be used with |X509_LOOKUP_add_dir| or
+// |X509_LOOKUP_load_file|, depending on |method|, to configure |store|.
+//
+// A single |X509_LOOKUP| may be configured with multiple paths, and an
+// |X509_STORE| only contains one |X509_LOOKUP| of each type, so there is no
+// need to call this function multiple times for a single type. Calling it
+// multiple times will return the previous |X509_LOOKUP| of that type.
+OPENSSL_EXPORT X509_LOOKUP *X509_STORE_add_lookup(
+    X509_STORE *store, const X509_LOOKUP_METHOD *method);
+
+// X509_LOOKUP_hash_dir creates |X509_LOOKUP|s that may be used with
+// |X509_LOOKUP_add_dir|.
+OPENSSL_EXPORT const X509_LOOKUP_METHOD *X509_LOOKUP_hash_dir(void);
+
+// X509_LOOKUP_file creates |X509_LOOKUP|s that may be used with
+// |X509_LOOKUP_load_file|.
+//
+// Although this is modeled as an |X509_LOOKUP|, this function is redundant. It
+// has the same effect as loading a certificate or CRL from the filesystem, in
+// the caller's desired format, and then adding it with |X509_STORE_add_cert|
+// and |X509_STORE_add_crl|.
+OPENSSL_EXPORT const X509_LOOKUP_METHOD *X509_LOOKUP_file(void);
+
+// The following constants are used to specify the format of files in an
+// |X509_LOOKUP|.
+#define X509_FILETYPE_PEM 1
+#define X509_FILETYPE_ASN1 2
+#define X509_FILETYPE_DEFAULT 3
+
+// X509_LOOKUP_load_file calls |X509_load_cert_crl_file|. |lookup| must have
+// been constructed with |X509_LOOKUP_file|.
+//
+// If |type| is |X509_FILETYPE_DEFAULT|, it ignores |file| and instead uses some
+// default system path with |X509_FILETYPE_PEM|. See also
+// |X509_STORE_set_default_paths|.
+OPENSSL_EXPORT int X509_LOOKUP_load_file(X509_LOOKUP *lookup, const char *file,
+                                         int type);
+
+// X509_LOOKUP_add_dir configures |lookup| to load CRLs and trusted certificates
+// from the directories in |path|. It returns one on success and zero on error.
+// |lookup| must have been constructed with |X509_LOOKUP_hash_dir|.
+//
+// WARNING: |path| is interpreted as a colon-separated (semicolon-separated on
+// Windows) list of paths. It is not possible to configure a path containing the
+// separator character. https://crbug.com/boringssl/691 tracks removing this
+// behavior.
+//
+// |type| should be one of the |X509_FILETYPE_*| constants and determines the
+// format of the files. If |type| is |X509_FILETYPE_DEFAULT|, |path| is ignored
+// and some default system path is used with |X509_FILETYPE_PEM|. See also
+// |X509_STORE_set_default_paths|.
+//
+// Trusted certificates should be named HASH.N and CRLs should be
+// named HASH.rN. HASH is |X509_NAME_hash| of the certificate subject and CRL
+// issuer, respectively, in hexadecimal. N is in decimal and counts hash
+// collisions consecutively, starting from zero. For example, "002c0b4f.0" and
+// "002c0b4f.r0".
+//
+// WARNING: Objects from |path| are loaded on demand, but cached in memory on
+// the |X509_STORE|. If a CA is removed from the directory, existing
+// |X509_STORE|s will continue to trust it. Cache entries are not evicted for
+// the lifetime of the |X509_STORE|.
+//
+// WARNING: This mechanism is also not well-suited for CRL updates.
+// |X509_STORE|s rely on this cache and never load the same CRL file twice. CRL
+// updates must use a new file, with an incremented suffix, to be reflected in
+// existing |X509_STORE|s. However, this means each CRL update will use
+// additional storage and memory. Instead, configure inputs that vary per
+// verification, such as CRLs, on each |X509_STORE_CTX| separately, using
+// functions like |X509_STORE_CTX_set0_crl|.
+OPENSSL_EXPORT int X509_LOOKUP_add_dir(X509_LOOKUP *lookup, const char *path,
+                                       int type);
+
+// X509_L_* are commands for |X509_LOOKUP_ctrl|.
+#define X509_L_FILE_LOAD 1
+#define X509_L_ADD_DIR 2
+
+// X509_LOOKUP_ctrl implements commands on |lookup|. |cmd| specifies the
+// command. The other arguments specify the operation in a command-specific way.
+// Use |X509_LOOKUP_load_file| or |X509_LOOKUP_add_dir| instead.
+OPENSSL_EXPORT int X509_LOOKUP_ctrl(X509_LOOKUP *lookup, int cmd,
+                                    const char *argc, long argl, char **ret);
+
+// X509_load_cert_file loads trusted certificates from |file| and adds them to
+// |lookup|'s |X509_STORE|. It returns one on success and zero on error.
+//
+// If |type| is |X509_FILETYPE_ASN1|, it loads a single DER-encoded certificate.
+// If |type| is |X509_FILETYPE_PEM|, it loads a sequence of PEM-encoded
+// certificates. |type| may not be |X509_FILETYPE_DEFAULT|.
+OPENSSL_EXPORT int X509_load_cert_file(X509_LOOKUP *lookup, const char *file,
+                                       int type);
+
+// X509_load_crl_file loads CRLs from |file| and add them it to |lookup|'s
+// |X509_STORE|. It returns one on success and zero on error.
+//
+// If |type| is |X509_FILETYPE_ASN1|, it loads a single DER-encoded CRL. If
+// |type| is |X509_FILETYPE_PEM|, it loads a sequence of PEM-encoded CRLs.
+// |type| may not be |X509_FILETYPE_DEFAULT|.
+OPENSSL_EXPORT int X509_load_crl_file(X509_LOOKUP *lookup, const char *file,
+                                      int type);
+
+// X509_load_cert_crl_file loads CRLs and trusted certificates from |file| and
+// adds them to |lookup|'s |X509_STORE|. It returns one on success and zero on
+// error.
+//
+// If |type| is |X509_FILETYPE_ASN1|, it loads a single DER-encoded certificate.
+// This function cannot be used to load a DER-encoded CRL. If |type| is
+// |X509_FILETYPE_PEM|, it loads a sequence of PEM-encoded certificates and
+// CRLs. |type| may not be |X509_FILETYPE_DEFAULT|.
+OPENSSL_EXPORT int X509_load_cert_crl_file(X509_LOOKUP *lookup,
+                                           const char *file, int type);
+
+// X509_NAME_hash returns a hash of |name|, or zero on error. This is the new
+// hash used by |X509_LOOKUP_add_dir|.
+//
+// This hash is specific to the |X509_LOOKUP_add_dir| filesystem format and is
+// not suitable for general-purpose X.509 name processing. It is very short, so
+// there will be hash collisions. It also depends on an OpenSSL-specific
+// canonicalization process.
+//
+// TODO(https://crbug.com/boringssl/407): This should be const and thread-safe
+// but currently is neither, notably if |name| was modified from its parsed
+// value.
+OPENSSL_EXPORT uint32_t X509_NAME_hash(X509_NAME *name);
+
+// X509_NAME_hash_old returns a hash of |name|, or zero on error. This is the
+// legacy hash used by |X509_LOOKUP_add_dir|, which is still supported for
+// compatibility.
+//
+// This hash is specific to the |X509_LOOKUP_add_dir| filesystem format and is
+// not suitable for general-purpose X.509 name processing. It is very short, so
+// there will be hash collisions.
+//
+// TODO(https://crbug.com/boringssl/407): This should be const and thread-safe
+// but currently is neither, notably if |name| was modified from its parsed
+// value.
+OPENSSL_EXPORT uint32_t X509_NAME_hash_old(X509_NAME *name);
+
+// X509_STORE_set_default_paths configures |store| to read from some "default"
+// filesystem paths. It returns one on success and zero on error. The filesystem
+// paths are determined by a combination of hardcoded paths and the SSL_CERT_DIR
+// and SSL_CERT_FILE environment variables.
+//
+// Using this function is not recommended. In OpenSSL, these defaults are
+// determined by OpenSSL's install prefix. There is no corresponding concept for
+// BoringSSL. Future versions of BoringSSL may change or remove this
+// functionality.
+OPENSSL_EXPORT int X509_STORE_set_default_paths(X509_STORE *store);
+
+// The following functions return filesystem paths used to determine the above
+// "default" paths, when the corresponding environment variables are not set.
+//
+// Using these functions is not recommended. In OpenSSL, these defaults are
+// determined by OpenSSL's install prefix. There is no corresponding concept for
+// BoringSSL. Future versions of BoringSSL may change or remove this
+// functionality.
+OPENSSL_EXPORT const char *X509_get_default_cert_area(void);
+OPENSSL_EXPORT const char *X509_get_default_cert_dir(void);
+OPENSSL_EXPORT const char *X509_get_default_cert_file(void);
+OPENSSL_EXPORT const char *X509_get_default_private_dir(void);
+
+// X509_get_default_cert_dir_env returns "SSL_CERT_DIR", an environment variable
+// used to determine the above "default" paths.
+OPENSSL_EXPORT const char *X509_get_default_cert_dir_env(void);
+
+// X509_get_default_cert_file_env returns "SSL_CERT_FILE", an environment
+// variable used to determine the above "default" paths.
+OPENSSL_EXPORT const char *X509_get_default_cert_file_env(void);
+
+
 // SignedPublicKeyAndChallenge structures.
 //
 // The SignedPublicKeyAndChallenge (SPKAC) is a legacy structure to request
@@ -3821,7 +4016,7 @@
 // X509_issuer_name_hash returns the hash of |x509|'s issuer name with
 // |X509_NAME_hash|.
 //
-// This hash is specific to the |X509_LOOKUP_hash_dir| filesystem format and is
+// This hash is specific to the |X509_LOOKUP_add_dir| filesystem format and is
 // not suitable for general-purpose X.509 name processing. It is very short, so
 // there will be hash collisions. It also depends on an OpenSSL-specific
 // canonicalization process.
@@ -3830,7 +4025,7 @@
 // X509_subject_name_hash returns the hash of |x509|'s subject name with
 // |X509_NAME_hash|.
 //
-// This hash is specific to the |X509_LOOKUP_hash_dir| filesystem format and is
+// This hash is specific to the |X509_LOOKUP_add_dir| filesystem format and is
 // not suitable for general-purpose X.509 name processing. It is very short, so
 // there will be hash collisions. It also depends on an OpenSSL-specific
 // canonicalization process.
@@ -3839,7 +4034,7 @@
 // X509_issuer_name_hash_old returns the hash of |x509|'s issuer name with
 // |X509_NAME_hash_old|.
 //
-// This hash is specific to the |X509_LOOKUP_hash_dir| filesystem format and is
+// This hash is specific to the |X509_LOOKUP_add_dir| filesystem format and is
 // not suitable for general-purpose X.509 name processing. It is very short, so
 // there will be hash collisions.
 OPENSSL_EXPORT uint32_t X509_issuer_name_hash_old(X509 *x509);
@@ -3847,7 +4042,7 @@
 // X509_subject_name_hash_old returns the hash of |x509|'s usjbect name with
 // |X509_NAME_hash_old|.
 //
-// This hash is specific to the |X509_LOOKUP_hash_dir| filesystem format and is
+// This hash is specific to the |X509_LOOKUP_add_dir| filesystem format and is
 // not suitable for general-purpose X.509 name processing. It is very short, so
 // there will be hash collisions.
 OPENSSL_EXPORT uint32_t X509_subject_name_hash_old(X509 *x509);
@@ -4689,7 +4884,7 @@
 // WARNING: This function is not thread-safe. If |store| is shared across
 // multiple threads, callers cannot safely inspect the result of this function,
 // because another thread may have concurrently added to it. In particular,
-// |X509_LOOKUP_hash_dir| treats this list as a cache and may add to it in the
+// |X509_LOOKUP_add_dir| treats this list as a cache and may add to it in the
 // course of certificate verification. This API additionally prevents fixing
 // some quadratic worst-case behavior in |X509_STORE| and may be removed in the
 // future. Use |X509_STORE_get1_objects| instead.
@@ -4735,115 +4930,8 @@
 // TODO(https://crbug.com/boringssl/426): Functions below this point have not
 // yet been documented or organized into sections.
 
-// This stuff is certificate "auxiliary info"
-// it contains details which are useful in certificate
-// stores and databases. When used this is tagged onto
-// the end of the certificate itself
-
 DECLARE_STACK_OF(DIST_POINT)
 
-OPENSSL_EXPORT const char *X509_get_default_cert_area(void);
-OPENSSL_EXPORT const char *X509_get_default_cert_dir(void);
-OPENSSL_EXPORT const char *X509_get_default_cert_file(void);
-OPENSSL_EXPORT const char *X509_get_default_cert_dir_env(void);
-OPENSSL_EXPORT const char *X509_get_default_cert_file_env(void);
-OPENSSL_EXPORT const char *X509_get_default_private_dir(void);
-
-// X509_NAME_hash returns a hash of |name|, or zero on error. This is the new
-// hash used by |X509_LOOKUP_hash_dir|.
-//
-// This hash is specific to the |X509_LOOKUP_hash_dir| filesystem format and is
-// not suitable for general-purpose X.509 name processing. It is very short, so
-// there will be hash collisions. It also depends on an OpenSSL-specific
-// canonicalization process.
-//
-// TODO(https://crbug.com/boringssl/407): This should be const and thread-safe
-// but currently is neither, notably if |name| was modified from its parsed
-// value.
-OPENSSL_EXPORT uint32_t X509_NAME_hash(X509_NAME *name);
-
-// X509_NAME_hash_old returns a hash of |name|, or zero on error. This is the
-// legacy hash used by |X509_LOOKUP_hash_dir|, which is still supported for
-// compatibility.
-//
-// This hash is specific to the |X509_LOOKUP_hash_dir| filesystem format and is
-// not suitable for general-purpose X.509 name processing. It is very short, so
-// there will be hash collisions.
-//
-// TODO(https://crbug.com/boringssl/407): This should be const and thread-safe
-// but currently is neither, notably if |name| was modified from its parsed
-// value.
-OPENSSL_EXPORT uint32_t X509_NAME_hash_old(X509_NAME *name);
-
-/*
-SSL_CTX -> X509_STORE
-                -> X509_LOOKUP
-                        ->X509_LOOKUP_METHOD
-                -> X509_LOOKUP
-                        ->X509_LOOKUP_METHOD
-
-SSL	-> X509_STORE_CTX
-                ->X509_STORE
-
-The X509_STORE holds the tables etc for verification stuff.
-A X509_STORE_CTX is used while validating a single certificate.
-The X509_STORE has X509_LOOKUPs for looking up certs.
-The X509_STORE then calls a function to actually verify the
-certificate chain.
-*/
-
-#define X509_L_FILE_LOAD 1
-#define X509_L_ADD_DIR 2
-
-// The following constants are used to specify the format of files in an
-// |X509_LOOKUP|.
-#define X509_FILETYPE_PEM 1
-#define X509_FILETYPE_ASN1 2
-#define X509_FILETYPE_DEFAULT 3
-
-// X509_LOOKUP_load_file configures |lookup| to load information from the file
-// at |path|. It returns one on success and zero on error. |type| should be one
-// of the |X509_FILETYPE_*| constants to determine if the contents are PEM or
-// DER. If |type| is |X509_FILETYPE_DEFAULT|, |path| is ignored and instead some
-// default system path is used.
-OPENSSL_EXPORT int X509_LOOKUP_load_file(X509_LOOKUP *lookup, const char *path,
-                                         int type);
-
-// X509_LOOKUP_add_dir configures |lookup| to load information from the
-// directory at |path|. It returns one on success and zero on error. |type|
-// should be one of the |X509_FILETYPE_*| constants to determine if the contents
-// are PEM or DER. If |type| is |X509_FILETYPE_DEFAULT|, |path| is ignored and
-// instead some default system path is used.
-//
-// WARNING: |path| is interpreted as a colon-separated (semicolon-separated on
-// Windows) list of paths. It is not possible to configure a path containing the
-// separator character.
-//
-// TODO(crbug.com/boringssl/691): The colon handling is surprising. Consider
-// removing it.
-OPENSSL_EXPORT int X509_LOOKUP_add_dir(X509_LOOKUP *lookup, const char *path,
-                                       int type);
-
-OPENSSL_EXPORT X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *v,
-                                                  const X509_LOOKUP_METHOD *m);
-
-OPENSSL_EXPORT const X509_LOOKUP_METHOD *X509_LOOKUP_hash_dir(void);
-OPENSSL_EXPORT const X509_LOOKUP_METHOD *X509_LOOKUP_file(void);
-
-OPENSSL_EXPORT int X509_LOOKUP_ctrl(X509_LOOKUP *ctx, int cmd, const char *argc,
-                                    long argl, char **ret);
-
-OPENSSL_EXPORT int X509_load_cert_file(X509_LOOKUP *ctx, const char *file,
-                                       int type);
-OPENSSL_EXPORT int X509_load_crl_file(X509_LOOKUP *ctx, const char *file,
-                                      int type);
-OPENSSL_EXPORT int X509_load_cert_crl_file(X509_LOOKUP *ctx, const char *file,
-                                           int type);
-
-OPENSSL_EXPORT int X509_STORE_load_locations(X509_STORE *ctx, const char *file,
-                                             const char *dir);
-OPENSSL_EXPORT int X509_STORE_set_default_paths(X509_STORE *ctx);
-
 struct BASIC_CONSTRAINTS_st {
   ASN1_BOOLEAN ca;
   ASN1_INTEGER *pathlen;
@@ -4952,8 +5040,6 @@
   ASN1_BOOLEAN onlyattr;
 };
 
-// X509_PURPOSE stuff
-
 #define NS_SSL_CLIENT 0x80
 #define NS_SSL_SERVER 0x40
 #define NS_SMIME 0x20