Name |select_certificate_cb| return values

The |select_certificate_cb| return values are somewhat confusing due
to the fact that they don't match the |cert_cb| ones, despite the
similarities between the two callbacks (they both have "certificate" in
the name! well, sort of).

This also documents the error return value (-1) which was previously
undocumented, and it expands the |SSL_CTX_set_select_certificate_cb|
documentation regarding retrial (by shamelessly copying from
|SSL_CTX_set_ticket_aead_method|).

Also updates other scattered documentation that was missed by previous
changes.

Change-Id: Ib962b31d08e6475e09954cbc3c939988b0ba13f7
Reviewed-on: https://boringssl-review.googlesource.com/14245
Reviewed-by: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
diff --git a/ssl/handshake_server.c b/ssl/handshake_server.c
index 81e45ef..fd6c8e9 100644
--- a/ssl/handshake_server.c
+++ b/ssl/handshake_server.c
@@ -812,11 +812,11 @@
   /* Run the early callback. */
   if (ssl->ctx->select_certificate_cb != NULL) {
     switch (ssl->ctx->select_certificate_cb(&client_hello)) {
-      case 0:
+      case ssl_select_cert_retry:
         ssl->rwstate = SSL_CERTIFICATE_SELECTION_PENDING;
         return -1;
 
-      case -1:
+      case ssl_select_cert_error:
         /* Connection rejected. */
         OPENSSL_PUT_ERROR(SSL, SSL_R_CONNECTION_REJECTED);
         ssl3_send_alert(ssl, SSL3_AL_FATAL, SSL_AD_HANDSHAKE_FAILURE);
diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index d16c952..e3bcb91 100644
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -2397,8 +2397,9 @@
 
 int SSL_is_dtls(const SSL *ssl) { return ssl->method->is_dtls; }
 
-void SSL_CTX_set_select_certificate_cb(SSL_CTX *ctx,
-                                       int (*cb)(const SSL_CLIENT_HELLO *)) {
+void SSL_CTX_set_select_certificate_cb(
+    SSL_CTX *ctx,
+    enum ssl_select_cert_result_t (*cb)(const SSL_CLIENT_HELLO *)) {
   ctx->select_certificate_cb = cb;
 }
 
diff --git a/ssl/ssl_test.cc b/ssl/ssl_test.cc
index 411ddb7..6678b57 100644
--- a/ssl/ssl_test.cc
+++ b/ssl/ssl_test.cc
@@ -2144,17 +2144,6 @@
   return SSL_TLSEXT_ERR_OK;
 }
 
-static int SwitchSessionIDContextEarly(const SSL_CLIENT_HELLO *client_hello) {
-  static const uint8_t kContext[] = {3};
-
-  if (!SSL_set_session_id_context(client_hello->ssl, kContext,
-                                  sizeof(kContext))) {
-    return -1;
-  }
-
-  return 1;
-}
-
 static bool TestSessionIDContext(bool is_dtls, const SSL_METHOD *method,
                                  uint16_t version) {
   bssl::UniquePtr<X509> cert = GetTestCertificate();
@@ -2226,8 +2215,18 @@
 
   // Switch the session ID context with the early callback instead.
   SSL_CTX_set_tlsext_servername_callback(server_ctx.get(), nullptr);
-  SSL_CTX_set_select_certificate_cb(server_ctx.get(),
-                                    SwitchSessionIDContextEarly);
+  SSL_CTX_set_select_certificate_cb(
+      server_ctx.get(),
+      [](const SSL_CLIENT_HELLO *client_hello) -> ssl_select_cert_result_t {
+        static const uint8_t kContext[] = {3};
+
+        if (!SSL_set_session_id_context(client_hello->ssl, kContext,
+                                        sizeof(kContext))) {
+          return ssl_select_cert_error;
+        }
+
+        return ssl_select_cert_success;
+      });
 
   if (!ExpectSessionReused(client_ctx.get(), server_ctx.get(), session.get(),
                            false /* expect session not reused */)) {
@@ -2614,12 +2613,13 @@
   ASSERT_TRUE(SSL_CTX_set_max_proto_version(server_ctx.get(), TLS1_3_VERSION));
 
   SSL_CTX_set_select_certificate_cb(
-      server_ctx.get(), [](const SSL_CLIENT_HELLO *client_hello) -> int {
+      server_ctx.get(),
+      [](const SSL_CLIENT_HELLO *client_hello) -> ssl_select_cert_result_t {
         if (!SSL_set_max_proto_version(client_hello->ssl, TLS1_2_VERSION)) {
-          return -1;
+          return ssl_select_cert_error;
         }
 
-        return 1;
+        return ssl_select_cert_success;
       });
 
   bssl::UniquePtr<SSL> client, server;
diff --git a/ssl/test/bssl_shim.cc b/ssl/test/bssl_shim.cc
index 804cbbb..2c70361 100644
--- a/ssl/test/bssl_shim.cc
+++ b/ssl/test/bssl_shim.cc
@@ -533,7 +533,8 @@
   return true;
 }
 
-static int SelectCertificateCallback(const SSL_CLIENT_HELLO *client_hello) {
+static enum ssl_select_cert_result_t SelectCertificateCallback(
+    const SSL_CLIENT_HELLO *client_hello) {
   const TestConfig *config = GetTestConfig(client_hello->ssl);
   GetTestState(client_hello->ssl)->early_callback_called = true;
 
@@ -547,7 +548,7 @@
             client_hello, TLSEXT_TYPE_server_name, &extension_data,
             &extension_len)) {
       fprintf(stderr, "Could not find server_name extension.\n");
-      return -1;
+      return ssl_select_cert_error;
     }
 
     CBS_init(&extension, extension_data, extension_len);
@@ -558,7 +559,7 @@
         !CBS_get_u16_length_prefixed(&server_name_list, &host_name) ||
         CBS_len(&server_name_list) != 0) {
       fprintf(stderr, "Could not decode server_name extension.\n");
-      return -1;
+      return ssl_select_cert_error;
     }
 
     if (!CBS_mem_equal(&host_name,
@@ -569,7 +570,7 @@
   }
 
   if (config->fail_early_callback) {
-    return -1;
+    return ssl_select_cert_error;
   }
 
   // Install the certificate in the early callback.
@@ -578,13 +579,13 @@
         GetTestState(client_hello->ssl)->early_callback_ready;
     if (config->async && !early_callback_ready) {
       // Install the certificate asynchronously.
-      return 0;
+      return ssl_select_cert_retry;
     }
     if (!InstallCertificate(client_hello->ssl)) {
-      return -1;
+      return ssl_select_cert_error;
     }
   }
-  return 1;
+  return ssl_select_cert_success;
 }
 
 static bool CheckCertificateRequest(SSL *ssl) {