Don't use the buffer BIO in TLS.

On the TLS side, we introduce a running buffer of ciphertext. Queuing up
pending data consists of encrypting the record into the buffer. This
effectively reimplements what the buffer BIO was doing previously, but
this resizes to fit the whole flight.

As part of this, rename all the functions to add to the pending flight
to be more uniform. This CL proposes "add_foo" to add to the pending
flight and "flush_flight" to drain it.

We add an add_alert hook for alerts but, for now, only the SSL 3.0
warning alert (sent mid-handshake) uses this mechanism.  Later work will
push this down to the rest of the write path so closure alerts use it
too, as in DTLS. The intended end state is that all the ssl_buffer.c and
wpend_ret logic will only be used for application data and eventually
optionally replaced by the in-place API, while all "incidental" data
will be handled internally.

For now, the two buffers are mutually exclusive. Moving closure alerts
to "incidentals" will change this, but flushing application data early
is tricky due to wpend_ret. (If we call ssl_write_buffer_flush,
do_ssl3_write doesn't realize it still has a wpend_ret to replay.) That
too is all left alone in this change.

To keep the diff down, write_message is retained for now and will be
removed from the state machines in a follow-up change.

BUG=72

Change-Id: Ibce882f5f7196880648f25d5005322ca4055c71d
Reviewed-on: https://boringssl-review.googlesource.com/13224
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/ssl/d1_both.c b/ssl/d1_both.c
index 1d89636..bbb6b32 100644
--- a/ssl/d1_both.c
+++ b/ssl/d1_both.c
@@ -579,7 +579,7 @@
   return 1;
 }
 
-int dtls1_queue_message(SSL *ssl, uint8_t *data, size_t len) {
+int dtls1_add_message(SSL *ssl, uint8_t *data, size_t len) {
   return add_outgoing(ssl, 0 /* handshake */, data, len);
 }
 
@@ -588,10 +588,19 @@
   return 1;
 }
 
-int dtls1_send_change_cipher_spec(SSL *ssl) {
+int dtls1_add_change_cipher_spec(SSL *ssl) {
   return add_outgoing(ssl, 1 /* ChangeCipherSpec */, NULL, 0);
 }
 
+int dtls1_add_alert(SSL *ssl, uint8_t level, uint8_t desc) {
+  /* The |add_alert| path is only used for warning alerts for now, which DTLS
+   * never sends. This will be implemented later once closure alerts are
+   * converted. */
+  assert(0);
+  OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
+  return 0;
+}
+
 /* dtls1_update_mtu updates the current MTU from the BIO, ensuring it is above
  * the minimum. */
 static void dtls1_update_mtu(SSL *ssl) {
diff --git a/ssl/dtls_method.c b/ssl/dtls_method.c
index a774c82..e7cd21e 100644
--- a/ssl/dtls_method.c
+++ b/ssl/dtls_method.c
@@ -151,14 +151,15 @@
     dtls1_supports_cipher,
     dtls1_init_message,
     dtls1_finish_message,
-    dtls1_queue_message,
-    dtls1_write_message,
-    dtls1_send_change_cipher_spec,
+    dtls1_add_message,
+    dtls1_add_change_cipher_spec,
+    dtls1_add_alert,
     dtls1_flush_flight,
     dtls1_expect_flight,
     dtls1_received_flight,
     dtls1_set_read_state,
     dtls1_set_write_state,
+    dtls1_write_message,
 };
 
 const SSL_METHOD *DTLS_method(void) {
diff --git a/ssl/handshake_client.c b/ssl/handshake_client.c
index 3389d6c..29b59d4 100644
--- a/ssl/handshake_client.c
+++ b/ssl/handshake_client.c
@@ -206,12 +206,6 @@
 
       case SSL_ST_CONNECT:
         ssl_do_info_callback(ssl, SSL_CB_HANDSHAKE_START, 1);
-
-        if (!ssl_init_wbio_buffer(ssl)) {
-          ret = -1;
-          goto end;
-        }
-
         hs->state = SSL3_ST_CW_CLNT_HELLO_A;
         break;
 
@@ -362,18 +356,13 @@
         break;
 
       case SSL3_ST_CW_CHANGE:
-        ret = ssl->method->send_change_cipher_spec(ssl);
-        if (ret <= 0) {
-          goto end;
-        }
-
-        hs->state = SSL3_ST_CW_NEXT_PROTO_A;
-
-        if (!tls1_change_cipher_state(hs, SSL3_CHANGE_CIPHER_CLIENT_WRITE)) {
+        if (!ssl->method->add_change_cipher_spec(ssl) ||
+            !tls1_change_cipher_state(hs, SSL3_CHANGE_CIPHER_CLIENT_WRITE)) {
           ret = -1;
           goto end;
         }
 
+        hs->state = SSL3_ST_CW_NEXT_PROTO_A;
         break;
 
       case SSL3_ST_CW_NEXT_PROTO_A:
@@ -437,8 +426,6 @@
       case SSL3_ST_FALSE_START:
         hs->state = SSL3_ST_CR_SESSION_TICKET_A;
         hs->in_false_start = 1;
-
-        ssl_free_wbio_buffer(ssl);
         ret = 1;
         goto end;
 
@@ -527,9 +514,6 @@
           ssl->s3->new_session = NULL;
         }
 
-        /* Remove write buffering now. */
-        ssl_free_wbio_buffer(ssl);
-
         const int is_initial_handshake = !ssl->s3->initial_handshake_complete;
         ssl->s3->initial_handshake_complete = 1;
         if (is_initial_handshake) {
@@ -713,7 +697,7 @@
     goto err;
   }
 
-  return ssl->method->queue_message(ssl, msg, len);
+  return ssl->method->add_message(ssl, msg, len);
 
  err:
   CBB_cleanup(&cbb);
@@ -1469,9 +1453,12 @@
     /* Without a client certificate, the handshake buffer may be released. */
     ssl3_free_handshake_buffer(ssl);
 
+    /* In SSL 3.0, the Certificate message is replaced with a warning alert. */
     if (ssl->version == SSL3_VERSION) {
-      /* In SSL 3.0, send no certificate by skipping both messages. */
-      ssl3_send_alert(ssl, SSL3_AL_WARNING, SSL_AD_NO_CERTIFICATE);
+      if (!ssl->method->add_alert(ssl, SSL3_AL_WARNING,
+                                  SSL_AD_NO_CERTIFICATE)) {
+        return -1;
+      }
       return 1;
     }
   }
@@ -1647,7 +1634,7 @@
 
   /* The message must be added to the finished hash before calculating the
    * master secret. */
-  if (!ssl_complete_message(ssl, &cbb)) {
+  if (!ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
   hs->state = SSL3_ST_CW_KEY_EXCH_B;
@@ -1765,7 +1752,7 @@
   }
 
   if (!CBB_did_write(&child, sig_len) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
 
@@ -1795,7 +1782,7 @@
                      ssl->s3->next_proto_negotiated_len) ||
       !CBB_add_u8_length_prefixed(&body, &child) ||
       !CBB_add_bytes(&child, kZero, padding_len) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
     CBB_cleanup(&cbb);
     return -1;
@@ -1825,7 +1812,7 @@
   CBB cbb, body;
   if (!ssl->method->init_message(ssl, &cbb, &body, SSL3_MT_CHANNEL_ID) ||
       !tls1_write_channel_id(ssl, &body) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
     CBB_cleanup(&cbb);
     return -1;
diff --git a/ssl/handshake_server.c b/ssl/handshake_server.c
index a4396f4..00b374c 100644
--- a/ssl/handshake_server.c
+++ b/ssl/handshake_server.c
@@ -206,13 +206,6 @@
       case SSL_ST_ACCEPT:
         ssl_do_info_callback(ssl, SSL_CB_HANDSHAKE_START, 1);
 
-        /* Enable a write buffer. This groups handshake messages within a flight
-         * into a single write. */
-        if (!ssl_init_wbio_buffer(ssl)) {
-          ret = -1;
-          goto end;
-        }
-
         if (!ssl3_init_handshake_buffer(ssl)) {
           OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
           ret = -1;
@@ -423,16 +416,13 @@
         break;
 
       case SSL3_ST_SW_CHANGE:
-        ret = ssl->method->send_change_cipher_spec(ssl);
-        if (ret <= 0) {
-          goto end;
-        }
-        hs->state = SSL3_ST_SW_FINISHED_A;
-
-        if (!tls1_change_cipher_state(hs, SSL3_CHANGE_CIPHER_SERVER_WRITE)) {
+        if (!ssl->method->add_change_cipher_spec(ssl) ||
+            !tls1_change_cipher_state(hs, SSL3_CHANGE_CIPHER_SERVER_WRITE)) {
           ret = -1;
           goto end;
         }
+
+        hs->state = SSL3_ST_SW_FINISHED_A;
         break;
 
       case SSL3_ST_SW_FINISHED_A:
@@ -493,9 +483,6 @@
           ssl->s3->new_session = NULL;
         }
 
-        /* remove buffering on output */
-        ssl_free_wbio_buffer(ssl);
-
         ssl->s3->initial_handshake_complete = 1;
         ssl_update_cache(hs, SSL_SESS_CACHE_SERVER);
 
@@ -1101,7 +1088,7 @@
       !CBB_add_u16(&body, ssl_cipher_get_value(ssl->s3->tmp.new_cipher)) ||
       !CBB_add_u8(&body, 0 /* no compression */) ||
       !ssl_add_serverhello_tlsext(hs, &body) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
     CBB_cleanup(&cbb);
     return -1;
@@ -1142,7 +1129,7 @@
       !CBB_add_u24_length_prefixed(&body, &ocsp_response) ||
       !CBB_add_bytes(&ocsp_response, CRYPTO_BUFFER_data(ssl->ocsp_response),
                      CRYPTO_BUFFER_len(ssl->ocsp_response)) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
     CBB_cleanup(&cbb);
     return -1;
@@ -1317,7 +1304,7 @@
     }
   }
 
-  if (!ssl_complete_message(ssl, &cbb)) {
+  if (!ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
 
@@ -1400,7 +1387,7 @@
   }
 
   if (!ssl_add_client_CA_list(ssl, &body) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
 
@@ -1421,7 +1408,7 @@
 
   CBB cbb, body;
   if (!ssl->method->init_message(ssl, &cbb, &body, SSL3_MT_SERVER_HELLO_DONE) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
     CBB_cleanup(&cbb);
     return -1;
@@ -1970,7 +1957,7 @@
       CBB_add_u32(&body, session->timeout) &&
       CBB_add_u16_length_prefixed(&body, &ticket) &&
       ssl_encrypt_ticket(ssl, &ticket, session) &&
-      ssl_complete_message(ssl, &cbb);
+      ssl_add_message_cbb(ssl, &cbb);
 
   SSL_SESSION_free(session_copy);
   CBB_cleanup(&cbb);
diff --git a/ssl/internal.h b/ssl/internal.h
index ffe4d62..877822d 100644
--- a/ssl/internal.h
+++ b/ssl/internal.h
@@ -1333,16 +1333,17 @@
    * release it with |OPENSSL_free| when done. It returns one on success and
    * zero on error. */
   int (*finish_message)(SSL *ssl, CBB *cbb, uint8_t **out_msg, size_t *out_len);
-  /* queue_message queues a handshake message and prepares it to be written. It
-   * takes ownership of |msg| and releases it with |OPENSSL_free| when done. It
-   * returns one on success and zero on error. */
-  int (*queue_message)(SSL *ssl, uint8_t *msg, size_t len);
-  /* write_message writes the next message to the transport. It returns one on
-   * success and <= 0 on error. */
-  int (*write_message)(SSL *ssl);
-  /* send_change_cipher_spec sends a ChangeCipherSpec message. */
-  int (*send_change_cipher_spec)(SSL *ssl);
-  /* flush_flight flushes the current flight to the transport. It returns one on
+  /* add_message adds a handshake message to the pending flight. It returns one
+   * on success and zero on error. In either case, it takes ownership of |msg|
+   * and releases it with |OPENSSL_free| when done. */
+  int (*add_message)(SSL *ssl, uint8_t *msg, size_t len);
+  /* add_change_cipher_spec adds a ChangeCipherSpec record to the pending
+   * flight. It returns one on success and zero on error. */
+  int (*add_change_cipher_spec)(SSL *ssl);
+  /* add_alert adds an alert to the pending flight. It returns one on success
+   * and zero on error. */
+  int (*add_alert)(SSL *ssl, uint8_t level, uint8_t desc);
+  /* flush_flight flushes the pending flight to the transport. It returns one on
    * success and <= 0 on error. */
   int (*flush_flight)(SSL *ssl);
   /* expect_flight is called when the handshake expects a flight of messages from
@@ -1359,6 +1360,8 @@
    * ownership of |aead_ctx|. It returns one on success and zero if changing the
    * write state is forbidden at this point. */
   int (*set_write_state)(SSL *ssl, SSL_AEAD_CTX *aead_ctx);
+  /* write_message returns one. */
+  int (*write_message)(SSL *ssl);
 };
 
 /* This is for the SSLv3/TLSv1.0 differences in crypto/hash stuff It is a bit
@@ -1491,9 +1494,13 @@
 
   uint8_t send_alert[2];
 
-  /* pending_message is the current outgoing handshake message. */
-  uint8_t *pending_message;
-  uint32_t pending_message_len;
+  /* pending_flight is the pending outgoing flight. This is used to flush each
+   * handshake flight in a single write. */
+  BUF_MEM *pending_flight;
+
+  /* pending_flight_offset is the number of bytes of |pending_flight| which have
+   * been successfully written. */
+  uint32_t pending_flight_offset;
 
   /* aead_read_ctx is the current read cipher state. */
   SSL_AEAD_CTX *aead_read_ctx;
@@ -1750,7 +1757,6 @@
 int ssl_verify_alarm_type(long type);
 
 int ssl3_get_finished(SSL_HANDSHAKE *hs);
-int ssl3_send_change_cipher_spec(SSL *ssl);
 int ssl3_send_alert(SSL *ssl, int level, int desc);
 int ssl3_get_message(SSL *ssl, int msg_type,
                      enum ssl_hash_message_t hash_message);
@@ -1782,20 +1788,24 @@
 
 int ssl3_init_message(SSL *ssl, CBB *cbb, CBB *body, uint8_t type);
 int ssl3_finish_message(SSL *ssl, CBB *cbb, uint8_t **out_msg, size_t *out_len);
-int ssl3_queue_message(SSL *ssl, uint8_t *msg, size_t len);
+int ssl3_add_message(SSL *ssl, uint8_t *msg, size_t len);
+int ssl3_add_change_cipher_spec(SSL *ssl);
+int ssl3_add_alert(SSL *ssl, uint8_t level, uint8_t desc);
 int ssl3_write_message(SSL *ssl);
+int ssl3_flush_flight(SSL *ssl);
 
 int dtls1_init_message(SSL *ssl, CBB *cbb, CBB *body, uint8_t type);
 int dtls1_finish_message(SSL *ssl, CBB *cbb, uint8_t **out_msg,
                          size_t *out_len);
-int dtls1_queue_message(SSL *ssl, uint8_t *msg, size_t len);
+int dtls1_add_message(SSL *ssl, uint8_t *msg, size_t len);
+int dtls1_add_change_cipher_spec(SSL *ssl);
+int dtls1_add_alert(SSL *ssl, uint8_t level, uint8_t desc);
 int dtls1_write_message(SSL *ssl);
-int dtls1_send_change_cipher_spec(SSL *ssl);
 int dtls1_flush_flight(SSL *ssl);
 
-/* ssl_complete_message calls |finish_message| and |queue_message| on |cbb| to
- * queue the message for writing. */
-int ssl_complete_message(SSL *ssl, CBB *cbb);
+/* ssl_add_message_cbb finishes the handshake message in |cbb| and adds it to
+ * the pending flight. It returns one on success and zero on error. */
+int ssl_add_message_cbb(SSL *ssl, CBB *cbb);
 
 /* ssl_hash_current_message incorporates the current handshake message into the
  * handshake hash. It returns one on success and zero on allocation failure. */
@@ -1842,13 +1852,6 @@
 void dtls1_release_current_message(SSL *ssl, int free_buffer);
 int dtls1_dispatch_alert(SSL *ssl);
 
-/* ssl_is_wbio_buffered returns one if |ssl|'s write BIO is buffered and zero
- * otherwise. */
-int ssl_is_wbio_buffered(const SSL *ssl);
-
-int ssl_init_wbio_buffer(SSL *ssl);
-void ssl_free_wbio_buffer(SSL *ssl);
-
 int tls1_change_cipher_state(SSL_HANDSHAKE *hs, int which);
 int tls1_handshake_digest(SSL *ssl, uint8_t *out, size_t out_len);
 int tls1_generate_master_secret(SSL *ssl, uint8_t *out, const uint8_t *premaster,
diff --git a/ssl/s3_both.c b/ssl/s3_both.c
index 492884f..133cd2d 100644
--- a/ssl/s3_both.c
+++ b/ssl/s3_both.c
@@ -180,33 +180,44 @@
   OPENSSL_free(hs);
 }
 
-/* ssl3_do_write sends |ssl->init_buf| in records of type 'type'
- * (SSL3_RT_HANDSHAKE or SSL3_RT_CHANGE_CIPHER_SPEC). It returns 1 on success
- * and <= 0 on error. */
-static int ssl3_do_write(SSL *ssl, int type, const uint8_t *data, size_t len) {
-  int ret = ssl3_write_bytes(ssl, type, data, len);
-  if (ret <= 0) {
-    return ret;
+static int add_record_to_flight(SSL *ssl, uint8_t type, const uint8_t *in,
+                                size_t in_len) {
+  /* We'll never add a flight while in the process of writing it out. */
+  assert(ssl->s3->pending_flight_offset == 0);
+
+  if (ssl->s3->pending_flight == NULL) {
+    ssl->s3->pending_flight = BUF_MEM_new();
+    if (ssl->s3->pending_flight == NULL) {
+      return 0;
+    }
   }
 
-  /* ssl3_write_bytes writes the data in its entirety. */
-  assert((size_t)ret == len);
-  ssl_do_msg_callback(ssl, 1 /* write */, type, data, len);
+  size_t max_out = in_len + SSL_max_seal_overhead(ssl);
+  size_t new_cap = ssl->s3->pending_flight->length + max_out;
+  if (max_out < in_len || new_cap < max_out) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_OVERFLOW);
+    return 0;
+  }
+
+  size_t len;
+  if (!BUF_MEM_reserve(ssl->s3->pending_flight, new_cap) ||
+      !tls_seal_record(ssl, (uint8_t *)ssl->s3->pending_flight->data +
+                                ssl->s3->pending_flight->length,
+                       &len, max_out, type, in, in_len)) {
+    return 0;
+  }
+
+  ssl->s3->pending_flight->length += len;
   return 1;
 }
 
 int ssl3_init_message(SSL *ssl, CBB *cbb, CBB *body, uint8_t type) {
-  CBB_zero(cbb);
-  if (ssl->s3->pending_message != NULL) {
-    OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
-    return 0;
-  }
-
   /* Pick a modest size hint to save most of the |realloc| calls. */
   if (!CBB_init(cbb, 64) ||
       !CBB_add_u8(cbb, type) ||
       !CBB_add_u24_length_prefixed(cbb, body)) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
+    CBB_cleanup(cbb);
     return 0;
   }
 
@@ -223,47 +234,111 @@
   return 1;
 }
 
-int ssl3_queue_message(SSL *ssl, uint8_t *msg, size_t len) {
-  if (ssl->s3->pending_message != NULL ||
-      len > 0xffffffffu) {
-    OPENSSL_free(msg);
-    OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
+int ssl3_add_message(SSL *ssl, uint8_t *msg, size_t len) {
+  /* Add the message to the current flight, splitting into several records if
+   * needed. */
+  int ret = 0;
+  size_t added = 0;
+  do {
+    size_t todo = len - added;
+    if (todo > ssl->max_send_fragment) {
+      todo = ssl->max_send_fragment;
+    }
+
+    if (!add_record_to_flight(ssl, SSL3_RT_HANDSHAKE, msg + added, todo)) {
+      goto err;
+    }
+    added += todo;
+  } while (added < len);
+
+  ssl_do_msg_callback(ssl, 1 /* write */, SSL3_RT_HANDSHAKE, msg, len);
+  ssl3_update_handshake_hash(ssl, msg, len);
+  ret = 1;
+
+err:
+  OPENSSL_free(msg);
+  return ret;
+}
+
+int ssl3_add_change_cipher_spec(SSL *ssl) {
+  static const uint8_t kChangeCipherSpec[1] = {SSL3_MT_CCS};
+
+  if (!add_record_to_flight(ssl, SSL3_RT_CHANGE_CIPHER_SPEC, kChangeCipherSpec,
+                            sizeof(kChangeCipherSpec))) {
     return 0;
   }
 
-  ssl3_update_handshake_hash(ssl, msg, len);
-
-  ssl->s3->pending_message = msg;
-  ssl->s3->pending_message_len = (uint32_t)len;
+  ssl_do_msg_callback(ssl, 1 /* write */, SSL3_RT_CHANGE_CIPHER_SPEC,
+                      kChangeCipherSpec, sizeof(kChangeCipherSpec));
   return 1;
 }
 
-int ssl_complete_message(SSL *ssl, CBB *cbb) {
+int ssl3_add_alert(SSL *ssl, uint8_t level, uint8_t desc) {
+  uint8_t alert[2] = {level, desc};
+  if (!add_record_to_flight(ssl, SSL3_RT_ALERT, alert, sizeof(alert))) {
+    return 0;
+  }
+
+  ssl_do_msg_callback(ssl, 1 /* write */, SSL3_RT_ALERT, alert, sizeof(alert));
+  ssl_do_info_callback(ssl, SSL_CB_WRITE_ALERT, ((int)level << 8) | desc);
+  return 1;
+}
+
+int ssl_add_message_cbb(SSL *ssl, CBB *cbb) {
   uint8_t *msg;
   size_t len;
   if (!ssl->method->finish_message(ssl, cbb, &msg, &len) ||
-      !ssl->method->queue_message(ssl, msg, len)) {
+      !ssl->method->add_message(ssl, msg, len)) {
     return 0;
   }
 
   return 1;
 }
 
-int ssl3_write_message(SSL *ssl) {
-  if (ssl->s3->pending_message == NULL) {
-    OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
-    return 0;
+int ssl3_write_message(SSL *ssl) { return 1; }
+
+int ssl3_flush_flight(SSL *ssl) {
+  if (ssl->s3->pending_flight == NULL) {
+    return 1;
   }
 
-  int ret = ssl3_do_write(ssl, SSL3_RT_HANDSHAKE, ssl->s3->pending_message,
-                          ssl->s3->pending_message_len);
+  if (ssl->s3->pending_flight->length > 0xffffffff ||
+      ssl->s3->pending_flight->length > INT_MAX) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
+    return -1;
+  }
+
+  /* The handshake flight buffer is mutually exclusive with application data.
+   *
+   * TODO(davidben): This will not be true when closure alerts use this. */
+  if (ssl_write_buffer_is_pending(ssl)) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
+    return -1;
+  }
+
+  /* Write the pending flight. */
+  while (ssl->s3->pending_flight_offset < ssl->s3->pending_flight->length) {
+    int ret = BIO_write(
+        ssl->wbio,
+        ssl->s3->pending_flight->data + ssl->s3->pending_flight_offset,
+        ssl->s3->pending_flight->length - ssl->s3->pending_flight_offset);
+    if (ret <= 0) {
+      ssl->rwstate = SSL_WRITING;
+      return ret;
+    }
+
+    ssl->s3->pending_flight_offset += ret;
+  }
+
+  int ret = BIO_flush(ssl->wbio);
   if (ret <= 0) {
+    ssl->rwstate = SSL_WRITING;
     return ret;
   }
 
-  OPENSSL_free(ssl->s3->pending_message);
-  ssl->s3->pending_message = NULL;
-  ssl->s3->pending_message_len = 0;
+  BUF_MEM_free(ssl->s3->pending_flight);
+  ssl->s3->pending_flight = NULL;
+  ssl->s3->pending_flight_offset = 0;
   return 1;
 }
 
@@ -307,7 +382,7 @@
   CBB cbb, body;
   if (!ssl->method->init_message(ssl, &cbb, &body, SSL3_MT_FINISHED) ||
       !CBB_add_bytes(&body, finished, finished_len) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
     CBB_cleanup(&cbb);
     return -1;
@@ -365,18 +440,11 @@
   return 1;
 }
 
-int ssl3_send_change_cipher_spec(SSL *ssl) {
-  static const uint8_t kChangeCipherSpec[1] = {SSL3_MT_CCS};
-
-  return ssl3_do_write(ssl, SSL3_RT_CHANGE_CIPHER_SPEC, kChangeCipherSpec,
-                       sizeof(kChangeCipherSpec));
-}
-
 int ssl3_output_cert_chain(SSL *ssl) {
   CBB cbb, body;
   if (!ssl->method->init_message(ssl, &cbb, &body, SSL3_MT_CERTIFICATE) ||
       !ssl_add_cert_chain(ssl, &body) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
     CBB_cleanup(&cbb);
     return 0;
diff --git a/ssl/s3_lib.c b/ssl/s3_lib.c
index 7039418..3f44629 100644
--- a/ssl/s3_lib.c
+++ b/ssl/s3_lib.c
@@ -209,7 +209,7 @@
   OPENSSL_free(ssl->s3->alpn_selected);
   SSL_AEAD_CTX_free(ssl->s3->aead_read_ctx);
   SSL_AEAD_CTX_free(ssl->s3->aead_write_ctx);
-  OPENSSL_free(ssl->s3->pending_message);
+  BUF_MEM_free(ssl->s3->pending_flight);
 
   OPENSSL_cleanse(ssl->s3, sizeof *ssl->s3);
   OPENSSL_free(ssl->s3);
diff --git a/ssl/s3_pkt.c b/ssl/s3_pkt.c
index 9bd9f1f..5a31e1f 100644
--- a/ssl/s3_pkt.c
+++ b/ssl/s3_pkt.c
@@ -267,6 +267,14 @@
     return ssl3_write_pending(ssl, type, buf, len);
   }
 
+  /* The handshake flight buffer is mutually exclusive with application data.
+   *
+   * TODO(davidben): This will not be true when closure alerts use this. */
+  if (ssl->s3->pending_flight != NULL) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_INTERNAL_ERROR);
+    return -1;
+  }
+
   /* If we have an alert to send, lets send it */
   if (ssl->s3->alert_dispatch) {
     int ret = ssl->method->dispatch_alert(ssl);
diff --git a/ssl/ssl_lib.c b/ssl/ssl_lib.c
index d653378..101ee4d 100644
--- a/ssl/ssl_lib.c
+++ b/ssl/ssl_lib.c
@@ -504,9 +504,6 @@
 
   CRYPTO_free_ex_data(&g_ex_data_class_ssl, ssl, &ssl->ex_data);
 
-  ssl_free_wbio_buffer(ssl);
-  assert(ssl->bbio == NULL);
-
   BIO_free_all(ssl->rbio);
   BIO_free_all(ssl->wbio);
 
@@ -553,18 +550,8 @@
 }
 
 void SSL_set0_wbio(SSL *ssl, BIO *wbio) {
-  /* If the output buffering BIO is still in place, remove it. */
-  if (ssl->bbio != NULL) {
-    ssl->wbio = BIO_pop(ssl->wbio);
-  }
-
   BIO_free_all(ssl->wbio);
   ssl->wbio = wbio;
-
-  /* Re-attach |bbio| to the new |wbio|. */
-  if (ssl->bbio != NULL) {
-    ssl->wbio = BIO_push(ssl->bbio, ssl->wbio);
-  }
 }
 
 void SSL_set_bio(SSL *ssl, BIO *rbio, BIO *wbio) {
@@ -603,14 +590,7 @@
 
 BIO *SSL_get_rbio(const SSL *ssl) { return ssl->rbio; }
 
-BIO *SSL_get_wbio(const SSL *ssl) {
-  if (ssl->bbio != NULL) {
-    /* If |bbio| is active, the true caller-configured BIO is its |next_bio|. */
-    assert(ssl->bbio == ssl->wbio);
-    return ssl->bbio->next_bio;
-  }
-  return ssl->wbio;
-}
+BIO *SSL_get_wbio(const SSL *ssl) { return ssl->wbio; }
 
 void ssl_reset_error_state(SSL *ssl) {
   /* Functions which use |SSL_get_error| must reset I/O and error state on
@@ -2023,48 +2003,6 @@
 
 int *SSL_get_server_tmp_key(SSL *ssl, EVP_PKEY **out_key) { return 0; }
 
-int ssl_is_wbio_buffered(const SSL *ssl) {
-  return ssl->bbio != NULL;
-}
-
-int ssl_init_wbio_buffer(SSL *ssl) {
-  if (SSL_is_dtls(ssl)) {
-    /* DTLS does not use the BIO buffer.
-     * TODO(davidben): Remove this altogether when TLS no longer uses it.
-     * https://crbug.com/boringssl/72. */
-    return 1;
-  }
-
-  if (ssl->bbio != NULL) {
-    /* Already buffered. */
-    assert(ssl->bbio == ssl->wbio);
-    return 1;
-  }
-
-  BIO *bbio = BIO_new(BIO_f_buffer());
-  if (bbio == NULL ||
-      !BIO_set_read_buffer_size(bbio, 1)) {
-    BIO_free(bbio);
-    return 0;
-  }
-
-  ssl->bbio = bbio;
-  ssl->wbio = BIO_push(bbio, ssl->wbio);
-  return 1;
-}
-
-void ssl_free_wbio_buffer(SSL *ssl) {
-  if (ssl->bbio == NULL) {
-    return;
-  }
-
-  assert(ssl->bbio == ssl->wbio);
-
-  ssl->wbio = BIO_pop(ssl->wbio);
-  BIO_free(ssl->bbio);
-  ssl->bbio = NULL;
-}
-
 void SSL_CTX_set_quiet_shutdown(SSL_CTX *ctx, int mode) {
   ctx->quiet_shutdown = (mode != 0);
 }
diff --git a/ssl/tls13_both.c b/ssl/tls13_both.c
index 1425665..3805bb1 100644
--- a/ssl/tls13_both.c
+++ b/ssl/tls13_both.c
@@ -453,7 +453,7 @@
   }
 
   if (!ssl_has_certificate(ssl)) {
-    if (!ssl_complete_message(ssl, &cbb)) {
+    if (!ssl_add_message_cbb(ssl, &cbb)) {
       goto err;
     }
 
@@ -507,7 +507,7 @@
     }
   }
 
-  if (!ssl_complete_message(ssl, &cbb)) {
+  if (!ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
 
@@ -569,7 +569,7 @@
   }
 
   if (!CBB_did_write(&child, sig_len) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
 
@@ -595,7 +595,7 @@
   CBB cbb, body;
   if (!ssl->method->init_message(ssl, &cbb, &body, SSL3_MT_FINISHED) ||
       !CBB_add_bytes(&body, verify_data, verify_data_len) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     CBB_cleanup(&cbb);
     return 0;
   }
diff --git a/ssl/tls13_client.c b/ssl/tls13_client.c
index 6f2bb21..2d46bf2 100644
--- a/ssl/tls13_client.c
+++ b/ssl/tls13_client.c
@@ -542,7 +542,7 @@
   CBB cbb, body;
   if (!ssl->method->init_message(ssl, &cbb, &body, SSL3_MT_CHANNEL_ID) ||
       !tls1_write_channel_id(ssl, &body) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     CBB_cleanup(&cbb);
     return ssl_hs_error;
   }
diff --git a/ssl/tls13_server.c b/ssl/tls13_server.c
index 750e47f..1f10038 100644
--- a/ssl/tls13_server.c
+++ b/ssl/tls13_server.c
@@ -349,7 +349,7 @@
       !CBB_add_u16(&extensions, TLSEXT_TYPE_key_share) ||
       !CBB_add_u16(&extensions, 2 /* length */) ||
       !CBB_add_u16(&extensions, group_id) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     CBB_cleanup(&cbb);
     return ssl_hs_error;
   }
@@ -417,7 +417,7 @@
     }
   }
 
-  if (!ssl_complete_message(ssl, &cbb)) {
+  if (!ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
 
@@ -443,7 +443,7 @@
   if (!ssl->method->init_message(ssl, &cbb, &body,
                                  SSL3_MT_ENCRYPTED_EXTENSIONS) ||
       !ssl_add_serverhello_tlsext(hs, &body) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     CBB_cleanup(&cbb);
     return ssl_hs_error;
   }
@@ -488,7 +488,7 @@
 
   if (!ssl_add_client_CA_list(ssl, &body) ||
       !CBB_add_u16(&body, 0 /* empty certificate_extensions. */) ||
-      !ssl_complete_message(ssl, &cbb)) {
+      !ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
 
@@ -692,7 +692,7 @@
     goto err;
   }
 
-  if (!ssl_complete_message(ssl, &cbb)) {
+  if (!ssl_add_message_cbb(ssl, &cbb)) {
     goto err;
   }
 
diff --git a/ssl/tls_method.c b/ssl/tls_method.c
index a6584c1..21f9e2a 100644
--- a/ssl/tls_method.c
+++ b/ssl/tls_method.c
@@ -100,14 +100,6 @@
 
 static int ssl3_supports_cipher(const SSL_CIPHER *cipher) { return 1; }
 
-static int ssl3_flush_flight(SSL *ssl) {
-  int ret = BIO_flush(ssl->wbio);
-  if (ret <= 0) {
-    ssl->rwstate = SSL_WRITING;
-  }
-  return ret;
-}
-
 static void ssl3_expect_flight(SSL *ssl) {}
 
 static void ssl3_received_flight(SSL *ssl) {}
@@ -155,14 +147,15 @@
     ssl3_supports_cipher,
     ssl3_init_message,
     ssl3_finish_message,
-    ssl3_queue_message,
-    ssl3_write_message,
-    ssl3_send_change_cipher_spec,
+    ssl3_add_message,
+    ssl3_add_change_cipher_spec,
+    ssl3_add_alert,
     ssl3_flush_flight,
     ssl3_expect_flight,
     ssl3_received_flight,
     ssl3_set_read_state,
     ssl3_set_write_state,
+    ssl3_write_message,
 };
 
 const SSL_METHOD *TLS_method(void) {