shim: perform split handshakes in a separate binary.

The new binary, called |handshaker|, allows split-handshakes to be
tested using shim and handshaker binaries built at different
revisions.

The shim now proxies traffic to the handshaker during the split
handshake.  The handoff and handback steps serialize additional state
about the test being performed, and its results.

The proxy and handshaker make heavy use of Unix-isms, and so
split-handshake tests are now restricted to Linux.

Change-Id: I048f0540c3978a31b3e573e00da17caf41a8059e
Reviewed-on: https://boringssl-review.googlesource.com/29348
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/include/openssl/base.h b/include/openssl/base.h
index 0083a5d..dc30651 100644
--- a/include/openssl/base.h
+++ b/include/openssl/base.h
@@ -139,6 +139,10 @@
 #define OPENSSL_NO_THREADS
 #endif
 
+#if defined(__ANDROID_API__)
+#define OPENSSL_ANDROID
+#endif
+
 #if !defined(OPENSSL_NO_THREADS)
 #define OPENSSL_THREADS
 #endif
diff --git a/ssl/test/CMakeLists.txt b/ssl/test/CMakeLists.txt
index ca2cd38..8d9aa07 100644
--- a/ssl/test/CMakeLists.txt
+++ b/ssl/test/CMakeLists.txt
@@ -15,3 +15,21 @@
 )
 
 target_link_libraries(bssl_shim ssl crypto)
+
+if(UNIX AND NOT APPLE AND NOT ANDROID)
+  add_executable(
+    handshaker
+
+    async_bio.cc
+    handshake_util.cc
+    handshaker.cc
+    packeted_bio.cc
+    settings_writer.cc
+    test_config.cc
+    test_state.cc
+
+    $<TARGET_OBJECTS:test_support>
+  )
+
+  target_link_libraries(handshaker ssl crypto)
+endif()
diff --git a/ssl/test/bssl_shim.cc b/ssl/test/bssl_shim.cc
index cae9b05..8778bb8 100644
--- a/ssl/test/bssl_shim.cc
+++ b/ssl/test/bssl_shim.cc
@@ -758,10 +758,15 @@
 
   if (!config->implicit_handshake) {
     if (config->handoff) {
+#if defined(OPENSSL_LINUX) && !defined(OPENSSL_ANDROID)
       if (!DoSplitHandshake(ssl_uniqueptr, writer, is_resume)) {
         return false;
       }
       ssl = ssl_uniqueptr->get();
+#else
+      fprintf(stderr, "The external handshaker can only be used on Linux\n");
+      return false;
+#endif
     }
 
     do {
diff --git a/ssl/test/handshake_util.cc b/ssl/test/handshake_util.cc
index b1440fc..751ea34 100644
--- a/ssl/test/handshake_util.cc
+++ b/ssl/test/handshake_util.cc
@@ -15,6 +15,15 @@
 #include "handshake_util.h"
 
 #include <assert.h>
+#if defined(OPENSSL_LINUX) && !defined(OPENSSL_ANDROID)
+#include <fcntl.h>
+#include <spawn.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#endif
 
 #include <functional>
 
@@ -113,6 +122,8 @@
   return ret;
 }
 
+#if defined(OPENSSL_LINUX) && !defined(OPENSSL_ANDROID)
+
 // MoveBIOs moves the |BIO|s of |src| to |dst|.  It is used for handoff.
 static void MoveBIOs(SSL *dest, SSL *src) {
   BIO *rbio = SSL_get_rbio(src);
@@ -131,81 +142,329 @@
   return ret < 0 && SSL_get_error(ssl, ret) == SSL_ERROR_HANDOFF;
 }
 
-static bool HandbackReady(SSL *ssl, int ret) {
-  return ret < 0 && SSL_get_error(ssl, ret) == SSL_ERROR_HANDBACK;
+static ssize_t read_eintr(int fd, void *out, size_t len) {
+  ssize_t ret;
+  do {
+    ret = read(fd, out, len);
+  } while (ret < 0 && errno == EINTR);
+  return ret;
 }
 
-bool DoSplitHandshake(UniquePtr<SSL> *ssl_uniqueptr, SettingsWriter *writer,
-                      bool is_resume) {
-  SSL *ssl = ssl_uniqueptr->get();
+static ssize_t write_eintr(int fd, const void *in, size_t len) {
+  ssize_t ret;
+  do {
+    ret = write(fd, in, len);
+  } while (ret < 0 && errno == EINTR);
+  return ret;
+}
+
+static ssize_t waitpid_eintr(pid_t pid, int *wstatus, int options) {
+  pid_t ret;
+  do {
+    ret = waitpid(pid, wstatus, options);
+  } while (ret < 0 && errno == EINTR);
+  return ret;
+}
+
+// Proxy relays data between |socket|, which is connected to the client, and the
+// handshaker, which is connected to the numerically specified file descriptors,
+// until the handshaker returns control.
+static bool Proxy(BIO *socket, bool async, int control, int rfd, int wfd) {
+  for (;;) {
+    fd_set rfds;
+    FD_ZERO(&rfds);
+    FD_SET(wfd, &rfds);
+    FD_SET(control, &rfds);
+    int fd_max = wfd > control ? wfd : control;
+    if (select(fd_max + 1, &rfds, nullptr, nullptr, nullptr) == -1) {
+      perror("select");
+      return false;
+    }
+
+    char buf[64];
+    ssize_t bytes;
+    if (FD_ISSET(wfd, &rfds) &&
+        (bytes = read_eintr(wfd, buf, sizeof(buf))) > 0) {
+      char *b = buf;
+      while (bytes) {
+        int written = BIO_write(socket, b, bytes);
+        if (!written) {
+          fprintf(stderr, "BIO_write wrote nothing\n");
+          return false;
+        }
+        if (written < 0) {
+          if (async) {
+            AsyncBioAllowWrite(socket, 1);
+            continue;
+          }
+          fprintf(stderr, "BIO_write failed\n");
+          return false;
+        }
+        b += written;
+        bytes -= written;
+      }
+      // Flush all pending data from the handshaker to the client before
+      // considering control messages.
+      continue;
+    }
+
+    if (!FD_ISSET(control, &rfds)) {
+      continue;
+    }
+
+    char msg;
+    if (read_eintr(control, &msg, 1) != 1) {
+      perror("read");
+      return false;
+    }
+    switch (msg) {
+      case kControlMsgHandback:
+        return true;
+      case kControlMsgError:
+        return false;
+      case kControlMsgWantRead:
+        break;
+      default:
+        fprintf(stderr, "Unknown control message from handshaker: %c\n", msg);
+        return false;
+    }
+
+    char readbuf[64];
+    if (async) {
+      AsyncBioAllowRead(socket, 1);
+    }
+    int read = BIO_read(socket, readbuf, sizeof(readbuf));
+    if (read < 1) {
+      fprintf(stderr, "BIO_read failed\n");
+      return false;
+    }
+    ssize_t written = write_eintr(rfd, readbuf, read);
+    if (written == -1) {
+      perror("write");
+      return false;
+    }
+    if (written != read) {
+      fprintf(stderr, "short write (%zu of %d bytes)\n", written, read);
+      return false;
+    }
+    // The handshaker blocks on the control channel, so we have to signal
+    // it that the data have been written.
+    msg = kControlMsgWriteCompleted;
+    if (write_eintr(control, &msg, 1) != 1) {
+      perror("write");
+      return false;
+    }
+  }
+}
+
+class ScopedFD {
+ public:
+  explicit ScopedFD(int fd): fd_(fd) {}
+  ~ScopedFD() { close(fd_); }
+ private:
+  const int fd_;
+};
+
+// RunHandshaker forks and execs the handshaker binary, handing off |input|,
+// and, after proxying some amount of handshake traffic, handing back |out|.
+static bool RunHandshaker(BIO *bio, const TestConfig *config, bool is_resume,
+                          const Array<uint8_t> &input,
+                          Array<uint8_t> *out) {
+  if (config->handshaker_path.empty()) {
+    fprintf(stderr, "no -handshaker-path specified\n");
+    return false;
+  }
+  struct stat dummy;
+  if (stat(config->handshaker_path.c_str(), &dummy) == -1) {
+    perror(config->handshaker_path.c_str());
+    return false;
+  }
+
+  // A datagram socket guarantees that writes are all-or-nothing.
+  int control[2];
+  if (socketpair(AF_LOCAL, SOCK_DGRAM, 0, control) != 0) {
+    perror("socketpair");
+    return false;
+  }
+  int rfd[2], wfd[2];
+  // We use pipes, rather than some other mechanism, for their buffers.  During
+  // the handshake, this process acts as a dumb proxy until receiving the
+  // handback signal, which arrives asynchronously.  The race condition means
+  // that this process could incorrectly proxy post-handshake data from the
+  // client to the handshaker.
+  //
+  // To avoid this, this process never proxies data to the handshaker that the
+  // handshaker has not explicitly requested as a result of hitting
+  // |SSL_ERROR_WANT_READ|.  Pipes allow the data to sit in a buffer while the
+  // two processes synchronize over the |control| channel.
+  if (pipe(rfd) != 0 || pipe(wfd) != 0) {
+    perror("pipe2");
+    return false;
+  }
+
+  fflush(stdout);
+  fflush(stderr);
+
+  std::vector<char *> args;
+  bssl::UniquePtr<char> handshaker_path(
+      OPENSSL_strdup(config->handshaker_path.c_str()));
+  args.push_back(handshaker_path.get());
+  char resume[] = "-handshaker-resume";
+  if (is_resume) {
+    args.push_back(resume);
+  }
+  // config->argv omits argv[0].
+  for (int j = 0; j < config->argc; ++j) {
+    args.push_back(config->argv[j]);
+  }
+  args.push_back(nullptr);
+
+  posix_spawn_file_actions_t actions;
+  if (posix_spawn_file_actions_init(&actions) != 0 ||
+      posix_spawn_file_actions_addclose(&actions, control[0]) ||
+      posix_spawn_file_actions_addclose(&actions, rfd[1]) ||
+      posix_spawn_file_actions_addclose(&actions, wfd[0])) {
+    return false;
+  }
+  assert(kFdControl != rfd[0]);
+  assert(kFdControl != wfd[1]);
+  if (control[1] != kFdControl &&
+      posix_spawn_file_actions_adddup2(&actions, control[1], kFdControl) != 0) {
+    return false;
+  }
+  assert(kFdProxyToHandshaker != wfd[1]);
+  if (rfd[0] != kFdProxyToHandshaker &&
+      posix_spawn_file_actions_adddup2(&actions, rfd[0],
+                                       kFdProxyToHandshaker) != 0) {
+    return false;
+  }
+  if (wfd[1] != kFdHandshakerToProxy &&
+      posix_spawn_file_actions_adddup2(&actions, wfd[1],
+                                       kFdHandshakerToProxy) != 0) {
+      return false;
+  }
+
+  pid_t handshaker_pid;
+  int ret = posix_spawn(&handshaker_pid, args[0], &actions, nullptr,
+                        args.data(), nullptr);
+  if (posix_spawn_file_actions_destroy(&actions) != 0 ||
+      ret != 0) {
+    return false;
+  }
+
+  close(control[1]);
+  close(rfd[0]);
+  close(wfd[1]);
+  ScopedFD rfd_closer(rfd[1]);
+  ScopedFD wfd_closer(wfd[0]);
+  ScopedFD control_closer(control[0]);
+
+  if (write_eintr(control[0], input.data(), input.size()) == -1) {
+    perror("write");
+    return false;
+  }
+  bool ok = Proxy(bio, config->async, control[0], rfd[1], wfd[0]);
+  int wstatus;
+  if (waitpid_eintr(handshaker_pid, &wstatus, 0) != handshaker_pid) {
+    perror("waitpid");
+    return false;
+  }
+  if (ok && wstatus) {
+    fprintf(stderr, "handshaker exited irregularly\n");
+    return false;
+  }
+  if (!ok) {
+    return false;  // This is a "good", i.e. expected, error.
+  }
+
+  constexpr size_t kBufSize = 1024 * 1024;
+  bssl::UniquePtr<uint8_t> buf((uint8_t *) OPENSSL_malloc(kBufSize));
+  int len = read_eintr(control[0], buf.get(), kBufSize);
+  if (len == -1) {
+    perror("read");
+    return false;
+  }
+  out->CopyFrom({buf.get(), (size_t)len});
+  return true;
+}
+
+// PrepareHandoff accepts the |ClientHello| from |ssl| and serializes state to
+// be passed to the handshaker.  The serialized state includes both the SSL
+// handoff, as well test-related state.
+static bool PrepareHandoff(SSL *ssl, SettingsWriter *writer,
+                           Array<uint8_t> *out_handoff) {
   SSL_set_handoff_mode(ssl, 1);
 
   const TestConfig *config = GetTestConfig(ssl);
   int ret = -1;
   do {
-    ret = CheckIdempotentError("SSL_do_handshake", ssl,
-                               [&]() -> int { return SSL_do_handshake(ssl); });
+    ret = CheckIdempotentError(
+        "SSL_do_handshake", ssl,
+        [&]() -> int { return SSL_do_handshake(ssl); });
   } while (!HandoffReady(ssl, ret) &&
            config->async &&
            RetryAsync(ssl, ret));
+  if (!HandoffReady(ssl, ret)) {
+    fprintf(stderr, "Handshake failed while waiting for handoff.\n");
+    return false;
+  }
 
   ScopedCBB cbb;
-  Array<uint8_t> handoff;
-  if (!HandoffReady(ssl, ret) ||
-      !CBB_init(cbb.get(), 512) ||
+  if (!CBB_init(cbb.get(), 512) ||
       !SSL_serialize_handoff(ssl, cbb.get()) ||
-      !CBBFinishArray(cbb.get(), &handoff) ||
-      !writer->WriteHandoff(handoff)) {
+      !writer->WriteHandoff({CBB_data(cbb.get()), CBB_len(cbb.get())}) ||
+      !SerializeContextState(ssl->ctx.get(), cbb.get()) ||
+      !GetTestState(ssl)->Serialize(cbb.get())) {
+    fprintf(stderr, "Handoff serialisation failed.\n");
+    return false;
+  }
+  return CBBFinishArray(cbb.get(), out_handoff);
+}
+
+// DoSplitHandshake delegates the SSL handshake to a separate process, called
+// the handshaker.  This process proxies I/O between the handshaker and the
+// client, using the |BIO| from |ssl|.  After a successful handshake, |ssl| is
+// replaced with a new |SSL| object, in a way that is intended to be invisible
+// to the caller.
+bool DoSplitHandshake(UniquePtr<SSL> *ssl, SettingsWriter *writer,
+                      bool is_resume) {
+  assert(SSL_get_rbio(ssl->get()) == SSL_get_wbio(ssl->get()));
+  Array<uint8_t> handshaker_input;
+  const TestConfig *config = GetTestConfig(ssl->get());
+  // out is the response from the handshaker, which includes a serialized
+  // handback message, but also serialized updates to the |TestState|.
+  Array<uint8_t> out;
+  if (!PrepareHandoff(ssl->get(), writer, &handshaker_input) ||
+      !RunHandshaker(SSL_get_rbio(ssl->get()), config, is_resume,
+                     handshaker_input, &out)) {
     fprintf(stderr, "Handoff failed.\n");
     return false;
   }
 
-  UniquePtr<SSL_CTX> ctx = config->SetupCtx(ssl->ctx.get());
-  if (!ctx) {
-    return false;
-  }
-  UniquePtr<SSL> ssl_handshaker =
-      config->NewSSL(ctx.get(), nullptr, false, nullptr);
-  if (!ssl_handshaker) {
-    return false;
-  }
-  MoveBIOs(ssl_handshaker.get(), ssl);
-
-  if (!MoveTestState(ssl_handshaker.get(), ssl) ||
-      !SSL_apply_handoff(ssl_handshaker.get(), handoff)) {
-    fprintf(stderr, "Handoff application failed.\n");
-    return false;
-  }
-
-  do {
-    ret = CheckIdempotentError(
-        "SSL_do_handshake", ssl_handshaker.get(),
-        [&]() -> int { return SSL_do_handshake(ssl_handshaker.get()); });
-  } while (config->async && RetryAsync(ssl_handshaker.get(), ret));
-
-  Array<uint8_t> handback;
-  if (!HandbackReady(ssl_handshaker.get(), ret) ||
-      !CBB_init(cbb.get(), 512) ||
-      !SSL_serialize_handback(ssl_handshaker.get(), cbb.get()) ||
-      !CBBFinishArray(cbb.get(), &handback) ||
-      !writer->WriteHandback(handback)) {
-    fprintf(stderr, "Handback failed.\n");
-    return false;
-  }
-
   UniquePtr<SSL> ssl_handback =
-      config->NewSSL(ctx.get(), nullptr, false, nullptr);
+      config->NewSSL((*ssl)->ctx.get(), nullptr, false, nullptr);
   if (!ssl_handback) {
     return false;
   }
-  MoveBIOs(ssl_handback.get(), ssl_handshaker.get());
-
-  if (!MoveTestState(ssl_handback.get(), ssl_handshaker.get()) ||
+  CBS output, handback;
+  CBS_init(&output, out.data(), out.size());
+  if (!CBS_get_u24_length_prefixed(&output, &handback) ||
+      !DeserializeContextState(&output, ssl_handback->ctx.get()) ||
+      !SetTestState(ssl_handback.get(), TestState::Deserialize(
+          &output, ssl_handback->ctx.get())) ||
+      !GetTestState(ssl_handback.get()) ||
+      !writer->WriteHandback(handback) ||
       !SSL_apply_handback(ssl_handback.get(), handback)) {
-    fprintf(stderr, "Handback application failed.\n");
+    fprintf(stderr, "Handback failed.\n");
     return false;
   }
+  MoveBIOs(ssl_handback.get(), ssl->get());
+  GetTestState(ssl_handback.get())->async_bio =
+      GetTestState(ssl->get())->async_bio;
+  GetTestState(ssl->get())->async_bio = nullptr;
 
-  *ssl_uniqueptr = std::move(ssl_handback);
+  *ssl = std::move(ssl_handback);
   return true;
 }
+
+#endif  // defined(OPENSSL_LINUX) && !defined(OPENSSL_ANDROID)
diff --git a/ssl/test/handshake_util.h b/ssl/test/handshake_util.h
index ec86b8e..4fb46db 100644
--- a/ssl/test/handshake_util.h
+++ b/ssl/test/handshake_util.h
@@ -30,9 +30,24 @@
 // errors are idempotent.
 int CheckIdempotentError(const char *name, SSL *ssl, std::function<int()> func);
 
-// DoSplitHandshake performs a handoff and handback of an in-progress handshake,
-// updating |ssl_uniqueptr| in place.
-bool DoSplitHandshake(bssl::UniquePtr<SSL> *ssl_uniqueptr,
-                      SettingsWriter *writer, bool is_resume);
+// DoSplitHandshake delegates the SSL handshake to a separate process, called
+// the handshaker.  This process proxies I/O between the handshaker and the
+// client, using the |BIO| from |ssl|.  After a successful handshake, |ssl| is
+// replaced with a new |SSL| object, in a way that is intended to be invisible
+// to the caller.
+bool DoSplitHandshake(bssl::UniquePtr<SSL> *ssl, SettingsWriter *writer,
+                      bool is_resume);
+
+// The protocol between the proxy and the handshaker is defined by these
+// single-character prefixes.
+constexpr char kControlMsgWantRead = 'R';        // Handshaker wants data
+constexpr char kControlMsgWriteCompleted = 'W';  // Proxy has sent data
+constexpr char kControlMsgHandback = 'H';        // Proxy should resume control
+constexpr char kControlMsgError = 'E';           // Handshaker hit an error
+
+// The protocol between the proxy and handshaker uses these file descriptors.
+constexpr int kFdControl = 3;                    // Bi-directional dgram socket.
+constexpr int kFdProxyToHandshaker = 4;          // Uni-directional pipe.
+constexpr int kFdHandshakerToProxy = 5;          // Uni-directional pipe.
 
 #endif  // HEADER_TEST_HANDSHAKE
diff --git a/ssl/test/handshaker.cc b/ssl/test/handshaker.cc
new file mode 100644
index 0000000..01c7151
--- /dev/null
+++ b/ssl/test/handshaker.cc
@@ -0,0 +1,159 @@
+/* Copyright (c) 2018, Google Inc.
+ *
+ * Permission to use, copy, modify, and/or distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
+ * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
+ * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+ * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <signal.h>
+#include <unistd.h>
+
+#include <openssl/bytestring.h>
+#include <openssl/ssl.h>
+
+#include "../internal.h"
+#include "handshake_util.h"
+#include "test_config.h"
+#include "test_state.h"
+
+using namespace bssl;
+
+namespace {
+
+bool HandbackReady(SSL *ssl, int ret) {
+  return ret < 0 && SSL_get_error(ssl, ret) == SSL_ERROR_HANDBACK;
+}
+
+bool Handshaker(const TestConfig *config, int rfd, int wfd,
+                       Span<const uint8_t> input, int control) {
+  UniquePtr<SSL_CTX> ctx = config->SetupCtx(/*old_ctx=*/nullptr);
+  if (!ctx) {
+    return false;
+  }
+  UniquePtr<SSL> ssl = config->NewSSL(ctx.get(), nullptr, false, nullptr);
+
+  // Set |O_NONBLOCK| in order to break out of the loop when we hit
+  // |SSL_ERROR_WANT_READ|, so that we can send |kControlMsgWantRead| to the
+  // proxy.
+  if (fcntl(rfd, F_SETFL, O_NONBLOCK) != 0) {
+    perror("fcntl");
+    return false;
+  }
+  SSL_set_rfd(ssl.get(), rfd);
+  SSL_set_wfd(ssl.get(), wfd);
+
+  CBS cbs, handoff;
+  CBS_init(&cbs, input.data(), input.size());
+  if (!CBS_get_asn1_element(&cbs, &handoff, CBS_ASN1_SEQUENCE) ||
+      !DeserializeContextState(&cbs, ctx.get()) ||
+      !SetTestState(ssl.get(), TestState::Deserialize(&cbs, ctx.get())) ||
+      !GetTestState(ssl.get()) ||
+      !SSL_apply_handoff(ssl.get(), handoff)) {
+    fprintf(stderr, "Handoff application failed.\n");
+    return false;
+  }
+
+  int ret = 0;
+  for (;;) {
+    ret = CheckIdempotentError(
+        "SSL_do_handshake", ssl.get(),
+        [&]() -> int { return SSL_do_handshake(ssl.get()); });
+    if (SSL_get_error(ssl.get(), ret) == SSL_ERROR_WANT_READ) {
+      // Synchronize with the proxy, i.e. don't let the handshake continue until
+      // the proxy has sent more data.
+      char msg = kControlMsgWantRead;
+      if (write(control, &msg, 1) != 1 ||
+          read(control, &msg, 1) != 1 ||
+          msg != kControlMsgWriteCompleted) {
+        fprintf(stderr, "read via proxy failed\n");
+        return false;
+      }
+      continue;
+    }
+    if (!config->async || !RetryAsync(ssl.get(), ret)) {
+      break;
+    }
+  }
+  if (!HandbackReady(ssl.get(), ret)) {
+    ERR_print_errors_fp(stderr);
+    return false;
+  }
+
+  ScopedCBB output;
+  CBB handback;
+  Array<uint8_t> bytes;
+  if (!CBB_init(output.get(), 1024) ||
+      !CBB_add_u24_length_prefixed(output.get(), &handback) ||
+      !SSL_serialize_handback(ssl.get(), &handback) ||
+      !SerializeContextState(ssl->ctx.get(), output.get()) ||
+      !GetTestState(ssl.get())->Serialize(output.get()) ||
+      !CBBFinishArray(output.get(), &bytes)) {
+    fprintf(stderr, "Handback serialisation failed.\n");
+    return false;
+  }
+
+  char msg = kControlMsgHandback;
+  if (write(control, &msg, 1) == -1 ||
+      write(control, bytes.data(), bytes.size()) == -1) {
+    perror("write");
+    return false;
+  }
+  return true;
+}
+
+ssize_t read_eintr(int fd, void *out, size_t len) {
+  ssize_t ret;
+  do {
+    ret = read(fd, out, len);
+  } while (ret < 0 && errno == EINTR);
+  return ret;
+}
+
+ssize_t write_eintr(int fd, const void *in, size_t len) {
+  ssize_t ret;
+  do {
+    ret = write(fd, in, len);
+  } while (ret < 0 && errno == EINTR);
+  return ret;
+}
+
+}  // namespace
+
+int main(int argc, char **argv) {
+  TestConfig initial_config, resume_config, retry_config;
+  if (!ParseConfig(argc - 1, argv + 1, &initial_config, &resume_config,
+                   &retry_config)) {
+    return 2;
+  }
+  const TestConfig *config = initial_config.handshaker_resume
+      ? &resume_config : &initial_config;
+
+  // read() will return the entire message in one go, because it's a datagram
+  // socket.
+  constexpr size_t kBufSize = 1024 * 1024;
+  bssl::UniquePtr<uint8_t> buf((uint8_t *) OPENSSL_malloc(kBufSize));
+  ssize_t len = read_eintr(kFdControl, buf.get(), kBufSize);
+  if (len == -1) {
+    perror("read");
+    return 2;
+  }
+  Span<uint8_t> handoff(buf.get(), len);
+  if (!Handshaker(config, kFdProxyToHandshaker, kFdHandshakerToProxy, handoff,
+                  kFdControl)) {
+    char msg = kControlMsgError;
+    if (write_eintr(kFdControl, &msg, 1) != 1) {
+      return 3;
+    }
+    return 1;
+  }
+  return 0;
+}
diff --git a/ssl/test/runner/runner.go b/ssl/test/runner/runner.go
index 7597764..6f2e111 100644
--- a/ssl/test/runner/runner.go
+++ b/ssl/test/runner/runner.go
@@ -56,6 +56,7 @@
 	testToRun          = flag.String("test", "", "The pattern to filter tests to run, or empty to run all tests")
 	numWorkers         = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.")
 	shimPath           = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.")
+	handshakerPath     = flag.String("handshaker-path", "../../../build/ssl/test/handshaker", "The location of the handshaker binary.")
 	resourceDir        = flag.String("resource-dir", ".", "The directory in which to find certificate and key files.")
 	fuzzer             = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.")
 	transcriptDir      = flag.String("transcript-dir", "", "The directory in which to write transcripts.")
@@ -1124,6 +1125,8 @@
 		flags = append(flags, "-tls13-variant", strconv.Itoa(test.tls13Variant))
 	}
 
+	flags = append(flags, "-handshaker-path", *handshakerPath)
+
 	var transcriptPrefix string
 	var transcripts [][]byte
 	if len(*transcriptDir) != 0 {
@@ -1475,6 +1478,9 @@
 }
 
 func convertToSplitHandshakeTests(tests []testCase) (splitHandshakeTests []testCase) {
+	if runtime.GOOS != "linux" {
+		return
+	}
 NextTest:
 	for _, test := range tests {
 		if test.protocol != tls ||
@@ -2812,7 +2818,7 @@
 			messageCount:            5,
 			keyUpdateRequest:        keyUpdateRequested,
 			readWithUnfinishedWrite: true,
-			flags:                   []string{"-async"},
+			flags: []string{"-async"},
 		},
 		{
 			name: "SendSNIWarningAlert",
diff --git a/ssl/test/test_config.cc b/ssl/test/test_config.cc
index d4e4475..fc87069 100644
--- a/ssl/test/test_config.cc
+++ b/ssl/test/test_config.cc
@@ -143,6 +143,7 @@
   { "-fail-ocsp-callback", &TestConfig::fail_ocsp_callback },
   { "-install-cert-compression-algs",
     &TestConfig::install_cert_compression_algs },
+  { "-handshaker-resume", &TestConfig::handshaker_resume },
 };
 
 const Flag<std::string> kStringFlags[] = {
@@ -170,6 +171,7 @@
   { "-use-client-ca-list", &TestConfig::use_client_ca_list },
   { "-expect-client-ca-list", &TestConfig::expected_client_ca_list },
   { "-expect-msg-callback", &TestConfig::expect_msg_callback },
+  { "-handshaker-path", &TestConfig::handshaker_path },
 };
 
 const Flag<std::string> kBase64Flags[] = {
@@ -314,6 +316,8 @@
                  TestConfig *out_initial,
                  TestConfig *out_resume,
                  TestConfig *out_retry) {
+  out_initial->argc = out_resume->argc = out_retry->argc = argc;
+  out_initial->argv = out_resume->argv = out_retry->argv = argv;
   for (int i = 0; i < argc; i++) {
     bool skip = false;
     char *flag = argv[i];
diff --git a/ssl/test/test_config.h b/ssl/test/test_config.h
index f6c66e6..8fd87ac 100644
--- a/ssl/test/test_config.h
+++ b/ssl/test/test_config.h
@@ -166,6 +166,11 @@
   bool decline_ocsp_callback = false;
   bool fail_ocsp_callback = false;
   bool install_cert_compression_algs = false;
+  bool handshaker_resume = false;
+  std::string handshaker_path;
+
+  int argc;
+  char **argv;
 
   bssl::UniquePtr<SSL_CTX> SetupCtx(SSL_CTX *old_ctx) const;
 
diff --git a/ssl/test/test_state.cc b/ssl/test/test_state.cc
index 83ab6bf..b9767e0 100644
--- a/ssl/test/test_state.cc
+++ b/ssl/test/test_state.cc
@@ -167,24 +167,3 @@
       reinterpret_cast<const char *>(CBS_data(&text)), CBS_len(&text));
   return out_state;
 }
-
-bool MoveTestState(SSL *dest, SSL *src) {
-  ScopedCBB out;
-  Array<uint8_t> serialized;
-  if (!CBB_init(out.get(), 512) ||
-      !SerializeContextState(src->ctx.get(), out.get()) ||
-      !GetTestState(src)->Serialize(out.get()) ||
-      !CBBFinishArray(out.get(), &serialized)) {
-    return false;
-  }
-  CBS in;
-  CBS_init(&in, serialized.data(), serialized.size());
-  if (!DeserializeContextState(&in, dest->ctx.get()) ||
-      !SetTestState(dest, TestState::Deserialize(&in, dest->ctx.get())) ||
-      !GetTestState(dest)) {
-    return false;
-  }
-  GetTestState(dest)->async_bio = GetTestState(src)->async_bio;
-  GetTestState(src)->async_bio = nullptr;
-  return true;
-}
diff --git a/ssl/test/test_state.h b/ssl/test/test_state.h
index 21c78d2..2364286 100644
--- a/ssl/test/test_state.h
+++ b/ssl/test/test_state.h
@@ -67,8 +67,6 @@
 
 TestState *GetTestState(const SSL *ssl);
 
-bool MoveTestState(SSL *dest, SSL *src);
-
 struct timeval *GetClock();
 
 void AdvanceClock(unsigned seconds);