ssl/test/runner/runner.go

// Copyright (c) 2016, Google Inc.
//
// Permission to use, copy, modify, and/or distribute this software for any
// purpose with or without fee is hereby granted, provided that the above
// copyright notice and this permission notice appear in all copies.
//
// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

package runner

import (
	"bytes"
	"crypto"
	"crypto/ecdsa"
	"crypto/ed25519"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"crypto/x509/pkix"
	_ "embed"
	"encoding/base64"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"encoding/pem"
	"errors"
	"flag"
	"fmt"
	"io"
	"math/big"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"boringssl.googlesource.com/boringssl/ssl/test/runner/hpke"
	"boringssl.googlesource.com/boringssl/util/testresult"
	"golang.org/x/crypto/cryptobyte"
)

var (
	useValgrind        = flag.Bool("valgrind", false, "If true, run code under valgrind")
	useGDB             = flag.Bool("gdb", false, "If true, run BoringSSL code under gdb")
	useLLDB            = flag.Bool("lldb", false, "If true, run BoringSSL code under lldb")
	useRR              = flag.Bool("rr-record", false, "If true, run BoringSSL code under `rr record`.")
	waitForDebugger    = flag.Bool("wait-for-debugger", false, "If true, jobs will run one at a time and pause for a debugger to attach")
	flagDebug          = flag.Bool("debug", false, "Hexdump the contents of the connection")
	mallocTest         = flag.Int64("malloc-test", -1, "If non-negative, run each test with each malloc in turn failing from the given number onwards.")
	mallocTestDebug    = flag.Bool("malloc-test-debug", false, "If true, ask bssl_shim to abort rather than fail a malloc. This can be used with a specific value for --malloc-test to identity the malloc failing that is causing problems.")
	jsonOutput         = flag.String("json-output", "", "The file to output JSON results to.")
	pipe               = flag.Bool("pipe", false, "If true, print status output suitable for piping into another program.")
	testToRun          = flag.String("test", "", "Semicolon-separated patterns of tests to run, or empty to run all tests")
	skipTest           = flag.String("skip", "", "Semicolon-separated patterns of tests to skip")
	allowHintMismatch  = flag.String("allow-hint-mismatch", "", "Semicolon-separated patterns of tests where hints may mismatch")
	numWorkersFlag     = flag.Int("num-workers", runtime.NumCPU(), "The number of workers to run in parallel.")
	shimPath           = flag.String("shim-path", "../../../build/ssl/test/bssl_shim", "The location of the shim binary.")
	shimExtraFlags     = flag.String("shim-extra-flags", "", "Semicolon-separated extra flags to pass to the shim binary on each invocation.")
	handshakerPath     = flag.String("handshaker-path", "../../../build/ssl/test/handshaker", "The location of the handshaker binary.")
	fuzzer             = flag.Bool("fuzzer", false, "If true, tests against a BoringSSL built in fuzzer mode.")
	transcriptDir      = flag.String("transcript-dir", "", "The directory in which to write transcripts.")
	idleTimeout        = flag.Duration("idle-timeout", 15*time.Second, "The number of seconds to wait for a read or write to bssl_shim.")
	deterministic      = flag.Bool("deterministic", false, "If true, uses a deterministic PRNG in the runner.")
	allowUnimplemented = flag.Bool("allow-unimplemented", false, "If true, report pass even if some tests are unimplemented.")
	looseErrors        = flag.Bool("loose-errors", false, "If true, allow shims to report an untranslated error code.")
	shimConfigFile     = flag.String("shim-config", "", "A config file to use to configure the tests for this shim.")
	includeDisabled    = flag.Bool("include-disabled", false, "If true, also runs disabled tests.")
	repeatUntilFailure = flag.Bool("repeat-until-failure", false, "If true, the first selected test will be run repeatedly until failure.")
)

// ShimConfigurations is used with the “json” package and represents a shim
// config file.
type ShimConfiguration struct {
	// DisabledTests maps from a glob-based pattern to a freeform string.
	// The glob pattern is used to exclude tests from being run and the
	// freeform string is unparsed but expected to explain why the test is
	// disabled.
	DisabledTests map[string]string

	// ErrorMap maps from expected error strings to the correct error
	// string for the shim in question. For example, it might map
	// “:NO_SHARED_CIPHER:” (a BoringSSL error string) to something
	// like “SSL_ERROR_NO_CYPHER_OVERLAP”.
	ErrorMap map[string]string

	// HalfRTTTickets is the number of half-RTT tickets the client should
	// expect before half-RTT data when testing 0-RTT.
	HalfRTTTickets int

	// AllCurves is the list of all curve code points supported by the shim.
	// This is currently used to control tests that enable all curves but may
	// automatically disable tests in the future.
	AllCurves []int
}

// Setup shimConfig defaults aligning with BoringSSL.
var shimConfig ShimConfiguration = ShimConfiguration{
	HalfRTTTickets: 2,
}

//go:embed rsa_2048_key.pem
var rsa2048KeyPEM []byte

//go:embed rsa_1024_key.pem
var rsa1024KeyPEM []byte

//go:embed ecdsa_p224_key.pem
var ecdsaP224KeyPEM []byte

//go:embed ecdsa_p256_key.pem
var ecdsaP256KeyPEM []byte

//go:embed ecdsa_p384_key.pem
var ecdsaP384KeyPEM []byte

//go:embed ecdsa_p521_key.pem
var ecdsaP521KeyPEM []byte

//go:embed ed25519_key.pem
var ed25519KeyPEM []byte

//go:embed channel_id_key.pem
var channelIDKeyPEM []byte

var (
	rsa1024Key rsa.PrivateKey
	rsa2048Key rsa.PrivateKey

	ecdsaP224Key ecdsa.PrivateKey
	ecdsaP256Key ecdsa.PrivateKey
	ecdsaP384Key ecdsa.PrivateKey
	ecdsaP521Key ecdsa.PrivateKey

	ed25519Key ed25519.PrivateKey

	channelIDKey ecdsa.PrivateKey
)

var channelIDKeyPath string

func initKeys() {
	// Since key generation is not particularly cheap (especially RSA), and the
	// runner is intended to run on systems which may be resouece constrained,
	// we load keys from disk instead of dynamically generating them. We treat
	// key files the same as dynamically generated certificates, writing them
	// out to temporary files before passing them to the shim.

	for _, k := range []struct {
		pemBytes []byte
		key      *rsa.PrivateKey
	}{
		{rsa1024KeyPEM, &rsa1024Key},
		{rsa2048KeyPEM, &rsa2048Key},
	} {
		key, err := loadPEMKey(k.pemBytes)
		if err != nil {
			panic(fmt.Sprintf("failed to load RSA test key: %s", err))
		}
		*k.key = *(key.(*rsa.PrivateKey))
	}

	for _, k := range []struct {
		pemBytes []byte
		key      *ecdsa.PrivateKey
	}{
		{ecdsaP224KeyPEM, &ecdsaP224Key},
		{ecdsaP256KeyPEM, &ecdsaP256Key},
		{ecdsaP384KeyPEM, &ecdsaP384Key},
		{ecdsaP521KeyPEM, &ecdsaP521Key},
		{channelIDKeyPEM, &channelIDKey},
	} {
		key, err := loadPEMKey(k.pemBytes)
		if err != nil {
			panic(fmt.Sprintf("failed to load ECDSA test key: %s", err))
		}
		*k.key = *(key.(*ecdsa.PrivateKey))
	}

	k, err := loadPEMKey(ed25519KeyPEM)
	if err != nil {
		panic(fmt.Sprintf("failed to load Ed25519 test key: %s", err))
	}
	ed25519Key = k.(ed25519.PrivateKey)

	channelIDKeyPath = writeTempKeyFile(&channelIDKey)
}

var channelIDBytes []byte

var testOCSPResponse = []byte{1, 2, 3, 4}
var testOCSPResponse2 = []byte{5, 6, 7, 8}
var testSCTList = []byte{0, 6, 0, 4, 5, 6, 7, 8}
var testSCTList2 = []byte{0, 6, 0, 4, 1, 2, 3, 4}

var testOCSPExtension = append([]byte{byte(extensionStatusRequest) >> 8, byte(extensionStatusRequest), 0, 8, statusTypeOCSP, 0, 0, 4}, testOCSPResponse...)
var testSCTExtension = append([]byte{byte(extensionSignedCertificateTimestamp) >> 8, byte(extensionSignedCertificateTimestamp), 0, byte(len(testSCTList))}, testSCTList...)

var (
	rsaCertificate       Credential
	rsaChainCertificate  Credential
	rsa1024Certificate   Credential
	ecdsaP224Certificate Credential
	ecdsaP256Certificate Credential
	ecdsaP384Certificate Credential
	ecdsaP521Certificate Credential
	ed25519Certificate   Credential
	garbageCertificate   Credential
)

func initCertificates() {
	for _, def := range []struct {
		key crypto.Signer
		out *Credential
	}{
		{&rsa1024Key, &rsa1024Certificate},
		{&rsa2048Key, &rsaCertificate},
		{&ecdsaP224Key, &ecdsaP224Certificate},
		{&ecdsaP256Key, &ecdsaP256Certificate},
		{&ecdsaP384Key, &ecdsaP384Certificate},
		{&ecdsaP521Key, &ecdsaP521Certificate},
		{ed25519Key, &ed25519Certificate},
	} {
		*def.out = generateSingleCertChain(nil, def.key)
	}

	channelIDBytes = make([]byte, 64)
	writeIntPadded(channelIDBytes[:32], channelIDKey.X)
	writeIntPadded(channelIDBytes[32:], channelIDKey.Y)

	garbageCertificate.Certificate = [][]byte{[]byte("GARBAGE")}
	garbageCertificate.PrivateKey = rsaCertificate.PrivateKey

	// Build a basic three cert chain for testing chain specific things.
	rootTmpl := *baseCertTemplate
	rootTmpl.Subject.CommonName = "test root"
	rootCert := generateTestCert(&rootTmpl, nil, &rsa2048Key)
	intermediateTmpl := *baseCertTemplate
	intermediateTmpl.Subject.CommonName = "test inter"
	intermediateCert := generateTestCert(&intermediateTmpl, rootCert, &rsa2048Key)
	leafTmpl := *baseCertTemplate
	leafTmpl.IsCA, leafTmpl.BasicConstraintsValid = false, false
	leafCert := generateTestCert(nil, intermediateCert, &rsa2048Key)

	keyPath := writeTempKeyFile(&rsa2048Key)
	rootCertPath, chainPath := writeTempCertFile([]*x509.Certificate{rootCert}), writeTempCertFile([]*x509.Certificate{leafCert, intermediateCert})

	rsaChainCertificate = Credential{
		Certificate: [][]byte{leafCert.Raw, intermediateCert.Raw},
		PrivateKey:  &rsa2048Key,
		Leaf:        leafCert,
		ChainPath:   chainPath,
		KeyPath:     keyPath,
		RootPath:    rootCertPath,
	}
}

func flagInts(flagName string, vals []int) []string {
	ret := make([]string, 0, 2*len(vals))
	for _, val := range vals {
		ret = append(ret, flagName, strconv.Itoa(val))
	}
	return ret
}

func base64FlagValue(in []byte) string {
	return base64.StdEncoding.EncodeToString(in)
}

func useDebugger() bool {
	return *useGDB || *useLLDB || *useRR || *waitForDebugger
}

// delegatedCredentialConfig specifies the shape of a delegated credential, not
// including the keys themselves.
type delegatedCredentialConfig struct {
	// lifetime is the amount of time, from the notBefore of the parent
	// certificate, that the delegated credential is valid for. If zero, then 24
	// hours is assumed.
	lifetime time.Duration
	// dcAlgo is the signature scheme that should be used with this delegated
	// credential. If zero, ECDSA with P-256 is assumed.
	dcAlgo signatureAlgorithm
	// algo is the signature algorithm that the delegated credential itself is
	// signed with. Cannot be zero.
	algo signatureAlgorithm
}

func loadPEMKey(pemBytes []byte) (crypto.PrivateKey, error) {
	block, _ := pem.Decode(pemBytes)
	if block == nil {
		return nil, fmt.Errorf("no PEM block found")
	}

	if block.Type != "PRIVATE KEY" {
		return nil, fmt.Errorf("unexpected PEM type (expected \"PRIVATE KEY\"): %s", block.Type)
	}

	k, err := x509.ParsePKCS8PrivateKey(block.Bytes)
	if err != nil {
		return nil, fmt.Errorf("failed to parse PKCS#8 key: %s", err)
	}

	return k, nil
}

func createDelegatedCredential(parent *Credential, config delegatedCredentialConfig) *Credential {
	if parent.Type != CredentialTypeX509 {
		panic("delegated credentials must be issued by X.509 credentials")
	}

	dcAlgo := config.dcAlgo
	if dcAlgo == 0 {
		dcAlgo = signatureECDSAWithP256AndSHA256
	}

	var dcPriv crypto.Signer
	switch dcAlgo {
	case signatureRSAPKCS1WithMD5, signatureRSAPKCS1WithSHA1, signatureRSAPKCS1WithSHA256, signatureRSAPKCS1WithSHA384, signatureRSAPKCS1WithSHA512, signatureRSAPSSWithSHA256, signatureRSAPSSWithSHA384, signatureRSAPSSWithSHA512:
		dcPriv = &rsa2048Key

	case signatureECDSAWithSHA1, signatureECDSAWithP256AndSHA256, signatureECDSAWithP384AndSHA384, signatureECDSAWithP521AndSHA512:
		var curve elliptic.Curve
		switch dcAlgo {
		case signatureECDSAWithSHA1, signatureECDSAWithP256AndSHA256:
			curve = elliptic.P256()
		case signatureECDSAWithP384AndSHA384:
			curve = elliptic.P384()
		case signatureECDSAWithP521AndSHA512:
			curve = elliptic.P521()
		default:
			panic("internal error")
		}

		priv, err := ecdsa.GenerateKey(curve, rand.Reader)
		if err != nil {
			panic(err)
		}
		dcPriv = priv

	default:
		panic(fmt.Errorf("unsupported DC signature algorithm: %x", dcAlgo))
	}

	lifetime := config.lifetime
	if lifetime == 0 {
		lifetime = 24 * time.Hour
	}
	lifetimeSecs := int64(lifetime.Seconds())
	if lifetimeSecs < 0 || lifetimeSecs > 1<<32 {
		panic(fmt.Errorf("lifetime %s is too long to be expressed", lifetime))
	}

	// https://www.rfc-editor.org/rfc/rfc9345.html#section-4
	dc := cryptobyte.NewBuilder(nil)
	dc.AddUint32(uint32(lifetimeSecs))
	dc.AddUint16(uint16(dcAlgo))

	pubBytes, err := x509.MarshalPKIXPublicKey(dcPriv.Public())
	if err != nil {
		panic(err)
	}
	addUint24LengthPrefixedBytes(dc, pubBytes)

	var dummyConfig Config
	parentSignature, err := signMessage(false /* server */, VersionTLS13, parent.PrivateKey, &dummyConfig, config.algo, delegatedCredentialSignedMessage(dc.BytesOrPanic(), config.algo, parent.Leaf.Raw))
	if err != nil {
		panic(err)
	}

	dc.AddUint16(uint16(config.algo))
	addUint16LengthPrefixedBytes(dc, parentSignature)

	dcCred := *parent
	dcCred.Type = CredentialTypeDelegated
	dcCred.DelegatedCredential = dc.BytesOrPanic()
	dcCred.PrivateKey = dcPriv
	dcCred.KeyPath = writeTempKeyFile(dcPriv)
	return &dcCred
}

// recordVersionToWire maps a record-layer protocol version to its wire
// representation.
func recordVersionToWire(vers uint16, protocol protocol) uint16 {
	if protocol == dtls {
		switch vers {
		case VersionTLS12:
			return VersionDTLS12
		case VersionTLS10:
			return VersionDTLS10
		}
	} else {
		switch vers {
		case VersionSSL30, VersionTLS10, VersionTLS11, VersionTLS12:
			return vers
		}
	}

	panic("unknown version")
}

// encodeDERValues encodes a series of bytestrings in comma-separated-hex form.
func encodeDERValues(values [][]byte) string {
	var ret string
	for i, v := range values {
		if i > 0 {
			ret += ","
		}
		ret += hex.EncodeToString(v)
	}

	return ret
}

func decodeHexOrPanic(in string) []byte {
	ret, err := hex.DecodeString(in)
	if err != nil {
		panic(err)
	}
	return ret
}

type testType int

const (
	clientTest testType = iota
	serverTest
)

func (t testType) String() string {
	switch t {
	case clientTest:
		return "Client"
	case serverTest:
		return "Server"
	}
	panic(fmt.Sprintf("bad test type %d", t))
}

type protocol int

const (
	tls protocol = iota
	dtls
	quic
)

func (p protocol) String() string {
	switch p {
	case tls:
		return "TLS"
	case dtls:
		return "DTLS"
	case quic:
		return "QUIC"
	}
	return "unknown protocol"
}

const (
	alpn = 1
	npn  = 2
)

// connectionExpectations contains connection-level test expectations to check
// on the runner side.
type connectionExpectations struct {
	// version, if non-zero, specifies the TLS version that must be negotiated.
	version uint16
	// cipher, if non-zero, specifies the TLS cipher suite that should be
	// negotiated.
	cipher uint16
	// channelID controls whether the connection should have negotiated a
	// Channel ID with channelIDKey.
	channelID bool
	// nextProto controls whether the connection should negotiate a next
	// protocol via NPN or ALPN.
	nextProto string
	// noNextProto, if true, means that no next protocol should be negotiated.
	noNextProto bool
	// nextProtoType, if non-zero, is the next protocol negotiation mechanism.
	nextProtoType int
	// srtpProtectionProfile is the DTLS-SRTP profile that should be negotiated.
	// If zero, none should be negotiated.
	srtpProtectionProfile uint16
	// peerSignatureAlgorithm, if not zero, is the signature algorithm that the
	// peer should have used in the handshake.
	peerSignatureAlgorithm signatureAlgorithm
	// curveID, if not zero, is the curve that the handshake should have used.
	curveID CurveID
	// peerCertificate, if not nil, is the credential the peer is expected to
	// send.
	peerCertificate *Credential
	// quicTransportParams contains the QUIC transport parameters that are to be
	// sent by the peer using codepoint 57.
	quicTransportParams []byte
	// quicTransportParamsLegacy contains the QUIC transport parameters that are
	// to be sent by the peer using legacy codepoint 0xffa5.
	quicTransportParamsLegacy []byte
	// peerApplicationSettings are the expected application settings for the
	// connection. If nil, no application settings are expected.
	peerApplicationSettings []byte
	// peerApplicationSettingsOld are the expected application settings for
	// the connection that are to be sent by the peer using old codepoint.
	// If nil, no application settings are expected.
	peerApplicationSettingsOld []byte
	// echAccepted is whether ECH should have been accepted on this connection.
	echAccepted bool
}

type testCase struct {
	testType      testType
	protocol      protocol
	name          string
	config        Config
	shouldFail    bool
	expectedError string
	// expectedLocalError, if not empty, contains a substring that must be
	// found in the local error.
	expectedLocalError string
	// expectations contains test expectations for the initial
	// connection.
	expectations connectionExpectations
	// resumeExpectations, if non-nil, contains test expectations for the
	// resumption connection. If nil, |expectations| is used.
	resumeExpectations *connectionExpectations
	// messageLen is the length, in bytes, of the test message that will be
	// sent.
	messageLen int
	// messageCount is the number of test messages that will be sent.
	messageCount int
	// resumeSession controls whether a second connection should be tested
	// which attempts to resume the first session.
	resumeSession bool
	// resumeRenewedSession controls whether a third connection should be
	// tested which attempts to resume the second connection's session.
	resumeRenewedSession bool
	// expectResumeRejected, if true, specifies that the attempted
	// resumption must be rejected by the client. This is only valid for a
	// serverTest.
	expectResumeRejected bool
	// resumeConfig, if not nil, points to a Config to be used on
	// resumption. Unless newSessionsOnResume is set,
	// SessionTicketKey, ServerSessionCache, and
	// ClientSessionCache are copied from the initial connection's
	// config. If nil, the initial connection's config is used.
	resumeConfig *Config
	// newSessionsOnResume, if true, will cause resumeConfig to
	// use a different session resumption context.
	newSessionsOnResume bool
	// noSessionCache, if true, will cause the server to run without a
	// session cache.
	noSessionCache bool
	// sendPrefix sends a prefix on the socket before actually performing a
	// handshake.
	sendPrefix string
	// shimWritesFirst controls whether the shim sends an initial "hello"
	// message before doing a roundtrip with the runner.
	shimWritesFirst bool
	// readWithUnfinishedWrite behaves like shimWritesFirst, but the shim
	// does not complete the write until responding to the first runner
	// message.
	readWithUnfinishedWrite bool
	// shimShutsDown, if true, runs a test where the shim shuts down the
	// connection immediately after the handshake rather than echoing
	// messages from the runner. The runner will default to not sending
	// application data.
	shimShutsDown bool
	// renegotiate indicates the number of times the connection should be
	// renegotiated during the exchange.
	renegotiate int
	// sendHalfHelloRequest, if true, causes the server to send half a
	// HelloRequest when the handshake completes.
	sendHalfHelloRequest bool
	// renegotiateCiphers is a list of ciphersuite ids that will be
	// switched in just before renegotiation.
	renegotiateCiphers []uint16
	// replayWrites, if true, configures the underlying transport
	// to replay every write it makes in DTLS tests.
	replayWrites bool
	// damageFirstWrite, if true, configures the underlying transport to
	// damage the final byte of the first application data write.
	damageFirstWrite bool
	// exportKeyingMaterial, if non-zero, configures the test to exchange
	// keying material and verify they match.
	exportKeyingMaterial int
	exportLabel          string
	exportContext        string
	useExportContext     bool
	// flags, if not empty, contains a list of command-line flags that will
	// be passed to the shim program.
	flags []string
	// testTLSUnique, if true, causes the shim to send the tls-unique value
	// which will be compared against the expected value.
	testTLSUnique bool
	// sendEmptyRecords is the number of consecutive empty records to send
	// before each test message.
	sendEmptyRecords int
	// sendWarningAlerts is the number of consecutive warning alerts to send
	// before each test message.
	sendWarningAlerts int
	// sendUserCanceledAlerts is the number of consecutive user_canceled alerts to
	// send before each test message.
	sendUserCanceledAlerts int
	// sendBogusAlertType, if true, causes a bogus alert of invalid type to
	// be sent before each test message.
	sendBogusAlertType bool
	// sendKeyUpdates is the number of consecutive key updates to send
	// before and after the test message.
	sendKeyUpdates int
	// keyUpdateRequest is the KeyUpdateRequest value to send in KeyUpdate messages.
	keyUpdateRequest byte
	// expectUnsolicitedKeyUpdate makes the test expect a one or more KeyUpdate
	// messages while reading data from the shim. Don't use this in combination
	// with any of the fields that send a KeyUpdate otherwise any received
	// KeyUpdate might not be as unsolicited as expected.
	expectUnsolicitedKeyUpdate bool
	// expectMessageDropped, if true, means the test message is expected to
	// be dropped by the client rather than echoed back.
	expectMessageDropped bool
	// shimPrefix is the prefix that the shim will send to the server.
	shimPrefix string
	// resumeShimPrefix is the prefix that the shim will send to the server on a
	// resumption.
	resumeShimPrefix string
	// exportTrafficSecrets, if true, configures the test to export the TLS 1.3
	// traffic secrets and confirms that they match.
	exportTrafficSecrets bool
	// skipTransportParamsConfig, if true, will skip automatic configuration of
	// sending QUIC transport parameters when protocol == quic.
	skipTransportParamsConfig bool
	// skipQUICALPNConfig, if true, will skip automatic configuration of
	// sending a fake ALPN when protocol == quic.
	skipQUICALPNConfig bool
	// earlyData, if true, configures default settings for an early data test.
	// expectEarlyDataRejected controls whether the test is for early data
	// accept or reject. In a client test, the shim will be configured to send
	// an initial write in early data which, on accept, the runner will enforce.
	// In a server test, the runner will send some default message in early
	// data, which the shim is expected to echo in half-RTT.
	earlyData bool
	// expectEarlyDataRejected, if earlyData is true, is whether early data is
	// expected to be rejected. In a client test, this controls whether the shim
	// should retry for early rejection. In a server test, this is whether the
	// test expects the shim to reject early data.
	expectEarlyDataRejected bool
	// skipSplitHandshake, if true, will skip the generation of a split
	// handshake copy of the test.
	skipSplitHandshake bool
	// skipHints, if true, will skip the generation of a handshake hints copy of
	// the test.
	skipHints bool
	// skipVersionNameCheck, if true, will skip the consistency check between
	// test name and the versions.
	skipVersionNameCheck bool
	// shimCertificate, if not nil, is the default credential which should be
	// configured at the shim. If set, it must be an X.509 credential.
	shimCertificate *Credential
	// handshakerCertificate, if not nil, overrides the default credential which
	// on the handshaker.
	handshakerCertificate *Credential
	// shimCredentials is a list of credentials which should be configured at
	// the shim. It differs from shimCertificate only in whether the old or
	// new APIs are used.
	shimCredentials []*Credential
}

var testCases []testCase

func appendTranscript(path string, data []byte) error {
	if len(data) == 0 {
		return nil
	}

	settings, err := os.ReadFile(path)
	if err != nil {
		if !os.IsNotExist(err) {
			return err
		}
		// If the shim aborted before writing a file, use a default
		// settings block, so the transcript is still somewhat valid.
		settings = []byte{0, 0} // kDataTag
	}

	settings = append(settings, data...)
	return os.WriteFile(path, settings, 0644)
}

// A timeoutConn implements an idle timeout on each Read and Write operation.
type timeoutConn struct {
	net.Conn
	timeout time.Duration
}

func (t *timeoutConn) Read(b []byte) (int, error) {
	if !*useGDB {
		if err := t.SetReadDeadline(time.Now().Add(t.timeout)); err != nil {
			return 0, err
		}
	}
	return t.Conn.Read(b)
}

func (t *timeoutConn) Write(b []byte) (int, error) {
	if !*useGDB {
		if err := t.SetWriteDeadline(time.Now().Add(t.timeout)); err != nil {
			return 0, err
		}
	}
	return t.Conn.Write(b)
}

func doExchange(test *testCase, config *Config, conn net.Conn, isResume bool, transcripts *[][]byte, num int) error {
	if !test.noSessionCache {
		if config.ClientSessionCache == nil {
			config.ClientSessionCache = NewLRUClientSessionCache(1)
		}
		if config.ServerSessionCache == nil {
			config.ServerSessionCache = NewLRUServerSessionCache(1)
		}
	}
	if test.testType != clientTest {
		// Supply a ServerName to ensure a constant session cache key,
		// rather than falling back to net.Conn.RemoteAddr.
		if len(config.ServerName) == 0 {
			config.ServerName = "test"
		}
	}

	if *fuzzer {
		config.Bugs.NullAllCiphers = true
	}
	if *deterministic {
		config.Time = func() time.Time { return time.Unix(1234, 1234) }
	}

	if !useDebugger() {
		conn = &timeoutConn{conn, *idleTimeout}
	}

	if test.protocol == dtls {
		config.Bugs.PacketAdaptor = newPacketAdaptor(conn)
		conn = config.Bugs.PacketAdaptor
	}

	if *flagDebug || len(*transcriptDir) != 0 {
		local, peer := "client", "server"
		if test.testType == clientTest {
			local, peer = peer, local
		}
		connDebug := &recordingConn{
			Conn:       conn,
			isDatagram: test.protocol == dtls,
			local:      local,
			peer:       peer,
		}
		conn = connDebug
		if *flagDebug {
			defer connDebug.WriteTo(os.Stdout)
		}
		if len(*transcriptDir) != 0 {
			defer func() {
				if num == len(*transcripts) {
					*transcripts = append(*transcripts, connDebug.Transcript())
				} else {
					panic("transcripts are out of sync")
				}
			}()

			// Record ClientHellos for the decode_client_hello_inner fuzzer.
			var clientHelloCount int
			config.Bugs.RecordClientHelloInner = func(encodedInner, outer []byte) error {
				name := fmt.Sprintf("%s-%d-%d", test.name, num, clientHelloCount)
				clientHelloCount++
				dir := filepath.Join(*transcriptDir, "decode_client_hello_inner")
				if err := os.MkdirAll(dir, 0755); err != nil {
					return err
				}
				bb := cryptobyte.NewBuilder(nil)
				addUint24LengthPrefixedBytes(bb, encodedInner)
				bb.AddBytes(outer)
				return os.WriteFile(filepath.Join(dir, name), bb.BytesOrPanic(), 0644)
			}
		}

		if config.Bugs.PacketAdaptor != nil {
			config.Bugs.PacketAdaptor.debug = connDebug
		}
	}
	if test.protocol == quic {
		config.Bugs.MockQUICTransport = newMockQUICTransport(conn)
		// The MockQUICTransport will panic if Read or Write is
		// called. When a MockQUICTransport is set, separate
		// methods should be used to actually read and write
		// records. By setting the conn to it here, it ensures
		// Read or Write aren't accidentally used instead of the
		// methods provided by MockQUICTransport.
		conn = config.Bugs.MockQUICTransport
	}

	if test.replayWrites {
		conn = newReplayAdaptor(conn)
	}

	var connDamage *damageAdaptor
	if test.damageFirstWrite {
		connDamage = newDamageAdaptor(conn)
		conn = connDamage
	}

	if test.sendPrefix != "" {
		if _, err := conn.Write([]byte(test.sendPrefix)); err != nil {
			return err
		}
	}

	var tlsConn *Conn
	if test.testType == clientTest {
		if test.protocol == dtls {
			tlsConn = DTLSServer(conn, config)
		} else {
			tlsConn = Server(conn, config)
		}
	} else {
		config.InsecureSkipVerify = true
		if test.protocol == dtls {
			tlsConn = DTLSClient(conn, config)
		} else {
			tlsConn = Client(conn, config)
		}
	}
	defer tlsConn.Close()

	if err := tlsConn.Handshake(); err != nil {
		return err
	}

	expectations := &test.expectations
	if isResume && test.resumeExpectations != nil {
		expectations = test.resumeExpectations
	}
	connState := tlsConn.ConnectionState()
	if vers := connState.Version; expectations.version != 0 && vers != expectations.version {
		return fmt.Errorf("got version %x, expected %x", vers, expectations.version)
	}

	if cipher := connState.CipherSuite; expectations.cipher != 0 && cipher != expectations.cipher {
		return fmt.Errorf("got cipher %x, expected %x", cipher, expectations.cipher)
	}
	if didResume := connState.DidResume; isResume && didResume == test.expectResumeRejected {
		return fmt.Errorf("didResume is %t, but we expected the opposite", didResume)
	}

	if expectations.channelID {
		channelID := connState.ChannelID
		if channelID == nil {
			return fmt.Errorf("no channel ID negotiated")
		}
		if channelID.Curve != channelIDKey.Curve ||
			channelIDKey.X.Cmp(channelIDKey.X) != 0 ||
			channelIDKey.Y.Cmp(channelIDKey.Y) != 0 {
			return fmt.Errorf("incorrect channel ID")
		}
	} else if connState.ChannelID != nil {
		return fmt.Errorf("channel ID unexpectedly negotiated")
	}

	if expected := expectations.nextProto; expected != "" {
		if actual := connState.NegotiatedProtocol; actual != expected {
			return fmt.Errorf("next proto mismatch: got %s, wanted %s", actual, expected)
		}
	}

	if expectations.noNextProto {
		if actual := connState.NegotiatedProtocol; actual != "" {
			return fmt.Errorf("got unexpected next proto %s", actual)
		}
	}

	if expectations.nextProtoType != 0 {
		if (expectations.nextProtoType == alpn) != connState.NegotiatedProtocolFromALPN {
			return fmt.Errorf("next proto type mismatch")
		}
	}

	if expectations.peerApplicationSettings != nil {
		if !connState.HasApplicationSettings {
			return errors.New("application settings should have been negotiated")
		}
		if !bytes.Equal(connState.PeerApplicationSettings, expectations.peerApplicationSettings) {
			return fmt.Errorf("peer application settings mismatch: got %q, wanted %q", connState.PeerApplicationSettings, expectations.peerApplicationSettings)
		}
	} else if connState.HasApplicationSettings {
		return errors.New("application settings unexpectedly negotiated")
	}

	if expectations.peerApplicationSettingsOld != nil {
		if !connState.HasApplicationSettingsOld {
			return errors.New("old application settings should have been negotiated")
		}
		if !bytes.Equal(connState.PeerApplicationSettingsOld, expectations.peerApplicationSettingsOld) {
			return fmt.Errorf("old peer application settings mismatch: got %q, wanted %q", connState.PeerApplicationSettingsOld, expectations.peerApplicationSettingsOld)
		}
	} else if connState.HasApplicationSettingsOld {
		return errors.New("old application settings unexpectedly negotiated")
	}

	if p := connState.SRTPProtectionProfile; p != expectations.srtpProtectionProfile {
		return fmt.Errorf("SRTP profile mismatch: got %d, wanted %d", p, expectations.srtpProtectionProfile)
	}

	if expected := expectations.peerSignatureAlgorithm; expected != 0 && expected != connState.PeerSignatureAlgorithm {
		return fmt.Errorf("expected peer to use signature algorithm %04x, but got %04x", expected, connState.PeerSignatureAlgorithm)
	}

	if expected := expectations.curveID; expected != 0 && expected != connState.CurveID {
		return fmt.Errorf("expected peer to use curve %04x, but got %04x", expected, connState.CurveID)
	}

	if expected := expectations.peerCertificate; expected != nil {
		if len(connState.PeerCertificates) != len(expected.Certificate) {
			return fmt.Errorf("expected peer to send %d certificates, but got %d", len(connState.PeerCertificates), len(expected.Certificate))
		}
		for i, cert := range connState.PeerCertificates {
			if !bytes.Equal(cert.Raw, expected.Certificate[i]) {
				return fmt.Errorf("peer certificate %d did not match", i+1)
			}
		}

		if !bytes.Equal(connState.OCSPResponse, expected.OCSPStaple) {
			return fmt.Errorf("peer OCSP response did not match")
		}

		if !bytes.Equal(connState.SCTList, expected.SignedCertificateTimestampList) {
			return fmt.Errorf("peer SCT list did not match")
		}

		if expected.Type == CredentialTypeDelegated {
			if connState.PeerDelegatedCredential == nil {
				return fmt.Errorf("peer unexpectedly did not use delegated credentials")
			}
			if !bytes.Equal(expected.DelegatedCredential, connState.PeerDelegatedCredential) {
				return fmt.Errorf("peer delegated credential did not match")
			}
		} else if connState.PeerDelegatedCredential != nil {
			return fmt.Errorf("peer unexpectedly used delegated credentials")
		}
	}

	if len(expectations.quicTransportParams) > 0 {
		if !bytes.Equal(expectations.quicTransportParams, connState.QUICTransportParams) {
			return errors.New("Peer did not send expected QUIC transport params")
		}
	}

	if len(expectations.quicTransportParamsLegacy) > 0 {
		if !bytes.Equal(expectations.quicTransportParamsLegacy, connState.QUICTransportParamsLegacy) {
			return errors.New("Peer did not send expected legacy QUIC transport params")
		}
	}

	if expectations.echAccepted {
		if !connState.ECHAccepted {
			return errors.New("tls: server did not accept ECH")
		}
	} else {
		if connState.ECHAccepted {
			return errors.New("tls: server unexpectedly accepted ECH")
		}
	}

	if test.exportKeyingMaterial > 0 {
		actual := make([]byte, test.exportKeyingMaterial)
		if _, err := io.ReadFull(tlsConn, actual); err != nil {
			return err
		}
		expected, err := tlsConn.ExportKeyingMaterial(test.exportKeyingMaterial, []byte(test.exportLabel), []byte(test.exportContext), test.useExportContext)
		if err != nil {
			return err
		}
		if !bytes.Equal(actual, expected) {
			return fmt.Errorf("keying material mismatch; got %x, wanted %x", actual, expected)
		}
	}

	if test.exportTrafficSecrets {
		secretLenBytes := make([]byte, 2)
		if _, err := io.ReadFull(tlsConn, secretLenBytes); err != nil {
			return err
		}
		secretLen := binary.LittleEndian.Uint16(secretLenBytes)

		theirReadSecret := make([]byte, secretLen)
		theirWriteSecret := make([]byte, secretLen)
		if _, err := io.ReadFull(tlsConn, theirReadSecret); err != nil {
			return err
		}
		if _, err := io.ReadFull(tlsConn, theirWriteSecret); err != nil {
			return err
		}

		myReadSecret := tlsConn.in.trafficSecret
		myWriteSecret := tlsConn.out.trafficSecret
		if !bytes.Equal(myWriteSecret, theirReadSecret) {
			return fmt.Errorf("read traffic-secret mismatch; got %x, wanted %x", theirReadSecret, myWriteSecret)
		}
		if !bytes.Equal(myReadSecret, theirWriteSecret) {
			return fmt.Errorf("write traffic-secret mismatch; got %x, wanted %x", theirWriteSecret, myReadSecret)
		}
	}

	if test.testTLSUnique {
		var peersValue [12]byte
		if _, err := io.ReadFull(tlsConn, peersValue[:]); err != nil {
			return err
		}
		expected := tlsConn.ConnectionState().TLSUnique
		if !bytes.Equal(peersValue[:], expected) {
			return fmt.Errorf("tls-unique mismatch: peer sent %x, but %x was expected", peersValue[:], expected)
		}
	}

	if test.sendHalfHelloRequest {
		tlsConn.SendHalfHelloRequest()
	}

	shimPrefix := test.shimPrefix
	if isResume {
		shimPrefix = test.resumeShimPrefix
	}
	if test.shimWritesFirst || test.readWithUnfinishedWrite {
		shimPrefix = shimInitialWrite
	}
	if test.renegotiate > 0 {
		// If readWithUnfinishedWrite is set, the shim prefix will be
		// available later.
		if shimPrefix != "" && !test.readWithUnfinishedWrite {
			var buf = make([]byte, len(shimPrefix))
			_, err := io.ReadFull(tlsConn, buf)
			if err != nil {
				return err
			}
			if string(buf) != shimPrefix {
				return fmt.Errorf("bad initial message %v vs %v", string(buf), shimPrefix)
			}
			shimPrefix = ""
		}

		if test.renegotiateCiphers != nil {
			config.CipherSuites = test.renegotiateCiphers
		}
		for i := 0; i < test.renegotiate; i++ {
			if err := tlsConn.Renegotiate(); err != nil {
				return err
			}
		}
	} else if test.renegotiateCiphers != nil {
		panic("renegotiateCiphers without renegotiate")
	}

	if test.damageFirstWrite {
		connDamage.setDamage(true)
		tlsConn.Write([]byte("DAMAGED WRITE"))
		connDamage.setDamage(false)
	}

	messageLen := test.messageLen
	if messageLen < 0 {
		if test.protocol == dtls {
			return fmt.Errorf("messageLen < 0 not supported for DTLS tests")
		}
		// Read until EOF.
		_, err := io.Copy(io.Discard, tlsConn)
		return err
	}
	if messageLen == 0 {
		messageLen = 32
	}

	messageCount := test.messageCount
	// shimShutsDown sets the default message count to zero.
	if messageCount == 0 && !test.shimShutsDown {
		messageCount = 1
	}

	for j := 0; j < messageCount; j++ {
		for i := 0; i < test.sendKeyUpdates; i++ {
			tlsConn.SendKeyUpdate(test.keyUpdateRequest)
		}

		for i := 0; i < test.sendEmptyRecords; i++ {
			tlsConn.Write(nil)
		}

		for i := 0; i < test.sendWarningAlerts; i++ {
			tlsConn.SendAlert(alertLevelWarning, alertUnexpectedMessage)
		}

		for i := 0; i < test.sendUserCanceledAlerts; i++ {
			tlsConn.SendAlert(alertLevelWarning, alertUserCanceled)
		}

		if test.sendBogusAlertType {
			tlsConn.SendAlert(0x42, alertUnexpectedMessage)
		}

		testMessage := make([]byte, messageLen)
		for i := range testMessage {
			testMessage[i] = 0x42 ^ byte(j)
		}
		tlsConn.Write(testMessage)

		// Consume the shim prefix if needed.
		if shimPrefix != "" {
			var buf = make([]byte, len(shimPrefix))
			_, err := io.ReadFull(tlsConn, buf)
			if err != nil {
				return err
			}
			if string(buf) != shimPrefix {
				return fmt.Errorf("bad initial message %v vs %v", string(buf), shimPrefix)
			}
			shimPrefix = ""
		}

		if test.shimShutsDown || test.expectMessageDropped {
			// The shim will not respond.
			continue
		}

		// Process the KeyUpdate ACK. However many KeyUpdates the runner
		// sends, the shim should respond only once.
		if test.sendKeyUpdates > 0 && test.keyUpdateRequest == keyUpdateRequested {
			if err := tlsConn.ReadKeyUpdateACK(); err != nil {
				return err
			}
		}

		buf := make([]byte, len(testMessage))
		if test.protocol == dtls {
			bufTmp := make([]byte, len(buf)+1)
			n, err := tlsConn.Read(bufTmp)
			if err != nil {
				return err
			}
			if config.Bugs.SplitAndPackAppData {
				m, err := tlsConn.Read(bufTmp[n:])
				if err != nil {
					return err
				}
				n += m
			}
			if n != len(buf) {
				return fmt.Errorf("bad reply; length mismatch (%d vs %d)", n, len(buf))
			}
			copy(buf, bufTmp)
		} else {
			_, err := io.ReadFull(tlsConn, buf)
			if err != nil {
				return err
			}
		}

		for i, v := range buf {
			if v != testMessage[i]^0xff {
				return fmt.Errorf("bad reply contents at byte %d; got %q and wanted %q", i, buf, testMessage)
			}
		}

		if seen := tlsConn.keyUpdateSeen; seen != test.expectUnsolicitedKeyUpdate {
			return fmt.Errorf("keyUpdateSeen (%t) != expectUnsolicitedKeyUpdate", seen)
		}
	}

	return nil
}

const xtermSize = "140x50"

func valgrindOf(dbAttach bool, path string, args ...string) *exec.Cmd {
	valgrindArgs := []string{"--error-exitcode=99", "--track-origins=yes", "--leak-check=full", "--quiet"}
	if dbAttach {
		valgrindArgs = append(valgrindArgs, "--db-attach=yes", "--db-command=xterm -geometry "+xtermSize+" -e gdb -nw %f %p")
	}
	valgrindArgs = append(valgrindArgs, path)
	valgrindArgs = append(valgrindArgs, args...)

	return exec.Command("valgrind", valgrindArgs...)
}

func gdbOf(path string, args ...string) *exec.Cmd {
	xtermArgs := []string{"-geometry", xtermSize, "-e", "gdb", "--args"}
	xtermArgs = append(xtermArgs, path)
	xtermArgs = append(xtermArgs, args...)

	return exec.Command("xterm", xtermArgs...)
}

func lldbOf(path string, args ...string) *exec.Cmd {
	xtermArgs := []string{"-geometry", xtermSize, "-e", "lldb", "--"}
	xtermArgs = append(xtermArgs, path)
	xtermArgs = append(xtermArgs, args...)

	return exec.Command("xterm", xtermArgs...)
}

func rrOf(path string, args ...string) *exec.Cmd {
	rrArgs := []string{"record", path}
	rrArgs = append(rrArgs, args...)
	return exec.Command("rr", rrArgs...)
}

func removeFirstLineIfSuffix(s, suffix string) string {
	idx := strings.IndexByte(s, '\n')
	if idx < 0 {
		return s
	}
	if strings.HasSuffix(s[:idx], suffix) {
		return s[idx+1:]
	}
	return s
}

var (
	errMoreMallocs   = errors.New("child process did not exhaust all allocation calls")
	errUnimplemented = errors.New("child process does not implement needed flags")
)

type shimProcess struct {
	cmd *exec.Cmd
	// done is closed when the process has exited. At that point, childErr may be
	// read for the result.
	done           chan struct{}
	childErr       error
	listener       *shimListener
	stdout, stderr bytes.Buffer
}

// newShimProcess starts a new shim with the specified executable, flags, and
// environment. It internally creates a TCP listener and adds the the -port
// flag.
func newShimProcess(dispatcher *shimDispatcher, shimPath string, flags []string, env []string) (*shimProcess, error) {
	listener, err := dispatcher.NewShim()
	if err != nil {
		return nil, err
	}

	shim := &shimProcess{listener: listener}
	cmdFlags := []string{
		"-port", strconv.Itoa(listener.Port()),
		"-shim-id", strconv.FormatUint(listener.ShimID(), 10),
	}
	if listener.IsIPv6() {
		cmdFlags = append(cmdFlags, "-ipv6")
	}
	cmdFlags = append(cmdFlags, flags...)

	if *useValgrind {
		shim.cmd = valgrindOf(false, shimPath, cmdFlags...)
	} else if *useGDB {
		shim.cmd = gdbOf(shimPath, cmdFlags...)
	} else if *useLLDB {
		shim.cmd = lldbOf(shimPath, cmdFlags...)
	} else if *useRR {
		shim.cmd = rrOf(shimPath, cmdFlags...)
	} else {
		shim.cmd = exec.Command(shimPath, cmdFlags...)
	}
	shim.cmd.Stdin = os.Stdin
	shim.cmd.Stdout = &shim.stdout
	shim.cmd.Stderr = &shim.stderr
	shim.cmd.Env = env

	if err := shim.cmd.Start(); err != nil {
		shim.listener.Close()
		return nil, err
	}

	shim.done = make(chan struct{})
	go func() {
		shim.childErr = shim.cmd.Wait()
		shim.listener.Close()
		close(shim.done)
	}()
	return shim, nil
}

// accept returns a new TCP connection with the shim process, or returns an
// error on timeout or shim exit.
func (s *shimProcess) accept() (net.Conn, error) {
	var deadline time.Time
	if !useDebugger() {
		deadline = time.Now().Add(*idleTimeout)
	}
	return s.listener.Accept(deadline)
}

// wait finishes the test and waits for the shim process to exit.
func (s *shimProcess) wait() error {
	// Close the listener now. This is to avoid hangs if the shim tries to open
	// more connections than expected.
	s.listener.Close()

	if !useDebugger() {
		waitTimeout := time.AfterFunc(*idleTimeout, func() {
			s.cmd.Process.Kill()
		})
		defer waitTimeout.Stop()
	}

	<-s.done
	return s.childErr
}

// close releases resources associated with the shimProcess. This is safe to
// call before or after |wait|.
func (s *shimProcess) close() {
	s.listener.Close()
	s.cmd.Process.Kill()
}

func doExchanges(test *testCase, shim *shimProcess, resumeCount int, transcripts *[][]byte) error {
	config := test.config
	if *deterministic {
		config.Rand = &deterministicRand{}
	}

	conn, err := shim.accept()
	if err != nil {
		return err
	}
	err = doExchange(test, &config, conn, false /* not a resumption */, transcripts, 0)
	conn.Close()
	if err != nil {
		return err
	}

	nextTicketKey := config.SessionTicketKey
	for i := 0; i < resumeCount; i++ {
		var resumeConfig Config
		if test.resumeConfig != nil {
			resumeConfig = *test.resumeConfig
			resumeConfig.Rand = config.Rand
			if resumeConfig.Credential == nil {
				resumeConfig.Credential = config.Credential
			}
		} else {
			resumeConfig = config
		}

		if test.newSessionsOnResume {
			resumeConfig.ClientSessionCache = nil
			resumeConfig.ServerSessionCache = nil
			if _, err := resumeConfig.rand().Read(resumeConfig.SessionTicketKey[:]); err != nil {
				return err
			}
		} else {
			resumeConfig.ClientSessionCache = config.ClientSessionCache
			resumeConfig.ServerSessionCache = config.ServerSessionCache
			// Rotate the ticket keys between each connection, with each connection
			// encrypting with next connection's keys. This ensures that we test
			// the renewed sessions.
			resumeConfig.SessionTicketKey = nextTicketKey
			if _, err := resumeConfig.rand().Read(nextTicketKey[:]); err != nil {
				return err
			}
			resumeConfig.Bugs.EncryptSessionTicketKey = &nextTicketKey
		}

		var connResume net.Conn
		connResume, err = shim.accept()
		if err != nil {
			return err
		}
		err = doExchange(test, &resumeConfig, connResume, true /* resumption */, transcripts, i+1)
		connResume.Close()
		if err != nil {
			return err
		}
	}

	return nil
}

func translateExpectedError(errorStr string) string {
	if translated, ok := shimConfig.ErrorMap[errorStr]; ok {
		return translated
	}

	if *looseErrors {
		return ""
	}

	return errorStr
}

// shimInitialWrite is the data we expect from the shim when the
// -shim-writes-first flag is used.
const shimInitialWrite = "hello"

func appendCredentialFlags(flags []string, cred *Credential, prefix string, newCredential bool) []string {
	if newCredential {
		switch cred.Type {
		case CredentialTypeX509:
			flags = append(flags, prefix+"-new-x509-credential")
		case CredentialTypeDelegated:
			flags = append(flags, prefix+"-new-delegated-credential")
		default:
			panic(fmt.Errorf("unknown credential type %d", cred.Type))
		}
	} else if cred.Type != CredentialTypeX509 {
		panic("default credential must be X.509")
	}

	if len(cred.ChainPath) != 0 {
		flags = append(flags, prefix+"-cert-file", cred.ChainPath)
	}
	if len(cred.KeyPath) != 0 {
		flags = append(flags, prefix+"-key-file", cred.KeyPath)
	}
	if len(cred.OCSPStaple) != 0 {
		flags = append(flags, prefix+"-ocsp-response", base64FlagValue(cred.OCSPStaple))
	}
	if len(cred.SignedCertificateTimestampList) != 0 {
		flags = append(flags, prefix+"-signed-cert-timestamps", base64FlagValue(cred.SignedCertificateTimestampList))
	}
	for _, sigAlg := range cred.SignatureAlgorithms {
		flags = append(flags, prefix+"-signing-prefs", strconv.Itoa(int(sigAlg)))
	}
	if len(cred.DelegatedCredential) != 0 {
		flags = append(flags, prefix+"-delegated-credential", base64FlagValue(cred.DelegatedCredential))
	}
	return flags
}

func runTest(dispatcher *shimDispatcher, statusChan chan statusMsg, test *testCase, shimPath string, mallocNumToFail int64) error {
	// Help debugging panics on the Go side.
	defer func() {
		if r := recover(); r != nil {
			fmt.Fprintf(os.Stderr, "Test '%s' panicked.\n", test.name)
			panic(r)
		}
	}()

	var flags []string
	if len(*shimExtraFlags) > 0 {
		flags = strings.Split(*shimExtraFlags, ";")
	}
	if test.testType == serverTest {
		flags = append(flags, "-server")
	}

	// Configure the default credential.
	shimCertificate := test.shimCertificate
	if shimCertificate == nil && len(test.shimCredentials) == 0 && test.testType == serverTest && len(test.config.PreSharedKey) == 0 {
		shimCertificate = &rsaCertificate
	}
	if shimCertificate != nil {
		var shimPrefix string
		if test.handshakerCertificate != nil {
			shimPrefix = "-on-shim"
		}
		flags = appendCredentialFlags(flags, shimCertificate, shimPrefix, false)
	}
	if test.handshakerCertificate != nil {
		flags = appendCredentialFlags(flags, test.handshakerCertificate, "-on-handshaker", false)
	}

	// Configure any additional credentials.
	for _, cred := range test.shimCredentials {
		flags = appendCredentialFlags(flags, cred, "", true)
	}

	if test.protocol == dtls {
		flags = append(flags, "-dtls")
	} else if test.protocol == quic {
		flags = append(flags, "-quic")
		if !test.skipTransportParamsConfig {
			test.config.QUICTransportParams = []byte{1, 2}
			test.config.QUICTransportParamsUseLegacyCodepoint = QUICUseCodepointStandard
			if test.resumeConfig != nil {
				test.resumeConfig.QUICTransportParams = []byte{1, 2}
				test.resumeConfig.QUICTransportParamsUseLegacyCodepoint = QUICUseCodepointStandard
			}
			test.expectations.quicTransportParams = []byte{3, 4}
			if test.resumeExpectations != nil {
				test.resumeExpectations.quicTransportParams = []byte{3, 4}
			}
			useCodepointFlag := "0"
			if test.config.QUICTransportParamsUseLegacyCodepoint == QUICUseCodepointLegacy {
				useCodepointFlag = "1"
			}
			flags = append(flags,
				"-quic-transport-params",
				base64FlagValue([]byte{3, 4}),
				"-expect-quic-transport-params",
				base64FlagValue([]byte{1, 2}),
				"-quic-use-legacy-codepoint", useCodepointFlag)
		}
		if !test.skipQUICALPNConfig {
			flags = append(flags,
				[]string{
					"-advertise-alpn", "\x03foo",
					"-select-alpn", "foo",
					"-expect-alpn", "foo",
				}...)
			test.config.NextProtos = []string{"foo"}
			if test.resumeConfig != nil {
				test.resumeConfig.NextProtos = []string{"foo"}
			}
			test.expectations.nextProto = "foo"
			test.expectations.nextProtoType = alpn
			if test.resumeExpectations != nil {
				test.resumeExpectations.nextProto = "foo"
				test.resumeExpectations.nextProtoType = alpn
			}
		}
	}

	if test.earlyData {
		if !test.resumeSession {
			panic("earlyData set without resumeSession in " + test.name)
		}

		resumeConfig := test.resumeConfig
		if resumeConfig == nil {
			resumeConfig = &test.config
		}
		if test.expectEarlyDataRejected {
			flags = append(flags, "-on-resume-expect-reject-early-data")
		} else {
			flags = append(flags, "-on-resume-expect-accept-early-data")
		}

		if test.protocol == quic {
			// QUIC requires an early data context string.
			flags = append(flags, "-quic-early-data-context", "context")
		}

		flags = append(flags, "-enable-early-data")
		if test.testType == clientTest {
			// Configure the runner with default maximum early data.
			flags = append(flags, "-expect-ticket-supports-early-data")
			if test.config.MaxEarlyDataSize == 0 {
				test.config.MaxEarlyDataSize = 16384
			}
			if resumeConfig.MaxEarlyDataSize == 0 {
				resumeConfig.MaxEarlyDataSize = 16384
			}

			// Configure the shim to send some data in early data.
			flags = append(flags, "-on-resume-shim-writes-first")
			if resumeConfig.Bugs.ExpectEarlyData == nil {
				resumeConfig.Bugs.ExpectEarlyData = [][]byte{[]byte(shimInitialWrite)}
			}
		} else {
			// By default, send some early data and expect half-RTT data response.
			if resumeConfig.Bugs.SendEarlyData == nil {
				resumeConfig.Bugs.SendEarlyData = [][]byte{{1, 2, 3, 4}}
			}
			if resumeConfig.Bugs.ExpectHalfRTTData == nil {
				resumeConfig.Bugs.ExpectHalfRTTData = [][]byte{{254, 253, 252, 251}}
			}
			resumeConfig.Bugs.ExpectEarlyDataAccepted = !test.expectEarlyDataRejected
		}
	}

	var resumeCount int
	if test.resumeSession {
		resumeCount++
		if test.resumeRenewedSession {
			resumeCount++
		}
	}

	if resumeCount > 0 {
		flags = append(flags, "-resume-count", strconv.Itoa(resumeCount))
	}

	if test.shimWritesFirst {
		flags = append(flags, "-shim-writes-first")
	}

	if test.readWithUnfinishedWrite {
		flags = append(flags, "-read-with-unfinished-write")
	}

	if test.shimShutsDown {
		flags = append(flags, "-shim-shuts-down")
	}

	if test.exportKeyingMaterial > 0 {
		flags = append(flags, "-export-keying-material", strconv.Itoa(test.exportKeyingMaterial))
		if test.useExportContext {
			flags = append(flags, "-use-export-context")
		}
	}
	if test.exportKeyingMaterial > 0 {
		flags = append(flags, "-export-label", test.exportLabel)
		flags = append(flags, "-export-context", test.exportContext)
	}

	if test.exportTrafficSecrets {
		flags = append(flags, "-export-traffic-secrets")
	}

	if test.expectResumeRejected {
		flags = append(flags, "-expect-session-miss")
	}

	if test.testTLSUnique {
		flags = append(flags, "-tls-unique")
	}

	if *waitForDebugger {
		flags = append(flags, "-wait-for-debugger")
	}

	var transcriptPrefix string
	var transcripts [][]byte
	if len(*transcriptDir) != 0 {
		protocol := "tls"
		if test.protocol == dtls {
			protocol = "dtls"
		} else if test.protocol == quic {
			protocol = "quic"
		}

		side := "client"
		if test.testType == serverTest {
			side = "server"
		}

		dir := filepath.Join(*transcriptDir, protocol, side)
		if err := os.MkdirAll(dir, 0755); err != nil {
			return err
		}
		transcriptPrefix = filepath.Join(dir, test.name+"-")
		flags = append(flags, "-write-settings", transcriptPrefix)
	}

	if test.testType == clientTest && test.config.Credential == nil {
		test.config.Credential = &rsaCertificate
	}
	if test.config.Credential != nil {
		flags = append(flags, "-trust-cert", test.config.Credential.RootPath)
	}

	flags = append(flags, test.flags...)

	var env []string
	if mallocNumToFail >= 0 {
		env = os.Environ()
		env = append(env, "MALLOC_NUMBER_TO_FAIL="+strconv.FormatInt(mallocNumToFail, 10))
		if *mallocTestDebug {
			env = append(env, "MALLOC_BREAK_ON_FAIL=1")
		}
		env = append(env, "_MALLOC_CHECK=1")
	}

	shim, err := newShimProcess(dispatcher, shimPath, flags, env)
	if err != nil {
		return err
	}
	statusChan <- statusMsg{test: test, statusType: statusShimStarted, pid: shim.cmd.Process.Pid}
	defer shim.close()

	localErr := doExchanges(test, shim, resumeCount, &transcripts)
	childErr := shim.wait()

	// Now that the shim has exited, all the settings files have been
	// written. Append the saved transcripts.
	for i, transcript := range transcripts {
		if err := appendTranscript(transcriptPrefix+strconv.Itoa(i), transcript); err != nil {
			return err
		}
	}

	var isValgrindError, mustFail bool
	if exitError, ok := childErr.(*exec.ExitError); ok {
		switch exitError.Sys().(syscall.WaitStatus).ExitStatus() {
		case 88:
			return errMoreMallocs
		case 89:
			return errUnimplemented
		case 90:
			mustFail = true
		case 99:
			isValgrindError = true
		}
	}

	// Account for Windows line endings.
	stdout := strings.Replace(shim.stdout.String(), "\r\n", "\n", -1)
	stderr := strings.Replace(shim.stderr.String(), "\r\n", "\n", -1)

	// Work around an NDK / Android bug. The NDK r16 sometimes generates
	// binaries with the DF_1_PIE, which the runtime linker on Android N
	// complains about. The next NDK revision should work around this but,
	// in the meantime, strip its error out.
	//
	// https://github.com/android-ndk/ndk/issues/602
	// https://android-review.googlesource.com/c/platform/bionic/+/259790
	// https://android-review.googlesource.com/c/toolchain/binutils/+/571550
	//
	// Remove this after switching to the r17 NDK.
	stderr = removeFirstLineIfSuffix(stderr, ": unsupported flags DT_FLAGS_1=0x8000001")

	// Separate the errors from the shim and those from tools like
	// AddressSanitizer.
	var extraStderr string
	if stderrParts := strings.SplitN(stderr, "--- DONE ---\n", 2); len(stderrParts) == 2 {
		stderr = stderrParts[0]
		extraStderr = stderrParts[1]
	}

	failed := localErr != nil || childErr != nil
	expectedError := translateExpectedError(test.expectedError)
	correctFailure := len(expectedError) == 0 || strings.Contains(stderr, expectedError)

	localErrString := "none"
	if localErr != nil {
		localErrString = localErr.Error()
	}
	if len(test.expectedLocalError) != 0 {
		correctFailure = correctFailure && strings.Contains(localErrString, test.expectedLocalError)
	}

	if failed != test.shouldFail || failed && !correctFailure || mustFail {
		childErrString := "none"
		if childErr != nil {
			childErrString = childErr.Error()
		}

		var msg string
		switch {
		case failed && !test.shouldFail:
			msg = "unexpected failure"
		case !failed && test.shouldFail:
			msg = fmt.Sprintf("unexpected success (wanted failure with %q / %q)", expectedError, test.expectedLocalError)
		case failed && !correctFailure:
			msg = fmt.Sprintf("bad error (wanted %q / %q)", expectedError, test.expectedLocalError)
		case mustFail:
			msg = "test failure"
		default:
			panic("internal error")
		}

		return fmt.Errorf("%s: local error %q, child error %q, stdout:\n%s\nstderr:\n%s\n%s", msg, localErrString, childErrString, stdout, stderr, extraStderr)
	}

	if len(extraStderr) > 0 || (!failed && len(stderr) > 0) {
		return fmt.Errorf("unexpected error output:\n%s\n%s", stderr, extraStderr)
	}

	if *useValgrind && isValgrindError {
		return fmt.Errorf("valgrind error:\n%s\n%s", stderr, extraStderr)
	}

	return nil
}

type tlsVersion struct {
	name string
	// version is the protocol version.
	version uint16
	// excludeFlag is the legacy shim flag to disable the version.
	excludeFlag string
	hasDTLS     bool
	hasQUIC     bool
	// versionDTLS, if non-zero, is the DTLS-specific representation of the version.
	versionDTLS uint16
	// versionWire, if non-zero, is the wire representation of the
	// version. Otherwise the wire version is the protocol version or
	// versionDTLS.
	versionWire uint16
}

func (vers tlsVersion) String() string {
	return vers.name
}

func (vers tlsVersion) shimFlag(protocol protocol) string {
	// The shim uses the protocol version in its public API, but uses the
	// DTLS-specific version if it exists.
	if protocol == dtls && vers.versionDTLS != 0 {
		return strconv.Itoa(int(vers.versionDTLS))
	}
	return strconv.Itoa(int(vers.version))
}

func (vers tlsVersion) wire(protocol protocol) uint16 {
	if protocol == dtls && vers.versionDTLS != 0 {
		return vers.versionDTLS
	}
	if vers.versionWire != 0 {
		return vers.versionWire
	}
	return vers.version
}

func (vers tlsVersion) supportsProtocol(protocol protocol) bool {
	if protocol == dtls {
		return vers.hasDTLS
	}
	if protocol == quic {
		return vers.hasQUIC
	}
	return true
}

var tlsVersions = []tlsVersion{
	{
		name:        "TLS1",
		version:     VersionTLS10,
		excludeFlag: "-no-tls1",
		hasDTLS:     true,
		versionDTLS: VersionDTLS10,
	},
	{
		name:        "TLS11",
		version:     VersionTLS11,
		excludeFlag: "-no-tls11",
	},
	{
		name:        "TLS12",
		version:     VersionTLS12,
		excludeFlag: "-no-tls12",
		hasDTLS:     true,
		versionDTLS: VersionDTLS12,
	},
	{
		name:        "TLS13",
		version:     VersionTLS13,
		excludeFlag: "-no-tls13",
		hasQUIC:     true,
		hasDTLS:     true,
		versionDTLS: VersionDTLS125Experimental,
		versionWire: VersionTLS13,
	},
}

func allVersions(protocol protocol) []tlsVersion {
	if protocol == tls {
		return tlsVersions
	}

	var ret []tlsVersion
	for _, vers := range tlsVersions {
		if vers.supportsProtocol(protocol) {
			ret = append(ret, vers)
		}
	}
	return ret
}

type testCipherSuite struct {
	name string
	id   uint16
}

var testCipherSuites = []testCipherSuite{
	{"RSA_WITH_3DES_EDE_CBC_SHA", TLS_RSA_WITH_3DES_EDE_CBC_SHA},
	{"RSA_WITH_AES_128_GCM_SHA256", TLS_RSA_WITH_AES_128_GCM_SHA256},
	{"RSA_WITH_AES_128_CBC_SHA", TLS_RSA_WITH_AES_128_CBC_SHA},
	{"RSA_WITH_AES_256_GCM_SHA384", TLS_RSA_WITH_AES_256_GCM_SHA384},
	{"RSA_WITH_AES_256_CBC_SHA", TLS_RSA_WITH_AES_256_CBC_SHA},
	{"ECDHE_ECDSA_WITH_AES_128_GCM_SHA256", TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256},
	{"ECDHE_ECDSA_WITH_AES_128_CBC_SHA", TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA},
	{"ECDHE_ECDSA_WITH_AES_256_GCM_SHA384", TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384},
	{"ECDHE_ECDSA_WITH_AES_256_CBC_SHA", TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA},
	{"ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256", TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256},
	{"ECDHE_RSA_WITH_AES_128_GCM_SHA256", TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
	{"ECDHE_RSA_WITH_AES_128_CBC_SHA", TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA},
	{"ECDHE_RSA_WITH_AES_128_CBC_SHA256", TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256},
	{"ECDHE_RSA_WITH_AES_256_GCM_SHA384", TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384},
	{"ECDHE_RSA_WITH_AES_256_CBC_SHA", TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA},
	{"ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256", TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256},
	{"PSK_WITH_AES_128_CBC_SHA", TLS_PSK_WITH_AES_128_CBC_SHA},
	{"PSK_WITH_AES_256_CBC_SHA", TLS_PSK_WITH_AES_256_CBC_SHA},
	{"ECDHE_PSK_WITH_AES_128_CBC_SHA", TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA},
	{"ECDHE_PSK_WITH_AES_256_CBC_SHA", TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA},
	{"ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256", TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256},
	{"CHACHA20_POLY1305_SHA256", TLS_CHACHA20_POLY1305_SHA256},
	{"AES_128_GCM_SHA256", TLS_AES_128_GCM_SHA256},
	{"AES_256_GCM_SHA384", TLS_AES_256_GCM_SHA384},
}

func hasComponent(suiteName, component string) bool {
	return strings.Contains("_"+suiteName+"_", "_"+component+"_")
}

func isTLS12Only(suiteName string) bool {
	return hasComponent(suiteName, "GCM") ||
		hasComponent(suiteName, "SHA256") ||
		hasComponent(suiteName, "SHA384") ||
		hasComponent(suiteName, "POLY1305")
}

func isTLS13Suite(suiteName string) bool {
	return !hasComponent(suiteName, "WITH")
}

func bigFromHex(hex string) *big.Int {
	ret, ok := new(big.Int).SetString(hex, 16)
	if !ok {
		panic("failed to parse hex number 0x" + hex)
	}
	return ret
}

func convertToSplitHandshakeTests(tests []testCase) (splitHandshakeTests []testCase, err error) {
	var stdout bytes.Buffer
	var flags []string
	if len(*shimExtraFlags) > 0 {
		flags = strings.Split(*shimExtraFlags, ";")
	}
	flags = append(flags, "-is-handshaker-supported")
	shim := exec.Command(*shimPath, flags...)
	shim.Stdout = &stdout
	if err := shim.Run(); err != nil {
		return nil, err
	}

	switch strings.TrimSpace(string(stdout.Bytes())) {
	case "No":
		return
	case "Yes":
		break
	default:
		return nil, fmt.Errorf("unknown output from shim: %q", stdout.Bytes())
	}

	var allowHintMismatchPattern []string
	if len(*allowHintMismatch) > 0 {
		allowHintMismatchPattern = strings.Split(*allowHintMismatch, ";")
	}

NextTest:
	for _, test := range tests {
		if test.protocol != tls ||
			test.testType != serverTest ||
			len(test.shimCredentials) != 0 ||
			strings.Contains(test.name, "ECH-Server") ||
			test.skipSplitHandshake {
			continue
		}

		for _, flag := range test.flags {
			if flag == "-implicit-handshake" {
				continue NextTest
			}
		}

		shTest := test
		shTest.name += "-Split"
		shTest.flags = make([]string, len(test.flags), len(test.flags)+3)
		copy(shTest.flags, test.flags)
		shTest.flags = append(shTest.flags, "-handoff", "-handshaker-path", *handshakerPath)

		splitHandshakeTests = append(splitHandshakeTests, shTest)
	}

	for _, test := range tests {
		if test.protocol == dtls ||
			test.testType != serverTest ||
			test.skipHints {
			continue
		}

		var matched bool
		if len(allowHintMismatchPattern) > 0 {
			matched, err = match(allowHintMismatchPattern, nil, test.name)
			if err != nil {
				return nil, fmt.Errorf("error matching pattern: %s", err)
			}
		}

		shTest := test
		shTest.name += "-Hints"
		shTest.flags = make([]string, len(test.flags), len(test.flags)+3)
		copy(shTest.flags, test.flags)
		shTest.flags = append(shTest.flags, "-handshake-hints", "-handshaker-path", *handshakerPath)
		if matched {
			shTest.flags = append(shTest.flags, "-allow-hint-mismatch")
		}

		splitHandshakeTests = append(splitHandshakeTests, shTest)
	}

	return splitHandshakeTests, nil
}

func addBasicTests() {
	basicTests := []testCase{
		{
			name: "NoFallbackSCSV",
			config: Config{
				Bugs: ProtocolBugs{
					FailIfNotFallbackSCSV: true,
				},
			},
			shouldFail:         true,
			expectedLocalError: "no fallback SCSV found",
		},
		{
			name: "SendFallbackSCSV",
			config: Config{
				Bugs: ProtocolBugs{
					FailIfNotFallbackSCSV: true,
				},
			},
			flags: []string{"-fallback-scsv"},
		},
		{
			name: "ClientCertificateTypes",
			config: Config{
				MaxVersion: VersionTLS12,
				ClientAuth: RequestClientCert,
				ClientCertificateTypes: []byte{
					CertTypeDSSSign,
					CertTypeRSASign,
					CertTypeECDSASign,
				},
			},
			flags: []string{
				"-expect-certificate-types",
				base64FlagValue([]byte{
					CertTypeDSSSign,
					CertTypeRSASign,
					CertTypeECDSASign,
				}),
			},
		},
		{
			name: "CheckClientCertificateTypes",
			config: Config{
				MaxVersion:             VersionTLS12,
				ClientAuth:             RequestClientCert,
				ClientCertificateTypes: []byte{CertTypeECDSASign},
			},
			shimCertificate: &rsaCertificate,
			shouldFail:      true,
			expectedError:   ":UNKNOWN_CERTIFICATE_TYPE:",
		},
		{
			name: "NoCheckClientCertificateTypes",
			config: Config{
				MaxVersion:             VersionTLS12,
				ClientAuth:             RequestClientCert,
				ClientCertificateTypes: []byte{CertTypeECDSASign},
			},
			shimCertificate: &rsaCertificate,
			flags:           []string{"-no-check-client-certificate-type"},
		},
		{
			name: "UnauthenticatedECDH",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				Bugs: ProtocolBugs{
					UnauthenticatedECDH: true,
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_MESSAGE:",
		},
		{
			name: "SkipCertificateStatus",
			config: Config{
				MaxVersion:   VersionTLS12,
				Credential:   rsaCertificate.WithOCSP(testOCSPResponse),
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				Bugs: ProtocolBugs{
					SkipCertificateStatus: true,
				},
			},
			flags: []string{
				"-enable-ocsp-stapling",
				// This test involves an optional message. Test the message callback
				// trace to ensure we do not miss or double-report any.
				"-expect-msg-callback",
				`write hs 1
read hs 2
read hs 11
read hs 12
read hs 14
write hs 16
write ccs
write hs 20
read hs 4
read ccs
read hs 20
read alert 1 0
`,
			},
		},
		{
			protocol: dtls,
			name:     "SkipCertificateStatus-DTLS",
			config: Config{
				MaxVersion:   VersionTLS12,
				Credential:   rsaCertificate.WithOCSP(testOCSPResponse),
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				Bugs: ProtocolBugs{
					SkipCertificateStatus: true,
				},
			},
			flags: []string{
				"-enable-ocsp-stapling",
				// This test involves an optional message. Test the message callback
				// trace to ensure we do not miss or double-report any.
				"-expect-msg-callback",
				`write hs 1
read hs 3
write hs 1
read hs 2
read hs 11
read hs 12
read hs 14
write hs 16
write ccs
write hs 20
read hs 4
read ccs
read hs 20
read alert 1 0
`,
			},
		},
		{
			name: "SkipServerKeyExchange",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				Bugs: ProtocolBugs{
					SkipServerKeyExchange: true,
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_MESSAGE:",
		},
		{
			testType: serverTest,
			name:     "ServerSkipCertificateVerify",
			config: Config{
				MaxVersion: VersionTLS12,
				Credential: &rsaCertificate,
				Bugs: ProtocolBugs{
					SkipCertificateVerify: true,
				},
			},
			expectations: connectionExpectations{
				peerCertificate: &rsaCertificate,
			},
			flags: []string{
				"-require-any-client-certificate",
			},
			shouldFail:         true,
			expectedError:      ":UNEXPECTED_RECORD:",
			expectedLocalError: "remote error: unexpected message",
		},
		{
			testType: serverTest,
			name:     "Alert",
			config: Config{
				Bugs: ProtocolBugs{
					SendSpuriousAlert: alertRecordOverflow,
				},
			},
			shouldFail:    true,
			expectedError: ":TLSV1_ALERT_RECORD_OVERFLOW:",
		},
		{
			protocol: dtls,
			testType: serverTest,
			name:     "Alert-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SendSpuriousAlert: alertRecordOverflow,
				},
			},
			shouldFail:    true,
			expectedError: ":TLSV1_ALERT_RECORD_OVERFLOW:",
		},
		{
			testType: serverTest,
			name:     "FragmentAlert",
			config: Config{
				Bugs: ProtocolBugs{
					FragmentAlert:     true,
					SendSpuriousAlert: alertRecordOverflow,
				},
			},
			shouldFail:    true,
			expectedError: ":BAD_ALERT:",
		},
		{
			protocol: dtls,
			testType: serverTest,
			name:     "FragmentAlert-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					FragmentAlert:     true,
					SendSpuriousAlert: alertRecordOverflow,
				},
			},
			shouldFail:    true,
			expectedError: ":BAD_ALERT:",
		},
		{
			testType: serverTest,
			name:     "DoubleAlert",
			config: Config{
				Bugs: ProtocolBugs{
					DoubleAlert:       true,
					SendSpuriousAlert: alertRecordOverflow,
				},
			},
			shouldFail:    true,
			expectedError: ":BAD_ALERT:",
		},
		{
			protocol: dtls,
			testType: serverTest,
			name:     "DoubleAlert-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					DoubleAlert:       true,
					SendSpuriousAlert: alertRecordOverflow,
				},
			},
			shouldFail:    true,
			expectedError: ":BAD_ALERT:",
		},
		{
			name: "SkipNewSessionTicket",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					SkipNewSessionTicket: true,
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			testType: serverTest,
			name:     "FallbackSCSV",
			config: Config{
				MaxVersion: VersionTLS11,
				Bugs: ProtocolBugs{
					SendFallbackSCSV: true,
				},
			},
			shouldFail:         true,
			expectedError:      ":INAPPROPRIATE_FALLBACK:",
			expectedLocalError: "remote error: inappropriate fallback",
		},
		{
			testType: serverTest,
			name:     "FallbackSCSV-VersionMatch-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					SendFallbackSCSV: true,
				},
			},
		},
		{
			testType: serverTest,
			name:     "FallbackSCSV-VersionMatch-TLS12",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					SendFallbackSCSV: true,
				},
			},
			flags: []string{"-max-version", strconv.Itoa(VersionTLS12)},
		},
		{
			testType: serverTest,
			name:     "FragmentedClientVersion",
			config: Config{
				Bugs: ProtocolBugs{
					MaxHandshakeRecordLength: 1,
					FragmentClientVersion:    true,
				},
			},
			expectations: connectionExpectations{
				version: VersionTLS13,
			},
		},
		{
			testType:      serverTest,
			name:          "HttpGET",
			sendPrefix:    "GET / HTTP/1.0\n",
			shouldFail:    true,
			expectedError: ":HTTP_REQUEST:",
		},
		{
			testType:      serverTest,
			name:          "HttpPOST",
			sendPrefix:    "POST / HTTP/1.0\n",
			shouldFail:    true,
			expectedError: ":HTTP_REQUEST:",
		},
		{
			testType:      serverTest,
			name:          "HttpHEAD",
			sendPrefix:    "HEAD / HTTP/1.0\n",
			shouldFail:    true,
			expectedError: ":HTTP_REQUEST:",
		},
		{
			testType:      serverTest,
			name:          "HttpPUT",
			sendPrefix:    "PUT / HTTP/1.0\n",
			shouldFail:    true,
			expectedError: ":HTTP_REQUEST:",
		},
		{
			testType:      serverTest,
			name:          "HttpCONNECT",
			sendPrefix:    "CONNECT www.google.com:443 HTTP/1.0\n",
			shouldFail:    true,
			expectedError: ":HTTPS_PROXY_REQUEST:",
		},
		{
			testType:      serverTest,
			name:          "Garbage",
			sendPrefix:    "blah",
			shouldFail:    true,
			expectedError: ":WRONG_VERSION_NUMBER:",
		},
		{
			name: "RSAEphemeralKey",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_RSA_WITH_AES_128_CBC_SHA},
				Bugs: ProtocolBugs{
					RSAEphemeralKey: true,
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_MESSAGE:",
		},
		{
			name:          "DisableEverything",
			flags:         []string{"-no-tls13", "-no-tls12", "-no-tls11", "-no-tls1"},
			shouldFail:    true,
			expectedError: ":NO_SUPPORTED_VERSIONS_ENABLED:",
		},
		{
			protocol:      dtls,
			name:          "DisableEverything-DTLS",
			flags:         []string{"-no-tls13", "-no-tls12", "-no-tls1"},
			shouldFail:    true,
			expectedError: ":NO_SUPPORTED_VERSIONS_ENABLED:",
		},
		{
			protocol: dtls,
			testType: serverTest,
			name:     "MTU",
			config: Config{
				Bugs: ProtocolBugs{
					MaxPacketLength: 256,
				},
			},
			flags: []string{"-mtu", "256"},
		},
		{
			protocol: dtls,
			testType: serverTest,
			name:     "MTUExceeded",
			config: Config{
				Bugs: ProtocolBugs{
					MaxPacketLength: 255,
				},
			},
			flags:              []string{"-mtu", "256"},
			shouldFail:         true,
			expectedLocalError: "dtls: exceeded maximum packet length",
		},
		{
			name: "EmptyCertificateList",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					EmptyCertificateList: true,
				},
			},
			shouldFail:    true,
			expectedError: ":DECODE_ERROR:",
		},
		{
			name: "EmptyCertificateList-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					EmptyCertificateList: true,
				},
			},
			shouldFail:    true,
			expectedError: ":PEER_DID_NOT_RETURN_A_CERTIFICATE:",
		},
		{
			name:             "TLSFatalBadPackets",
			damageFirstWrite: true,
			shouldFail:       true,
			expectedError:    ":DECRYPTION_FAILED_OR_BAD_RECORD_MAC:",
		},
		{
			protocol:         dtls,
			name:             "DTLSIgnoreBadPackets",
			damageFirstWrite: true,
		},
		{
			protocol:         dtls,
			name:             "DTLSIgnoreBadPackets-Async",
			damageFirstWrite: true,
			flags:            []string{"-async"},
		},
		{
			name: "AppDataBeforeHandshake",
			config: Config{
				Bugs: ProtocolBugs{
					AppDataBeforeHandshake: []byte("TEST MESSAGE"),
				},
			},
			shouldFail:    true,
			expectedError: ":APPLICATION_DATA_INSTEAD_OF_HANDSHAKE:",
		},
		{
			name: "AppDataBeforeHandshake-Empty",
			config: Config{
				Bugs: ProtocolBugs{
					AppDataBeforeHandshake: []byte{},
				},
			},
			shouldFail:    true,
			expectedError: ":APPLICATION_DATA_INSTEAD_OF_HANDSHAKE:",
		},
		{
			protocol: dtls,
			name:     "AppDataBeforeHandshake-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					AppDataBeforeHandshake: []byte("TEST MESSAGE"),
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			protocol: dtls,
			name:     "AppDataBeforeHandshake-DTLS-Empty",
			config: Config{
				Bugs: ProtocolBugs{
					AppDataBeforeHandshake: []byte{},
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			name: "AppDataAfterChangeCipherSpec",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					AppDataAfterChangeCipherSpec: []byte("TEST MESSAGE"),
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			name: "AppDataAfterChangeCipherSpec-Empty",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					AppDataAfterChangeCipherSpec: []byte{},
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			protocol: dtls,
			name:     "AppDataAfterChangeCipherSpec-DTLS",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					AppDataAfterChangeCipherSpec: []byte("TEST MESSAGE"),
				},
			},
			// BoringSSL's DTLS implementation will drop the out-of-order
			// application data.
		},
		{
			protocol: dtls,
			name:     "AppDataAfterChangeCipherSpec-DTLS-Empty",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					AppDataAfterChangeCipherSpec: []byte{},
				},
			},
			// BoringSSL's DTLS implementation will drop the out-of-order
			// application data.
		},
		{
			name: "AlertAfterChangeCipherSpec",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					AlertAfterChangeCipherSpec: alertRecordOverflow,
				},
			},
			shouldFail:    true,
			expectedError: ":TLSV1_ALERT_RECORD_OVERFLOW:",
		},
		{
			protocol: dtls,
			name:     "AlertAfterChangeCipherSpec-DTLS",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					AlertAfterChangeCipherSpec: alertRecordOverflow,
				},
			},
			shouldFail:    true,
			expectedError: ":TLSV1_ALERT_RECORD_OVERFLOW:",
		},
		{
			protocol: dtls,
			name:     "ReorderHandshakeFragments-Small-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					ReorderHandshakeFragments: true,
					// Small enough that every handshake message is
					// fragmented.
					MaxHandshakeRecordLength: 2,
				},
			},
		},
		{
			protocol: dtls,
			name:     "ReorderHandshakeFragments-Large-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					ReorderHandshakeFragments: true,
					// Large enough that no handshake message is
					// fragmented.
					MaxHandshakeRecordLength: 2048,
				},
			},
		},
		{
			protocol: dtls,
			name:     "MixCompleteMessageWithFragments-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					ReorderHandshakeFragments:       true,
					MixCompleteMessageWithFragments: true,
					MaxHandshakeRecordLength:        2,
				},
			},
		},
		{
			name: "SendInvalidRecordType",
			config: Config{
				Bugs: ProtocolBugs{
					SendInvalidRecordType: true,
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			protocol: dtls,
			name:     "SendInvalidRecordType-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SendInvalidRecordType: true,
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			name: "FalseStart-SkipServerSecondLeg",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				NextProtos:   []string{"foo"},
				Bugs: ProtocolBugs{
					SkipNewSessionTicket: true,
					SkipChangeCipherSpec: true,
					SkipFinished:         true,
					ExpectFalseStart:     true,
				},
			},
			flags: []string{
				"-false-start",
				"-handshake-never-done",
				"-advertise-alpn", "\x03foo",
				"-expect-alpn", "foo",
			},
			shimWritesFirst: true,
			shouldFail:      true,
			expectedError:   ":APPLICATION_DATA_INSTEAD_OF_HANDSHAKE:",
		},
		{
			name: "FalseStart-SkipServerSecondLeg-Implicit",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				NextProtos:   []string{"foo"},
				Bugs: ProtocolBugs{
					SkipNewSessionTicket: true,
					SkipChangeCipherSpec: true,
					SkipFinished:         true,
				},
			},
			flags: []string{
				"-implicit-handshake",
				"-false-start",
				"-handshake-never-done",
				"-advertise-alpn", "\x03foo",
			},
			shouldFail:    true,
			expectedError: ":APPLICATION_DATA_INSTEAD_OF_HANDSHAKE:",
		},
		{
			testType:           serverTest,
			name:               "FailEarlyCallback",
			flags:              []string{"-fail-early-callback"},
			shouldFail:         true,
			expectedError:      ":CONNECTION_REJECTED:",
			expectedLocalError: "remote error: handshake failure",
		},
		{
			name: "FailCertCallback-Client-TLS12",
			config: Config{
				MaxVersion: VersionTLS12,
				ClientAuth: RequestClientCert,
			},
			flags:              []string{"-fail-cert-callback"},
			shouldFail:         true,
			expectedError:      ":CERT_CB_ERROR:",
			expectedLocalError: "remote error: internal error",
		},
		{
			testType: serverTest,
			name:     "FailCertCallback-Server-TLS12",
			config: Config{
				MaxVersion: VersionTLS12,
			},
			flags:              []string{"-fail-cert-callback"},
			shouldFail:         true,
			expectedError:      ":CERT_CB_ERROR:",
			expectedLocalError: "remote error: internal error",
		},
		{
			name: "FailCertCallback-Client-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				ClientAuth: RequestClientCert,
			},
			flags:              []string{"-fail-cert-callback"},
			shouldFail:         true,
			expectedError:      ":CERT_CB_ERROR:",
			expectedLocalError: "remote error: internal error",
		},
		{
			testType: serverTest,
			name:     "FailCertCallback-Server-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			flags:              []string{"-fail-cert-callback"},
			shouldFail:         true,
			expectedError:      ":CERT_CB_ERROR:",
			expectedLocalError: "remote error: internal error",
		},
		{
			protocol: dtls,
			name:     "FragmentMessageTypeMismatch-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					MaxHandshakeRecordLength:    2,
					FragmentMessageTypeMismatch: true,
				},
			},
			shouldFail:    true,
			expectedError: ":FRAGMENT_MISMATCH:",
		},
		{
			protocol: dtls,
			name:     "FragmentMessageLengthMismatch-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					MaxHandshakeRecordLength:      2,
					FragmentMessageLengthMismatch: true,
				},
			},
			shouldFail:    true,
			expectedError: ":FRAGMENT_MISMATCH:",
		},
		{
			protocol: dtls,
			name:     "SplitFragments-Header-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SplitFragments: 2,
				},
			},
			shouldFail:    true,
			expectedError: ":BAD_HANDSHAKE_RECORD:",
		},
		{
			protocol: dtls,
			name:     "SplitFragments-Boundary-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SplitFragments: dtlsMaxRecordHeaderLen,
				},
			},
			shouldFail:    true,
			expectedError: ":BAD_HANDSHAKE_RECORD:",
		},
		{
			protocol: dtls,
			name:     "SplitFragments-Body-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SplitFragments: dtlsMaxRecordHeaderLen + 1,
				},
			},
			shouldFail:    true,
			expectedError: ":BAD_HANDSHAKE_RECORD:",
		},
		{
			protocol: dtls,
			name:     "SendEmptyFragments-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SendEmptyFragments: true,
				},
			},
		},
		{
			testType: serverTest,
			protocol: dtls,
			name:     "SendEmptyFragments-Padded-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					// Test empty fragments for a message with a
					// nice power-of-two length.
					PadClientHello:     64,
					SendEmptyFragments: true,
				},
			},
		},
		{
			name: "BadFinished-Client",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					BadFinished: true,
				},
			},
			shouldFail:    true,
			expectedError: ":DIGEST_CHECK_FAILED:",
		},
		{
			name: "BadFinished-Client-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					BadFinished: true,
				},
			},
			shouldFail:    true,
			expectedError: ":DIGEST_CHECK_FAILED:",
		},
		{
			testType: serverTest,
			name:     "BadFinished-Server",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					BadFinished: true,
				},
			},
			shouldFail:    true,
			expectedError: ":DIGEST_CHECK_FAILED:",
		},
		{
			testType: serverTest,
			name:     "BadFinished-Server-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					BadFinished: true,
				},
			},
			shouldFail:    true,
			expectedError: ":DIGEST_CHECK_FAILED:",
		},
		{
			name: "FalseStart-BadFinished",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				NextProtos:   []string{"foo"},
				Bugs: ProtocolBugs{
					BadFinished:      true,
					ExpectFalseStart: true,
				},
			},
			flags: []string{
				"-false-start",
				"-handshake-never-done",
				"-advertise-alpn", "\x03foo",
				"-expect-alpn", "foo",
			},
			shimWritesFirst: true,
			shouldFail:      true,
			expectedError:   ":DIGEST_CHECK_FAILED:",
		},
		{
			name: "NoFalseStart-NoALPN",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				Bugs: ProtocolBugs{
					ExpectFalseStart:          true,
					AlertBeforeFalseStartTest: alertAccessDenied,
				},
			},
			flags: []string{
				"-false-start",
			},
			shimWritesFirst:    true,
			shouldFail:         true,
			expectedError:      ":TLSV1_ALERT_ACCESS_DENIED:",
			expectedLocalError: "tls: peer did not false start: EOF",
		},
		{
			name: "FalseStart-NoALPNAllowed",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
				Bugs: ProtocolBugs{
					ExpectFalseStart: true,
				},
			},
			flags: []string{
				"-false-start",
				"-allow-false-start-without-alpn",
			},
			shimWritesFirst: true,
		},
		{
			name: "NoFalseStart-NoAEAD",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA},
				NextProtos:   []string{"foo"},
				Bugs: ProtocolBugs{
					ExpectFalseStart:          true,
					AlertBeforeFalseStartTest: alertAccessDenied,
				},
			},
			flags: []string{
				"-false-start",
				"-advertise-alpn", "\x03foo",
			},
			shimWritesFirst:    true,
			shouldFail:         true,
			expectedError:      ":TLSV1_ALERT_ACCESS_DENIED:",
			expectedLocalError: "tls: peer did not false start: EOF",
		},
		{
			name: "NoFalseStart-RSA",
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_RSA_WITH_AES_128_GCM_SHA256},
				NextProtos:   []string{"foo"},
				Bugs: ProtocolBugs{
					ExpectFalseStart:          true,
					AlertBeforeFalseStartTest: alertAccessDenied,
				},
			},
			flags: []string{
				"-false-start",
				"-advertise-alpn", "\x03foo",
			},
			shimWritesFirst:    true,
			shouldFail:         true,
			expectedError:      ":TLSV1_ALERT_ACCESS_DENIED:",
			expectedLocalError: "tls: peer did not false start: EOF",
		},
		{
			protocol: dtls,
			name:     "SendSplitAlert-Sync",
			config: Config{
				Bugs: ProtocolBugs{
					SendSplitAlert: true,
				},
			},
		},
		{
			protocol: dtls,
			name:     "SendSplitAlert-Async",
			config: Config{
				Bugs: ProtocolBugs{
					SendSplitAlert: true,
				},
			},
			flags: []string{"-async"},
		},
		{
			name:             "SendEmptyRecords-Pass",
			sendEmptyRecords: 32,
		},
		{
			name:             "SendEmptyRecords",
			sendEmptyRecords: 33,
			shouldFail:       true,
			expectedError:    ":TOO_MANY_EMPTY_FRAGMENTS:",
		},
		{
			name:             "SendEmptyRecords-Async",
			sendEmptyRecords: 33,
			flags:            []string{"-async"},
			shouldFail:       true,
			expectedError:    ":TOO_MANY_EMPTY_FRAGMENTS:",
		},
		{
			name: "SendWarningAlerts-Pass",
			config: Config{
				MaxVersion: VersionTLS12,
			},
			sendWarningAlerts: 4,
		},
		{
			protocol: dtls,
			name:     "SendWarningAlerts-DTLS-Pass",
			config: Config{
				MaxVersion: VersionTLS12,
			},
			sendWarningAlerts: 4,
		},
		{
			name: "SendWarningAlerts-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			sendWarningAlerts:  4,
			shouldFail:         true,
			expectedError:      ":BAD_ALERT:",
			expectedLocalError: "remote error: error decoding message",
		},
		// Although TLS 1.3 intended to remove warning alerts, it left in
		// user_canceled. JDK11 misuses this alert as a post-handshake
		// full-duplex signal. As a workaround, skip user_canceled as in
		// TLS 1.2, which is consistent with NSS and OpenSSL.
		{
			name: "SendUserCanceledAlerts-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			sendUserCanceledAlerts: 4,
		},
		{
			name: "SendUserCanceledAlerts-TooMany-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			sendUserCanceledAlerts: 5,
			shouldFail:             true,
			expectedError:          ":TOO_MANY_WARNING_ALERTS:",
		},
		{
			name: "SendWarningAlerts-TooMany",
			config: Config{
				MaxVersion: VersionTLS12,
			},
			sendWarningAlerts: 5,
			shouldFail:        true,
			expectedError:     ":TOO_MANY_WARNING_ALERTS:",
		},
		{
			name: "SendWarningAlerts-TooMany-Async",
			config: Config{
				MaxVersion: VersionTLS12,
			},
			sendWarningAlerts: 5,
			flags:             []string{"-async"},
			shouldFail:        true,
			expectedError:     ":TOO_MANY_WARNING_ALERTS:",
		},
		{
			name:               "SendBogusAlertType",
			sendBogusAlertType: true,
			shouldFail:         true,
			expectedError:      ":UNKNOWN_ALERT_TYPE:",
			expectedLocalError: "remote error: illegal parameter",
		},
		{
			protocol:           dtls,
			name:               "SendBogusAlertType-DTLS",
			sendBogusAlertType: true,
			shouldFail:         true,
			expectedError:      ":UNKNOWN_ALERT_TYPE:",
			expectedLocalError: "remote error: illegal parameter",
		},
		{
			name: "TooManyKeyUpdates",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			sendKeyUpdates:   33,
			keyUpdateRequest: keyUpdateNotRequested,
			shouldFail:       true,
			expectedError:    ":TOO_MANY_KEY_UPDATES:",
		},
		{
			name: "EmptySessionID",
			config: Config{
				MaxVersion:             VersionTLS12,
				SessionTicketsDisabled: true,
			},
			noSessionCache: true,
			flags:          []string{"-expect-no-session"},
		},
		{
			name: "Unclean-Shutdown",
			config: Config{
				Bugs: ProtocolBugs{
					NoCloseNotify:     true,
					ExpectCloseNotify: true,
				},
			},
			shimShutsDown: true,
			flags:         []string{"-check-close-notify"},
			shouldFail:    true,
			expectedError: "Unexpected SSL_shutdown result: -1 != 1",
		},
		{
			name: "Unclean-Shutdown-Ignored",
			config: Config{
				Bugs: ProtocolBugs{
					NoCloseNotify: true,
				},
			},
			shimShutsDown: true,
		},
		{
			name: "Unclean-Shutdown-Alert",
			config: Config{
				Bugs: ProtocolBugs{
					SendAlertOnShutdown: alertDecompressionFailure,
					ExpectCloseNotify:   true,
				},
			},
			shimShutsDown: true,
			flags:         []string{"-check-close-notify"},
			shouldFail:    true,
			expectedError: ":SSLV3_ALERT_DECOMPRESSION_FAILURE:",
		},
		{
			name: "LargePlaintext",
			config: Config{
				Bugs: ProtocolBugs{
					SendLargeRecords: true,
				},
			},
			messageLen:         maxPlaintext + 1,
			shouldFail:         true,
			expectedError:      ":DATA_LENGTH_TOO_LONG:",
			expectedLocalError: "remote error: record overflow",
		},
		{
			protocol: dtls,
			name:     "LargePlaintext-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SendLargeRecords: true,
				},
			},
			messageLen:         maxPlaintext + 1,
			shouldFail:         true,
			expectedError:      ":DATA_LENGTH_TOO_LONG:",
			expectedLocalError: "remote error: record overflow",
		},
		{
			name: "LargePlaintext-TLS13-Padded-8192-8192",
			config: Config{
				MinVersion: VersionTLS13,
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					RecordPadding:    8192,
					SendLargeRecords: true,
				},
			},
			messageLen: 8192,
		},
		{
			name: "LargePlaintext-TLS13-Padded-8193-8192",
			config: Config{
				MinVersion: VersionTLS13,
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					RecordPadding:    8193,
					SendLargeRecords: true,
				},
			},
			messageLen:         8192,
			shouldFail:         true,
			expectedError:      ":DATA_LENGTH_TOO_LONG:",
			expectedLocalError: "remote error: record overflow",
		},
		{
			name: "LargePlaintext-TLS13-Padded-16383-1",
			config: Config{
				MinVersion: VersionTLS13,
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					RecordPadding:    1,
					SendLargeRecords: true,
				},
			},
			messageLen: 16383,
		},
		{
			name: "LargePlaintext-TLS13-Padded-16384-1",
			config: Config{
				MinVersion: VersionTLS13,
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					RecordPadding:    1,
					SendLargeRecords: true,
				},
			},
			messageLen:         16384,
			shouldFail:         true,
			expectedError:      ":DATA_LENGTH_TOO_LONG:",
			expectedLocalError: "remote error: record overflow",
		},
		{
			name: "LargeCiphertext",
			config: Config{
				Bugs: ProtocolBugs{
					SendLargeRecords: true,
				},
			},
			messageLen:    maxPlaintext * 2,
			shouldFail:    true,
			expectedError: ":ENCRYPTED_LENGTH_TOO_LONG:",
		},
		{
			protocol: dtls,
			name:     "LargeCiphertext-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SendLargeRecords: true,
				},
			},
			messageLen: maxPlaintext * 2,
			// Unlike the other four cases, DTLS drops records which
			// are invalid before authentication, so the connection
			// does not fail.
			expectMessageDropped: true,
		},
		{
			name:        "BadHelloRequest-1",
			renegotiate: 1,
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					BadHelloRequest: []byte{typeHelloRequest, 0, 0, 1, 1},
				},
			},
			flags: []string{
				"-renegotiate-freely",
				"-expect-total-renegotiations", "1",
			},
			shouldFail:    true,
			expectedError: ":BAD_HELLO_REQUEST:",
		},
		{
			name:        "BadHelloRequest-2",
			renegotiate: 1,
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					BadHelloRequest: []byte{typeServerKeyExchange, 0, 0, 0},
				},
			},
			flags: []string{
				"-renegotiate-freely",
				"-expect-total-renegotiations", "1",
			},
			shouldFail:    true,
			expectedError: ":BAD_HELLO_REQUEST:",
		},
		{
			testType: serverTest,
			name:     "SupportTicketsWithSessionID",
			config: Config{
				MaxVersion:             VersionTLS12,
				SessionTicketsDisabled: true,
			},
			resumeConfig: &Config{
				MaxVersion: VersionTLS12,
			},
			resumeSession: true,
		},
		// TODO(crbug.com/boringssl/715): This test and the next shouldn't be
		// restricted to a max version of TLS 1.2, but they're broken in DTLS 1.3.
		{
			protocol: dtls,
			name:     "DTLS-SendExtraFinished",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					SendExtraFinished: true,
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			protocol: dtls,
			name:     "DTLS-SendExtraFinished-Reordered",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					MaxHandshakeRecordLength:  2,
					ReorderHandshakeFragments: true,
					SendExtraFinished:         true,
				},
			},
			shouldFail:    true,
			expectedError: ":UNEXPECTED_RECORD:",
		},
		{
			testType: serverTest,
			name:     "V2ClientHello-EmptyRecordPrefix",
			config: Config{
				// Choose a cipher suite that does not involve
				// elliptic curves, so no extensions are
				// involved.
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_RSA_WITH_AES_128_CBC_SHA},
				Bugs: ProtocolBugs{
					SendV2ClientHello: true,
				},
			},
			sendPrefix: string([]byte{
				byte(recordTypeHandshake),
				3, 1, // version
				0, 0, // length
			}),
			// A no-op empty record may not be sent before V2ClientHello.
			shouldFail:    true,
			expectedError: ":WRONG_VERSION_NUMBER:",
		},
		{
			testType: serverTest,
			name:     "V2ClientHello-WarningAlertPrefix",
			config: Config{
				// Choose a cipher suite that does not involve
				// elliptic curves, so no extensions are
				// involved.
				MaxVersion:   VersionTLS12,
				CipherSuites: []uint16{TLS_RSA_WITH_AES_128_CBC_SHA},
				Bugs: ProtocolBugs{
					SendV2ClientHello: true,
				},
			},
			sendPrefix: string([]byte{
				byte(recordTypeAlert),
				3, 1, // version
				0, 2, // length
				alertLevelWarning, byte(alertDecompressionFailure),
			}),
			// A no-op warning alert may not be sent before V2ClientHello.
			shouldFail:    true,
			expectedError: ":WRONG_VERSION_NUMBER:",
		},
		{
			name: "KeyUpdate-ToClient",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			sendKeyUpdates:   1,
			keyUpdateRequest: keyUpdateNotRequested,
		},
		{
			testType: serverTest,
			name:     "KeyUpdate-ToServer",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			sendKeyUpdates:   1,
			keyUpdateRequest: keyUpdateNotRequested,
		},
		{
			name: "KeyUpdate-FromClient",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			expectUnsolicitedKeyUpdate: true,
			flags:                      []string{"-key-update"},
		},
		{
			testType: serverTest,
			name:     "KeyUpdate-FromServer",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			expectUnsolicitedKeyUpdate: true,
			flags:                      []string{"-key-update"},
		},
		{
			name: "KeyUpdate-InvalidRequestMode",
			config: Config{
				MaxVersion: VersionTLS13,
			},
			sendKeyUpdates:   1,
			keyUpdateRequest: 42,
			shouldFail:       true,
			expectedError:    ":DECODE_ERROR:",
		},
		{
			// Test that KeyUpdates are acknowledged properly.
			name: "KeyUpdate-RequestACK",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					RejectUnsolicitedKeyUpdate: true,
				},
			},
			// Test the shim receiving many KeyUpdates in a row.
			sendKeyUpdates:   5,
			messageCount:     5,
			keyUpdateRequest: keyUpdateRequested,
		},
		{
			// Test that KeyUpdates are acknowledged properly if the
			// peer's KeyUpdate is discovered while a write is
			// pending.
			name: "KeyUpdate-RequestACK-UnfinishedWrite",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					RejectUnsolicitedKeyUpdate: true,
				},
			},
			// Test the shim receiving many KeyUpdates in a row.
			sendKeyUpdates:          5,
			messageCount:            5,
			keyUpdateRequest:        keyUpdateRequested,
			readWithUnfinishedWrite: true,
			flags:                   []string{"-async"},
		},
		{
			name: "SendSNIWarningAlert",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					SendSNIWarningAlert: true,
				},
			},
		},
		{
			testType: serverTest,
			name:     "ExtraCompressionMethods-TLS12",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					SendCompressionMethods: []byte{1, 2, 3, compressionNone, 4, 5, 6},
				},
			},
		},
		{
			testType: serverTest,
			name:     "ExtraCompressionMethods-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					SendCompressionMethods: []byte{1, 2, 3, compressionNone, 4, 5, 6},
				},
			},
			shouldFail:         true,
			expectedError:      ":INVALID_COMPRESSION_LIST:",
			expectedLocalError: "remote error: illegal parameter",
		},
		{
			testType: serverTest,
			name:     "NoNullCompression-TLS12",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					SendCompressionMethods: []byte{1, 2, 3, 4, 5, 6},
				},
			},
			shouldFail:         true,
			expectedError:      ":INVALID_COMPRESSION_LIST:",
			expectedLocalError: "remote error: illegal parameter",
		},
		{
			testType: serverTest,
			name:     "NoNullCompression-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					SendCompressionMethods: []byte{1, 2, 3, 4, 5, 6},
				},
			},
			shouldFail:         true,
			expectedError:      ":INVALID_COMPRESSION_LIST:",
			expectedLocalError: "remote error: illegal parameter",
		},
		// Test that the client rejects invalid compression methods
		// from the server.
		{
			testType: clientTest,
			name:     "InvalidCompressionMethod",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					SendCompressionMethod: 1,
				},
			},
			shouldFail:         true,
			expectedError:      ":UNSUPPORTED_COMPRESSION_ALGORITHM:",
			expectedLocalError: "remote error: illegal parameter",
		},
		{
			testType: clientTest,
			name:     "TLS13-InvalidCompressionMethod",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					SendCompressionMethod: 1,
				},
			},
			shouldFail:    true,
			expectedError: ":DECODE_ERROR:",
		},
		{
			testType: clientTest,
			name:     "TLS13-HRR-InvalidCompressionMethod",
			config: Config{
				MaxVersion:       VersionTLS13,
				CurvePreferences: []CurveID{CurveP384},
				Bugs: ProtocolBugs{
					SendCompressionMethod: 1,
				},
			},
			shouldFail:         true,
			expectedError:      ":DECODE_ERROR:",
			expectedLocalError: "remote error: error decoding message",
		},
		{
			name: "GREASE-Client-TLS12",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					ExpectGREASE: true,
				},
			},
			flags: []string{"-enable-grease"},
		},
		{
			name: "GREASE-Client-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					ExpectGREASE: true,
				},
			},
			flags: []string{"-enable-grease"},
		},
		{
			testType: serverTest,
			name:     "GREASE-Server-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					// TLS 1.3 servers are expected to
					// always enable GREASE. TLS 1.3 is new,
					// so there is no existing ecosystem to
					// worry about.
					ExpectGREASE: true,
				},
			},
		},
		{
			// Test the TLS 1.2 server so there is a large
			// unencrypted certificate as well as application data.
			testType: serverTest,
			name:     "MaxSendFragment-TLS12",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					MaxReceivePlaintext: 512,
				},
			},
			messageLen: 1024,
			flags: []string{
				"-max-send-fragment", "512",
				"-read-size", "1024",
			},
		},
		{
			// Test the TLS 1.2 server so there is a large
			// unencrypted certificate as well as application data.
			testType: serverTest,
			name:     "MaxSendFragment-TLS12-TooLarge",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					// Ensure that some of the records are
					// 512.
					MaxReceivePlaintext: 511,
				},
			},
			messageLen: 1024,
			flags: []string{
				"-max-send-fragment", "512",
				"-read-size", "1024",
			},
			shouldFail:         true,
			expectedLocalError: "local error: record overflow",
		},
		{
			// Test the TLS 1.3 server so there is a large encrypted
			// certificate as well as application data.
			testType: serverTest,
			name:     "MaxSendFragment-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					MaxReceivePlaintext:            512,
					ExpectPackedEncryptedHandshake: 512,
				},
			},
			messageLen: 1024,
			flags: []string{
				"-max-send-fragment", "512",
				"-read-size", "1024",
			},
		},
		{
			// Test the TLS 1.3 server so there is a large encrypted
			// certificate as well as application data.
			testType: serverTest,
			name:     "MaxSendFragment-TLS13-TooLarge",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					// Ensure that some of the records are
					// 512.
					MaxReceivePlaintext: 511,
				},
			},
			messageLen: 1024,
			flags: []string{
				"-max-send-fragment", "512",
				"-read-size", "1024",
			},
			shouldFail:         true,
			expectedLocalError: "local error: record overflow",
		},
		{
			// Test that handshake data is tightly packed in TLS 1.3.
			testType: serverTest,
			name:     "PackedEncryptedHandshake-TLS13",
			config: Config{
				MaxVersion: VersionTLS13,
				Bugs: ProtocolBugs{
					ExpectPackedEncryptedHandshake: 16384,
				},
			},
		},
		{
			// Test that DTLS can handle multiple application data
			// records in a single packet.
			protocol: dtls,
			name:     "SplitAndPackAppData-DTLS",
			config: Config{
				Bugs: ProtocolBugs{
					SplitAndPackAppData: true,
				},
			},
		},
		{
			protocol: dtls,
			name:     "SplitAndPackAppData-DTLS-Async",
			config: Config{
				Bugs: ProtocolBugs{
					SplitAndPackAppData: true,
				},
			},
			flags: []string{"-async"},
		},
		{
			// DTLS 1.2 allows up to a 255-byte HelloVerifyRequest cookie, which
			// is the largest encodable value.
			protocol: dtls,
			name:     "DTLS-HelloVerifyRequest-255",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					HelloVerifyRequestCookieLength: 255,
				},
			},
		},
		{
			// DTLS 1.2 allows up to a 0-byte HelloVerifyRequest cookie, which
			// was probably a mistake in the spec but test that it works
			// nonetheless.
			protocol: dtls,
			name:     "DTLS-HelloVerifyRequest-0",
			config: Config{
				MaxVersion: VersionTLS12,
				Bugs: ProtocolBugs{
					EmptyHelloVerifyRequestCookie: true,
				},
			},
		},
	}
	testCases = append(testCases, basicTests...)

	// Test that very large messages can be received.
	cert := rsaCertificate
	for i := 0; i < 50; i++ {
		cert.Certificate = append(cert.Certificate, cert.Certificate[0])
	}
	testCases = append(testCases, testCase{
		name: "LargeMessage",
		config: Config{
			Credential: &cert,
		},
	})
	testCases = append(testCases, testCase{
		protocol: dtls,
		name:     "LargeMessage-DTLS",
		config: Config{
			Credential: &cert,
		},
	})

	// They are rejected if the maximum certificate chain length is capped.
	testCases = append(testCases, testCase{
		name: "LargeMessage-Reject",
		config: Config{
			Credential: &cert,
		},
		flags:         []string{"-max-cert-list", "16384"},
		shouldFail:    true,
		expectedError: ":EXCESSIVE_MESSAGE_SIZE:",
	})
	testCases = append(testCases, testCase{
		protocol: dtls,
		name:     "LargeMessage-Reject-DTLS",
		config: Config{
			Credential: &cert,
		},
		flags:         []string{"-max-cert-list", "16384"},
		shouldFail:    true,
		expectedError: ":EXCESSIVE_MESSAGE_SIZE:",
	})

	// Servers echoing the TLS 1.3 compatibility mode session ID should be
	// rejected.
	testCases = append(testCases, testCase{
		name: "EchoTLS13CompatibilitySessionID",
		config: Config{
			MaxVersion: VersionTLS12,
			Bugs: ProtocolBugs{
				EchoSessionIDInFullHandshake: true,
			},
		},
		shouldFail:         true,
		expectedError:      ":SERVER_ECHOED_INVALID_SESSION_ID:",
		expectedLocalError: "remote error: illegal parameter",
	})

	// Servers should reject QUIC client hellos that have a legacy
	// session ID.
	testCases = append(testCases, testCase{
		name:     "QUICCompatibilityMode",
		testType: serverTest,
		protocol: quic,
		config: Config{
			MinVersion: VersionTLS13,
			Bugs: ProtocolBugs{
				CompatModeWithQUIC: true,
			},
		},
		shouldFail:    true,
		expectedError: ":UNEXPECTED_COMPATIBILITY_MODE:",
	})

	// Clients should reject DTLS 1.3 ServerHellos that echo the legacy
	// session ID.
	testCases = append(testCases, testCase{
		protocol:      dtls,
		name:          "DTLS13CompatibilityMode-EchoSessionID",
		resumeSession: true,
		config: Config{
			MaxVersion: VersionTLS12,
		},
		resumeConfig: &Config{
			MinVersion: VersionTLS13,
			Bugs: ProtocolBugs{
				DTLS13EchoSessionID: true,
			},
		},
		shouldFail:    true,
		expectedError: ":DECODE_ERROR:",
	})

	// DTLS 1.3 should work with record headers that don't set the
	// length bit or that use the short sequence number format.
	testCases = append(testCases, testCase{
		testType: clientTest,
		protocol: dtls,
		name:     "DTLS13RecordHeader-NoLength-Client",
		config: Config{
			MinVersion:                 VersionTLS13,
			DTLSRecordHeaderOmitLength: true,
		},
	})
	testCases = append(testCases, testCase{
		testType: serverTest,
		protocol: dtls,
		name:     "DTLS13RecordHeader-NoLength-Server",
		config: Config{
			MinVersion:                 VersionTLS13,
			DTLSRecordHeaderOmitLength: true,
		},
	})
	testCases = append(testCases, testCase{
		testType: clientTest,
		protocol: dtls,
		name:     "DTLS13RecordHeader-ShortSeqNums-Client",
		config: Config{
			MinVersion:          VersionTLS13,
			DTLSUseShortSeqNums: true,
		},
	})
	testCases = append(testCases, testCase{
		testType: serverTest,
		protocol: dtls,
		name:     "DTLS13RecordHeader-ShortSeqNums-Server",
		config: Config{
			MinVersion:          VersionTLS13,
			DTLSUseShortSeqNums: true,
		},
	})
	testCases = append(testCases, testCase{
		protocol: dtls,
		name:     "DTLS13RecordHeader-OldHeader",
		config: Config{
			MinVersion: VersionTLS13,
			Bugs: ProtocolBugs{
				DTLSUsePlaintextRecordHeader: true,
			},
		},
		expectMessageDropped: true,
	})
	testCases = append(testCases, testCase{
		protocol: dtls,
		name:     "DTLS13RecordHeader-CIDBit",
		config: Config{
			MinVersion: VersionTLS13,
			Bugs: ProtocolBugs{
				DTLS13RecordHeaderSetCIDBit: true,
			},
		},
		expectMessageDropped: true,
	})
}

func addTestForCipherSuite(suite testCipherSuite, ver tlsVersion, protocol protocol) {
	const psk = "12345"
	const pskIdentity = "luggage combo"

	if !ver.supportsProtocol(protocol) {
		return
	}
	prefix := protocol.String() + "-"

	var cert *Credential
	if isTLS13Suite(suite.name) {
		cert = &rsaCertificate
	} else if hasComponent(suite.name, "ECDSA") {
		cert = &ecdsaP256Certificate
	} else if hasComponent(suite.name, "RSA") {
		cert = &rsaCertificate
	}

	var flags []string
	if hasComponent(suite.name, "PSK") {
		flags = append(flags,
			"-psk", psk,
			"-psk-identity", pskIdentity)
	}

	if hasComponent(suite.name, "3DES") {
		// BoringSSL disables 3DES ciphers by default.
		flags = append(flags, "-cipher", "3DES")
	}

	var shouldFail bool
	if isTLS12Only(suite.name) && ver.version < VersionTLS12 {
		shouldFail = true
	}
	if !isTLS13Suite(suite.name) && ver.version >= VersionTLS13 {
		shouldFail = true
	}
	if isTLS13Suite(suite.name) && ver.version < VersionTLS13 {
		shouldFail = true
	}

	var sendCipherSuite uint16
	var expectedServerError, expectedClientError string
	serverCipherSuites := []uint16{suite.id}
	if shouldFail {
		expectedServerError = ":NO_SHARED_CIPHER:"
		if ver.version >= VersionTLS13 && cert == nil {
			// TLS 1.2 PSK ciphers won't configure a server certificate, but we
			// require one in TLS 1.3.
			expectedServerError = ":NO_CERTIFICATE_SET:"
		}
		expectedClientError = ":WRONG_CIPHER_RETURNED:"
		// Configure the server to select ciphers as normal but
		// select an incompatible cipher in ServerHello.
		serverCipherSuites = nil
		sendCipherSuite = suite.id
	}

	// Verify exporters interoperate.
	exportKeyingMaterial := 1024

	if ver.version != VersionTLS13 || !ver.hasDTLS {
		testCases = append(testCases, testCase{
			testType: serverTest,
			protocol: protocol,
			name:     prefix + ver.name + "-" + suite.name + "-server",
			config: Config{
				MinVersion:           ver.version,
				MaxVersion:           ver.version,
				CipherSuites:         []uint16{suite.id},
				Credential:           cert,
				PreSharedKey:         []byte(psk),
				PreSharedKeyIdentity: pskIdentity,
				Bugs: ProtocolBugs{
					AdvertiseAllConfiguredCiphers: true,
				},
			},
			shimCertificate:      cert,
			flags:                flags,
			resumeSession:        true,
			shouldFail:           shouldFail,
			expectedError:        expectedServerError,
			exportKeyingMaterial: exportKeyingMaterial,
		})

		testCases = append(testCases, testCase{
			testType: clientTest,
			protocol: protocol,
			name:     prefix + ver.name + "-" + suite.name + "-client",
			config: Config{
				MinVersion:           ver.version,
				MaxVersion:           ver.version,
				CipherSuites:         serverCipherSuites,
				Credential:           cert,
				PreSharedKey:         []byte(psk),
				PreSharedKeyIdentity: pskIdentity,
				Bugs: ProtocolBugs{
					IgnorePeerCipherPreferences: shouldFail,
					SendCipherSuite:             sendCipherSuite,
				},
			},
			flags:                flags,
			resumeSession:        true,
			shouldFail:           shouldFail,
			expectedError:        expectedClientError,
			exportKeyingMaterial: exportKeyingMaterial,
		})
	}

	if shouldFail {
		return
	}

	// Ensure the maximum record size is accepted.
	testCases = append(testCases, testCase{
		protocol: protocol,
		name:     prefix + ver.name + "-" + suite.name + "-LargeRecord",
		config: Config{
			MinVersion:           ver.version,
			MaxVersion:           ver.version,
			CipherSuites:         []uint16{suite.id},
			Credential:           cert,
			PreSharedKey:         []byte(psk),
			PreSharedKeyIdentity: pskIdentity,
		},
		flags:      flags,
		messageLen: maxPlaintext,
	})

	// Test bad records for all ciphers. Bad records are fatal in TLS
	// and ignored in DTLS.
	shouldFail = protocol == tls
	var expectedError string
	if shouldFail {
		expectedError = ":DECRYPTION_FAILED_OR_BAD_RECORD_MAC:"
	}

	// When QUIC is used, the QUIC stack handles record encryption/decryption.
	// Thus it is not possible for the TLS stack in QUIC mode to receive a
	// bad record (i.e. one that fails to decrypt).
	if protocol != quic {
		testCases = append(testCases, testCase{
			protocol: protocol,
			name:     prefix + ver.name + "-" + suite.name + "-BadRecord",
			config: Config{
				MinVersion:           ver.version,
				MaxVersion:           ver.version,
				CipherSuites:         []uint16{suite.id},
				Credential:           cert,
				PreSharedKey:         []byte(psk),
				PreSharedKeyIdentity: pskIdentity,
			},
			flags:            flags,
			damageFirstWrite: true,
			messageLen:       maxPlaintext,
			shouldFail:       shouldFail,
			expectedError:    expectedError,
		})
	}
}

func addCipherSuiteTests() {
	const bogusCipher = 0xfe00

	for _, suite := range testCipherSuites {
		for _, ver := range tlsVersions {
			for _, protocol := range []protocol{tls, dtls, quic} {
				addTestForCipherSuite(suite, ver, protocol)
			}
		}
	}

	testCases = append(testCases, testCase{
		name: "NoSharedCipher",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{},
		},
		shouldFail:    true,
		expectedError: ":HANDSHAKE_FAILURE_ON_CLIENT_HELLO:",
	})

	testCases = append(testCases, testCase{
		name: "NoSharedCipher-TLS13",
		config: Config{
			MaxVersion:   VersionTLS13,
			CipherSuites: []uint16{},
		},
		shouldFail:    true,
		expectedError: ":HANDSHAKE_FAILURE_ON_CLIENT_HELLO:",
	})

	testCases = append(testCases, testCase{
		name: "UnsupportedCipherSuite",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_RSA_WITH_AES_128_CBC_SHA},
			Bugs: ProtocolBugs{
				IgnorePeerCipherPreferences: true,
			},
		},
		flags:         []string{"-cipher", "DEFAULT:!AES"},
		shouldFail:    true,
		expectedError: ":WRONG_CIPHER_RETURNED:",
	})

	testCases = append(testCases, testCase{
		name: "ServerHelloBogusCipher",
		config: Config{
			MaxVersion: VersionTLS12,
			Bugs: ProtocolBugs{
				SendCipherSuite: bogusCipher,
			},
		},
		shouldFail:    true,
		expectedError: ":WRONG_CIPHER_RETURNED:",
	})
	testCases = append(testCases, testCase{
		name: "ServerHelloBogusCipher-TLS13",
		config: Config{
			MaxVersion: VersionTLS13,
			Bugs: ProtocolBugs{
				SendCipherSuite: bogusCipher,
			},
		},
		shouldFail:    true,
		expectedError: ":WRONG_CIPHER_RETURNED:",
	})

	// The server must be tolerant to bogus ciphers.
	testCases = append(testCases, testCase{
		testType: serverTest,
		name:     "UnknownCipher",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{bogusCipher, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
			Bugs: ProtocolBugs{
				AdvertiseAllConfiguredCiphers: true,
			},
		},
	})

	// The server must be tolerant to bogus ciphers.
	testCases = append(testCases, testCase{
		testType: serverTest,
		name:     "UnknownCipher-TLS13",
		config: Config{
			MaxVersion:   VersionTLS13,
			CipherSuites: []uint16{bogusCipher, TLS_AES_128_GCM_SHA256},
			Bugs: ProtocolBugs{
				AdvertiseAllConfiguredCiphers: true,
			},
		},
	})

	// Test empty ECDHE_PSK identity hints work as expected.
	testCases = append(testCases, testCase{
		name: "EmptyECDHEPSKHint",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA},
			PreSharedKey: []byte("secret"),
		},
		flags: []string{"-psk", "secret"},
	})

	// Test empty PSK identity hints work as expected, even if an explicit
	// ServerKeyExchange is sent.
	testCases = append(testCases, testCase{
		name: "ExplicitEmptyPSKHint",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_PSK_WITH_AES_128_CBC_SHA},
			PreSharedKey: []byte("secret"),
			Bugs: ProtocolBugs{
				AlwaysSendPreSharedKeyIdentityHint: true,
			},
		},
		flags: []string{"-psk", "secret"},
	})

	// Test that clients enforce that the server-sent certificate and cipher
	// suite match in TLS 1.2.
	testCases = append(testCases, testCase{
		name: "CertificateCipherMismatch-RSA",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
			Credential:   &rsaCertificate,
			Bugs: ProtocolBugs{
				SendCipherSuite: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			},
		},
		shouldFail:    true,
		expectedError: ":WRONG_CERTIFICATE_TYPE:",
	})
	testCases = append(testCases, testCase{
		name: "CertificateCipherMismatch-ECDSA",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256},
			Credential:   &ecdsaP256Certificate,
			Bugs: ProtocolBugs{
				SendCipherSuite: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			},
		},
		shouldFail:    true,
		expectedError: ":WRONG_CERTIFICATE_TYPE:",
	})
	testCases = append(testCases, testCase{
		name: "CertificateCipherMismatch-Ed25519",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256},
			Credential:   &ed25519Certificate,
			Bugs: ProtocolBugs{
				SendCipherSuite: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			},
		},
		shouldFail:    true,
		expectedError: ":WRONG_CERTIFICATE_TYPE:",
	})

	// Test that servers decline to select a cipher suite which is
	// inconsistent with their configured certificate.
	testCases = append(testCases, testCase{
		testType: serverTest,
		name:     "ServerCipherFilter-RSA",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256},
		},
		shimCertificate: &rsaCertificate,
		shouldFail:      true,
		expectedError:   ":NO_SHARED_CIPHER:",
	})
	testCases = append(testCases, testCase{
		testType: serverTest,
		name:     "ServerCipherFilter-ECDSA",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
		},
		shimCertificate: &ecdsaP256Certificate,
		shouldFail:      true,
		expectedError:   ":NO_SHARED_CIPHER:",
	})
	testCases = append(testCases, testCase{
		testType: serverTest,
		name:     "ServerCipherFilter-Ed25519",
		config: Config{
			MaxVersion:   VersionTLS12,
			CipherSuites: []uint16{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
		},
		shimCertificate: &ed25519Certificate,
		shouldFail:      true,
		expectedError:   ":NO_SHARED_CIPHER:",
	})

	// Test cipher suite negotiation works as expected. Configure a
	// complicated cipher suite configuration.
	const negotiationTestCiphers = "" +
		"TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:" +
		"[TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384|TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256|TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA]:" +
		"TLS_RSA_WITH_AES_128_GCM_SHA256:" +
		"TLS_RSA_WITH_AES_128_CBC_SHA:" +
		"[TLS_RSA_WITH_AES_256_GCM_SHA384|TLS_RSA_WITH_AES_256_CBC_SHA]"
	negotiationTests := []struct {
		ciphers  []uint16
		expected uint16
	}{
		// Server preferences are honored, including when
		// equipreference groups are involved.
		{
			[]uint16{
				TLS_RSA_WITH_AES_256_GCM_SHA384,
				TLS_RSA_WITH_AES_128_CBC_SHA,
				TLS_RSA_WITH_AES_128_GCM_SHA256,
				TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
				TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			},
			TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
		},
		{
			[]uint16{
				TLS_RSA_WITH_AES_256_GCM_SHA384,
				TLS_RSA_WITH_AES_128_CBC_SHA,
				TLS_RSA_WITH_AES_128_GCM_SHA256,
				TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
			},
			TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
		},
		{
			[]uint16{
				TLS_RSA_WITH_AES_256_GCM_SHA384,
				TLS_RSA_WITH_AES_128_CBC_SHA,
				TLS_RSA_WITH_AES_128_GCM_SHA256,
			},
			TLS_RSA_WITH_AES_128_GCM_SHA256,
		},
		{
			[]uint16{
				TLS_RSA_WITH_AES_256_GCM_SHA384,
				TLS_RSA_WITH_AES_128_CBC_SHA,
			},
			TLS_RSA_WITH_AES_128_CBC_SHA,
		},
		// Equipreference groups use the client preference.
		{
			[]uint16{
				TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
				TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
				TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			},
			TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,
		},
		{
			[]uint16{
				TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
				TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			},
			TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
		},
		{
			[]uint16{
				TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
				TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
			},
			TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
		},
		{
			[]uint16{
				TLS_RSA_WITH_AES_256_GCM_SHA384,
				TLS_RSA_WITH_AES_256_CBC_SHA,
			},
			TLS_RSA_WITH_AES_256_GCM_SHA384,
		},
		{
			[]uint16{
				TLS_RSA_WITH_AES_256_CBC_SHA,
				TLS_RSA_WITH_AES_256_GCM_SHA384,
			},
			TLS_RSA_WITH_AES_256_CBC_SHA,
		},
		// If there are two equipreference groups, the preferred one
		// takes precedence.
		{
			[]uint16{
				TLS_RSA_WITH_AES_256_GCM_SHA384,
				TLS_RSA_WITH_AES_256_CBC_SHA,
				TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
				TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256,
			},
			TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
		},
	}
	for i, t := range negotiationTests {
		testCases = append(testCases, testCase{
			testType: serverTest,
			name:     "CipherNegotiation-" + strconv.Itoa(i),
			config: Config{
				MaxVersion:   VersionTLS12,
				CipherSuites: t.ciphers,
			},
			flags: []string{"-cipher", negotiationTestCiphers},
			expectations: connectionExpectations{
				cipher: t.expected,
			},
		})
	}
}

func addBadECDSASignatureTests() {
	for badR := BadValue(1); badR < NumBadValues; badR++ {
		for badS := BadValue(1); badS < NumBadValues; badS++ {
			testCases = append(testCases, testCase{
				name: fmt.Sprintf("BadECDSA-%d-%d", badR, badS),
				config: Config{
					MaxVersion:   VersionTLS12,
					CipherSuites: []uint16{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256},
					Credential:   &ecdsaP256Certificate,
					Bugs: ProtocolBugs{
						BadECDSAR: badR,
						BadECDSAS: badS,
					},
				},
				shouldFail:    true,
				expectedError: ":BAD_SIGNATURE:",
			})
			testCases = append(testCases, testCase{
				name: fmt.Sprintf("BadECDSA-%d-%d-TLS13", badR, badS),
				config: Config{
					MaxVersion: VersionTLS13,
					Credential: &ecdsaP256Certificate,
					Bugs: ProtocolBugs{
						BadECDSAR: badR,
						BadECDSAS: badS,
					},
				},
				shouldFail:    true,
				expectedError: ":BAD_SIGNATURE:",
			})
		}