acvptool: implement pipelining.

This causes avcptool to send requests without blocking on responses. See
the diff in ACVP.md for details of how to use this feature.

Change-Id: I922b3bd2383cb7d22a5d12ead49d2fa733ee1b97
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/55345
Reviewed-by: David Benjamin <davidben@google.com>
Commit-Queue: Adam Langley <agl@google.com>
diff --git a/util/fipstools/acvp/ACVP.md b/util/fipstools/acvp/ACVP.md
index 1fd919f..066a39b 100644
--- a/util/fipstools/acvp/ACVP.md
+++ b/util/fipstools/acvp/ACVP.md
@@ -112,6 +112,12 @@
 
 ² Will always be one because MCT tests are not supported for CS3.
 
+### Batching
+
+Requests are written without waiting for responses. Implementations can run a read-execute-reply loop without worrying about this. However, if batching is useful then implementations may gather up multiple requests before executing them. But this risks deadlock because some requests depend on the result of the previous one. If the `getConfig` result contains a dummy entry for the algorithm `acvptool` it will be filtered out when running with `-regcap`. However, a list of strings called `features` in that block may include the string `batch` to indicate that the implementation would like to receive a `flush` command whenever previous results must be received in order to progress. Implementations that batch can observe this to avoid deadlock.
+
+The `flush` command must not produce a response itself; it only indicates that all previous responses must be received to progress. The `getConfig` command must always be serviced immediately because a flush command will not be sent prior to processing the `getConfig` response.
+
 ## Online operation
 
 If you have credentials to speak to either of the NIST ACVP servers then you can run the tool in online mode.
diff --git a/util/fipstools/acvp/acvptool/acvp.go b/util/fipstools/acvp/acvptool/acvp.go
index fd009f5..e92424b 100644
--- a/util/fipstools/acvp/acvptool/acvp.go
+++ b/util/fipstools/acvp/acvptool/acvp.go
@@ -579,6 +579,12 @@
 					continue
 				}
 			}
+			if value, ok := algo["algorithm"]; ok {
+				algorithm, ok := value.(string)
+				if ok && algorithm == "acvptool" {
+					continue
+				}
+			}
 			nonTestAlgos = append(nonTestAlgos, algo)
 		}
 
diff --git a/util/fipstools/acvp/acvptool/subprocess/aead.go b/util/fipstools/acvp/acvptool/subprocess/aead.go
index f773546..ba0eee9 100644
--- a/util/fipstools/acvp/acvptool/subprocess/aead.go
+++ b/util/fipstools/acvp/acvptool/subprocess/aead.go
@@ -161,46 +161,48 @@
 			testResp := aeadTestResponse{ID: test.ID}
 
 			if encrypt {
-				result, err := m.Transact(op, 1, uint32le(uint32(tagBytes)), key, input, nonce, aad)
-				if err != nil {
-					return nil, err
-				}
+				m.TransactAsync(op, 1, [][]byte{uint32le(uint32(tagBytes)), key, input, nonce, aad}, func(result [][]byte) error {
+					if len(result[0]) < tagBytes {
+						return fmt.Errorf("ciphertext from subprocess for test case %d/%d is shorter than the tag (%d vs %d)", group.ID, test.ID, len(result[0]), tagBytes)
+					}
 
-				if len(result[0]) < tagBytes {
-					return nil, fmt.Errorf("ciphertext from subprocess for test case %d/%d is shorter than the tag (%d vs %d)", group.ID, test.ID, len(result[0]), tagBytes)
-				}
-
-				if a.tagMergedWithCiphertext {
-					ciphertextHex := hex.EncodeToString(result[0])
-					testResp.CiphertextHex = &ciphertextHex
-				} else {
-					ciphertext := result[0][:len(result[0])-tagBytes]
-					ciphertextHex := hex.EncodeToString(ciphertext)
-					testResp.CiphertextHex = &ciphertextHex
-					tag := result[0][len(result[0])-tagBytes:]
-					testResp.TagHex = hex.EncodeToString(tag)
-				}
+					if a.tagMergedWithCiphertext {
+						ciphertextHex := hex.EncodeToString(result[0])
+						testResp.CiphertextHex = &ciphertextHex
+					} else {
+						ciphertext := result[0][:len(result[0])-tagBytes]
+						ciphertextHex := hex.EncodeToString(ciphertext)
+						testResp.CiphertextHex = &ciphertextHex
+						tag := result[0][len(result[0])-tagBytes:]
+						testResp.TagHex = hex.EncodeToString(tag)
+					}
+					response.Tests = append(response.Tests, testResp)
+					return nil
+				})
 			} else {
-				result, err := m.Transact(op, 2, uint32le(uint32(tagBytes)), key, append(input, tag...), nonce, aad)
-				if err != nil {
-					return nil, err
-				}
-
-				if len(result[0]) != 1 || (result[0][0]&0xfe) != 0 {
-					return nil, fmt.Errorf("invalid AEAD status result from subprocess")
-				}
-				passed := result[0][0] == 1
-				testResp.Passed = &passed
-				if passed {
-					plaintextHex := hex.EncodeToString(result[1])
-					testResp.PlaintextHex = &plaintextHex
-				}
+				m.TransactAsync(op, 2, [][]byte{uint32le(uint32(tagBytes)), key, append(input, tag...), nonce, aad}, func(result [][]byte) error {
+					if len(result[0]) != 1 || (result[0][0]&0xfe) != 0 {
+						return fmt.Errorf("invalid AEAD status result from subprocess")
+					}
+					passed := result[0][0] == 1
+					testResp.Passed = &passed
+					if passed {
+						plaintextHex := hex.EncodeToString(result[1])
+						testResp.PlaintextHex = &plaintextHex
+					}
+					response.Tests = append(response.Tests, testResp)
+					return nil
+				})
 			}
-
-			response.Tests = append(response.Tests, testResp)
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/block.go b/util/fipstools/acvp/acvptool/subprocess/block.go
index 0387e09..2f05802 100644
--- a/util/fipstools/acvp/acvptool/subprocess/block.go
+++ b/util/fipstools/acvp/acvptool/subprocess/block.go
@@ -397,31 +397,35 @@
 
 			testResp := blockCipherTestResponse{ID: test.ID}
 			if !mct {
-				var result [][]byte
-				var err error
-
+				var args [][]byte
 				if b.hasIV {
-					result, err = m.Transact(op, b.numResults, key, input, iv, uint32le(1))
+					args = [][]byte{key, input, iv, uint32le(1)}
 				} else {
-					result, err = m.Transact(op, b.numResults, key, input, uint32le(1))
-				}
-				if err != nil {
-					panic("block operation failed: " + err.Error())
+					args = [][]byte{key, input, uint32le(1)}
 				}
 
-				if encrypt {
-					testResp.CiphertextHex = hex.EncodeToString(result[0])
-				} else {
-					testResp.PlaintextHex = hex.EncodeToString(result[0])
-				}
+				m.TransactAsync(op, b.numResults, args, func(result [][]byte) error {
+					if encrypt {
+						testResp.CiphertextHex = hex.EncodeToString(result[0])
+					} else {
+						testResp.PlaintextHex = hex.EncodeToString(result[0])
+					}
+					response.Tests = append(response.Tests, testResp)
+					return nil
+				})
 			} else {
 				testResp.MCTResults = b.mctFunc(transact, encrypt, key, input, iv)
+				response.Tests = append(response.Tests, testResp)
 			}
-
-			response.Tests = append(response.Tests, testResp)
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/drbg.go b/util/fipstools/acvp/acvptool/subprocess/drbg.go
index 6db8a64..b403f04 100644
--- a/util/fipstools/acvp/acvptool/subprocess/drbg.go
+++ b/util/fipstools/acvp/acvptool/subprocess/drbg.go
@@ -116,7 +116,8 @@
 			var outLenBytes [4]byte
 			binary.LittleEndian.PutUint32(outLenBytes[:], uint32(outLen))
 
-			var result [][]byte
+			var cmd string
+			var args [][]byte
 			if group.PredictionResistance {
 				var a1, a2, a3, a4 []byte
 				if err := extractOtherInputs(test.Other, []drbgOtherInputExpectations{
@@ -124,7 +125,8 @@
 					{"generate", group.AdditionalDataBits, &a3, group.EntropyBits, &a4}}); err != nil {
 					return nil, fmt.Errorf("failed to parse other inputs from test case %d/%d: %s", group.ID, test.ID, err)
 				}
-				result, err = m.Transact(d.algo+"-pr/"+group.Mode, 1, outLenBytes[:], ent, perso, a1, a2, a3, a4, nonce)
+				cmd = d.algo + "-pr/" + group.Mode
+				args = [][]byte{outLenBytes[:], ent, perso, a1, a2, a3, a4, nonce}
 			} else if group.Reseed {
 				var a1, a2, a3, a4 []byte
 				if err := extractOtherInputs(test.Other, []drbgOtherInputExpectations{
@@ -133,7 +135,8 @@
 					{"generate", group.AdditionalDataBits, &a4, 0, nil}}); err != nil {
 					return nil, fmt.Errorf("failed to parse other inputs from test case %d/%d: %s", group.ID, test.ID, err)
 				}
-				result, err = m.Transact(d.algo+"-reseed/"+group.Mode, 1, outLenBytes[:], ent, perso, a1, a2, a3, a4, nonce)
+				cmd = d.algo + "-reseed/" + group.Mode
+				args = [][]byte{outLenBytes[:], ent, perso, a1, a2, a3, a4, nonce}
 			} else {
 				var a1, a2 []byte
 				if err := extractOtherInputs(test.Other, []drbgOtherInputExpectations{
@@ -141,25 +144,31 @@
 					{"generate", group.AdditionalDataBits, &a2, 0, nil}}); err != nil {
 					return nil, fmt.Errorf("failed to parse other inputs from test case %d/%d: %s", group.ID, test.ID, err)
 				}
-				result, err = m.Transact(d.algo+"/"+group.Mode, 1, outLenBytes[:], ent, perso, a1, a2, nonce)
+				cmd = d.algo + "/" + group.Mode
+				args = [][]byte{outLenBytes[:], ent, perso, a1, a2, nonce}
 			}
 
-			if err != nil {
-				return nil, fmt.Errorf("DRBG operation failed: %s", err)
-			}
+			m.TransactAsync(cmd, 1, args, func(result [][]byte) error {
+				if l := uint64(len(result[0])); l != outLen {
+					return fmt.Errorf("wrong length DRBG result: %d bytes but wanted %d", l, outLen)
+				}
 
-			if l := uint64(len(result[0])); l != outLen {
-				return nil, fmt.Errorf("wrong length DRBG result: %d bytes but wanted %d", l, outLen)
-			}
-
-			// https://pages.nist.gov/ACVP/draft-vassilev-acvp-drbg.html#name-responses
-			response.Tests = append(response.Tests, drbgTestResponse{
-				ID:     test.ID,
-				OutHex: hex.EncodeToString(result[0]),
+				// https://pages.nist.gov/ACVP/draft-vassilev-acvp-drbg.html#name-responses
+				response.Tests = append(response.Tests, drbgTestResponse{
+					ID:     test.ID,
+					OutHex: hex.EncodeToString(result[0]),
+				})
+				return nil
 			})
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/ecdsa.go b/util/fipstools/acvp/acvptool/subprocess/ecdsa.go
index 619323c..16d3a83 100644
--- a/util/fipstools/acvp/acvptool/subprocess/ecdsa.go
+++ b/util/fipstools/acvp/acvptool/subprocess/ecdsa.go
@@ -94,19 +94,20 @@
 
 		for _, test := range group.Tests {
 			var testResp ecdsaTestResponse
+			testResp.ID = test.ID
 
 			switch parsed.Mode {
 			case "keyGen":
 				if group.SecretGenerationMode != "testing candidates" {
 					return nil, fmt.Errorf("invalid secret generation mode in test group %d: %q", group.ID, group.SecretGenerationMode)
 				}
-				result, err := m.Transact(e.algo+"/"+"keyGen", 3, []byte(group.Curve))
-				if err != nil {
-					return nil, fmt.Errorf("key generation failed for test case %d/%d: %s", group.ID, test.ID, err)
-				}
-				testResp.DHex = hex.EncodeToString(result[0])
-				testResp.QxHex = hex.EncodeToString(result[1])
-				testResp.QyHex = hex.EncodeToString(result[2])
+				m.TransactAsync(e.algo+"/"+"keyGen", 3, [][]byte{[]byte(group.Curve)}, func(result [][]byte) error {
+					testResp.DHex = hex.EncodeToString(result[0])
+					testResp.QxHex = hex.EncodeToString(result[1])
+					testResp.QyHex = hex.EncodeToString(result[2])
+					response.Tests = append(response.Tests, testResp)
+					return nil
+				})
 
 			case "keyVer":
 				qx, err := hex.DecodeString(test.QxHex)
@@ -117,21 +118,21 @@
 				if err != nil {
 					return nil, fmt.Errorf("failed to decode qy in test case %d/%d: %s", group.ID, test.ID, err)
 				}
-				result, err := m.Transact(e.algo+"/"+"keyVer", 1, []byte(group.Curve), qx, qy)
-				if err != nil {
-					return nil, fmt.Errorf("key verification failed for test case %d/%d: %s", group.ID, test.ID, err)
-				}
-				// result[0] should be a single byte: zero if false, one if true
-				switch {
-				case bytes.Equal(result[0], []byte{00}):
-					f := false
-					testResp.Passed = &f
-				case bytes.Equal(result[0], []byte{01}):
-					t := true
-					testResp.Passed = &t
-				default:
-					return nil, fmt.Errorf("key verification returned unexpected result: %q", result[0])
-				}
+				m.TransactAsync(e.algo+"/"+"keyVer", 1, [][]byte{[]byte(group.Curve), qx, qy}, func(result [][]byte) error {
+					// result[0] should be a single byte: zero if false, one if true
+					switch {
+					case bytes.Equal(result[0], []byte{00}):
+						f := false
+						testResp.Passed = &f
+					case bytes.Equal(result[0], []byte{01}):
+						t := true
+						testResp.Passed = &t
+					default:
+						return fmt.Errorf("key verification returned unexpected result: %q", result[0])
+					}
+					response.Tests = append(response.Tests, testResp)
+					return nil
+				})
 
 			case "sigGen":
 				p := e.primitives[group.HashAlgo]
@@ -163,12 +164,12 @@
 					}
 					op += "/componentTest"
 				}
-				result, err := m.Transact(op, 2, []byte(group.Curve), sigGenPrivateKey, []byte(group.HashAlgo), msg)
-				if err != nil {
-					return nil, fmt.Errorf("signature generation failed for test case %d/%d: %s", group.ID, test.ID, err)
-				}
-				testResp.RHex = hex.EncodeToString(result[0])
-				testResp.SHex = hex.EncodeToString(result[1])
+				m.TransactAsync(op, 2, [][]byte{[]byte(group.Curve), sigGenPrivateKey, []byte(group.HashAlgo), msg}, func(result [][]byte) error {
+					testResp.RHex = hex.EncodeToString(result[0])
+					testResp.SHex = hex.EncodeToString(result[1])
+					response.Tests = append(response.Tests, testResp)
+					return nil
+				})
 
 			case "sigVer":
 				p := e.primitives[group.HashAlgo]
@@ -197,31 +198,34 @@
 				if err != nil {
 					return nil, fmt.Errorf("failed to decode S in test case %d/%d: %s", group.ID, test.ID, err)
 				}
-				result, err := m.Transact(e.algo+"/"+"sigVer", 1, []byte(group.Curve), []byte(group.HashAlgo), msg, qx, qy, r, s)
-				if err != nil {
-					return nil, fmt.Errorf("signature verification failed for test case %d/%d: %s", group.ID, test.ID, err)
-				}
-				// result[0] should be a single byte: zero if false, one if true
-				switch {
-				case bytes.Equal(result[0], []byte{00}):
-					f := false
-					testResp.Passed = &f
-				case bytes.Equal(result[0], []byte{01}):
-					t := true
-					testResp.Passed = &t
-				default:
-					return nil, fmt.Errorf("signature verification returned unexpected result: %q", result[0])
-				}
+				m.TransactAsync(e.algo+"/"+"sigVer", 1, [][]byte{[]byte(group.Curve), []byte(group.HashAlgo), msg, qx, qy, r, s}, func(result [][]byte) error {
+					// result[0] should be a single byte: zero if false, one if true
+					switch {
+					case bytes.Equal(result[0], []byte{00}):
+						f := false
+						testResp.Passed = &f
+					case bytes.Equal(result[0], []byte{01}):
+						t := true
+						testResp.Passed = &t
+					default:
+						return fmt.Errorf("signature verification returned unexpected result: %q", result[0])
+					}
+					response.Tests = append(response.Tests, testResp)
+					return nil
+				})
 
 			default:
 				return nil, fmt.Errorf("invalid mode %q in ECDSA vector set", parsed.Mode)
 			}
-
-			testResp.ID = test.ID
-			response.Tests = append(response.Tests, testResp)
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/hash.go b/util/fipstools/acvp/acvptool/subprocess/hash.go
index 33cea04..1f34d1a 100644
--- a/util/fipstools/acvp/acvptool/subprocess/hash.go
+++ b/util/fipstools/acvp/acvptool/subprocess/hash.go
@@ -89,14 +89,12 @@
 			// http://usnistgov.github.io/ACVP/artifacts/draft-celi-acvp-sha-00.html#rfc.section.3
 			switch group.Type {
 			case "AFT":
-				result, err := m.Transact(h.algo, 1, msg)
-				if err != nil {
-					panic(h.algo + " hash operation failed: " + err.Error())
-				}
-
-				response.Tests = append(response.Tests, hashTestResponse{
-					ID:        test.ID,
-					DigestHex: hex.EncodeToString(result[0]),
+				m.TransactAsync(h.algo, 1, [][]byte{msg}, func(result [][]byte) error {
+					response.Tests = append(response.Tests, hashTestResponse{
+						ID:        test.ID,
+						DigestHex: hex.EncodeToString(result[0]),
+					})
+					return nil
 				})
 
 			case "MCT":
@@ -124,7 +122,13 @@
 			}
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/hkdf.go b/util/fipstools/acvp/acvptool/subprocess/hkdf.go
index 555646a..3a6ba04 100644
--- a/util/fipstools/acvp/acvptool/subprocess/hkdf.go
+++ b/util/fipstools/acvp/acvptool/subprocess/hkdf.go
@@ -169,24 +169,29 @@
 			info = append(info, uData...)
 			info = append(info, vData...)
 
-			resp, err := m.Transact("HKDF/"+hashName, 1, key, salt, info, uint32le(outBytes))
-			if err != nil {
-				return nil, fmt.Errorf("HKDF operation failed: %s", err)
-			}
-			if len(resp[0]) != int(outBytes) {
-				return nil, fmt.Errorf("HKDF operation resulted in %d bytes but wanted %d", len(resp[0]), outBytes)
-			}
+			m.TransactAsync("HKDF/"+hashName, 1, [][]byte{key, salt, info, uint32le(outBytes)}, func(result [][]byte) error {
+				if len(result[0]) != int(outBytes) {
+					return fmt.Errorf("HKDF operation resulted in %d bytes but wanted %d", len(result[0]), outBytes)
+				}
+				if isValidationTest {
+					passed := bytes.Equal(expected, result[0])
+					testResp.Passed = &passed
+				} else {
+					testResp.KeyOut = hex.EncodeToString(result[0])
+				}
 
-			if isValidationTest {
-				passed := bytes.Equal(expected, resp[0])
-				testResp.Passed = &passed
-			} else {
-				testResp.KeyOut = hex.EncodeToString(resp[0])
-			}
-
-			groupResp.Tests = append(groupResp.Tests, testResp)
+				groupResp.Tests = append(groupResp.Tests, testResp)
+				return nil
+			})
 		}
-		respGroups = append(respGroups, groupResp)
+
+		m.Barrier(func() {
+			respGroups = append(respGroups, groupResp)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return respGroups, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/hmac.go b/util/fipstools/acvp/acvptool/subprocess/hmac.go
index 1859886..8fc7695 100644
--- a/util/fipstools/acvp/acvptool/subprocess/hmac.go
+++ b/util/fipstools/acvp/acvptool/subprocess/hmac.go
@@ -93,6 +93,10 @@
 		if group.MACBits > h.mdLen*8 {
 			return nil, fmt.Errorf("test group %d specifies MAC length should be %d, but maximum possible length is %d", group.ID, group.MACBits, h.mdLen*8)
 		}
+		if group.MACBits%8 != 0 {
+			return nil, fmt.Errorf("fractional-byte HMAC output length requested: %d", group.MACBits)
+		}
+		outBytes := group.MACBits / 8
 
 		for _, test := range group.Tests {
 			if len(test.MsgHex)*4 != group.MsgBits {
@@ -111,14 +115,27 @@
 				return nil, fmt.Errorf("failed to decode key in test case %d/%d: %s", group.ID, test.ID, err)
 			}
 
-			// https://pages.nist.gov/ACVP/draft-fussell-acvp-mac.html#name-test-vectors
-			response.Tests = append(response.Tests, hmacTestResponse{
-				ID:     test.ID,
-				MACHex: hex.EncodeToString(h.hmac(msg, key, group.MACBits, m)),
+			m.TransactAsync(h.algo, 1, [][]byte{msg, key}, func(result [][]byte) error {
+				if l := len(result[0]); l < outBytes {
+					return fmt.Errorf("HMAC result too short: %d bytes but wanted %d", l, outBytes)
+				}
+
+				// https://pages.nist.gov/ACVP/draft-fussell-acvp-mac.html#name-test-vectors
+				response.Tests = append(response.Tests, hmacTestResponse{
+					ID:     test.ID,
+					MACHex: hex.EncodeToString(result[0][:outBytes]),
+				})
+				return nil
 			})
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/kas.go b/util/fipstools/acvp/acvptool/subprocess/kas.go
index 9625334..cbc99ed 100644
--- a/util/fipstools/acvp/acvptool/subprocess/kas.go
+++ b/util/fipstools/acvp/acvptool/subprocess/kas.go
@@ -155,40 +155,42 @@
 					return nil, err
 				}
 
-				result, err := m.Transact(method, 3, peerX, peerY, privateKey)
-				if err != nil {
-					return nil, err
-				}
-
-				ok := bytes.Equal(result[2], expectedOutput)
-				response.Tests = append(response.Tests, kasTestResponse{
-					ID:     test.ID,
-					Passed: &ok,
+				m.TransactAsync(method, 3, [][]byte{peerX, peerY, privateKey}, func(result [][]byte) error {
+					ok := bytes.Equal(result[2], expectedOutput)
+					response.Tests = append(response.Tests, kasTestResponse{
+						ID:     test.ID,
+						Passed: &ok,
+					})
+					return nil
 				})
 			} else {
-				result, err := m.Transact(method, 3, peerX, peerY, nil)
-				if err != nil {
-					return nil, err
-				}
+				m.TransactAsync(method, 3, [][]byte{peerX, peerY, nil}, func(result [][]byte) error {
+					testResponse := kasTestResponse{
+						ID:        test.ID,
+						ResultHex: hex.EncodeToString(result[2]),
+					}
 
-				testResponse := kasTestResponse{
-					ID:        test.ID,
-					ResultHex: hex.EncodeToString(result[2]),
-				}
+					if useStaticNamedFields {
+						testResponse.StaticXHex = hex.EncodeToString(result[0])
+						testResponse.StaticYHex = hex.EncodeToString(result[1])
+					} else {
+						testResponse.EphemeralXHex = hex.EncodeToString(result[0])
+						testResponse.EphemeralYHex = hex.EncodeToString(result[1])
+					}
 
-				if useStaticNamedFields {
-					testResponse.StaticXHex = hex.EncodeToString(result[0])
-					testResponse.StaticYHex = hex.EncodeToString(result[1])
-				} else {
-					testResponse.EphemeralXHex = hex.EncodeToString(result[0])
-					testResponse.EphemeralYHex = hex.EncodeToString(result[1])
-				}
-
-				response.Tests = append(response.Tests, testResponse)
+					response.Tests = append(response.Tests, testResponse)
+					return nil
+				})
 			}
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/kasdh.go b/util/fipstools/acvp/acvptool/subprocess/kasdh.go
index a9de2e3..f262b82 100644
--- a/util/fipstools/acvp/acvptool/subprocess/kasdh.go
+++ b/util/fipstools/acvp/acvptool/subprocess/kasdh.go
@@ -139,31 +139,33 @@
 					return nil, err
 				}
 
-				result, err := m.Transact(method, 2, p, q, g, peerPublic, privateKey, publicKey)
-				if err != nil {
-					return nil, err
-				}
-
-				ok := bytes.Equal(result[1], expectedOutput)
-				response.Tests = append(response.Tests, kasDHTestResponse{
-					ID:     test.ID,
-					Passed: &ok,
+				m.TransactAsync(method, 2, [][]byte{p, q, g, peerPublic, privateKey, publicKey}, func(result [][]byte) error {
+					ok := bytes.Equal(result[1], expectedOutput)
+					response.Tests = append(response.Tests, kasDHTestResponse{
+						ID:     test.ID,
+						Passed: &ok,
+					})
+					return nil
 				})
 			} else {
-				result, err := m.Transact(method, 2, p, q, g, peerPublic, nil, nil)
-				if err != nil {
-					return nil, err
-				}
-
-				response.Tests = append(response.Tests, kasDHTestResponse{
-					ID:             test.ID,
-					LocalPublicHex: hex.EncodeToString(result[0]),
-					ResultHex:      hex.EncodeToString(result[1]),
+				m.TransactAsync(method, 2, [][]byte{p, q, g, peerPublic, nil, nil}, func(result [][]byte) error {
+					response.Tests = append(response.Tests, kasDHTestResponse{
+						ID:             test.ID,
+						LocalPublicHex: hex.EncodeToString(result[0]),
+						ResultHex:      hex.EncodeToString(result[1]),
+					})
+					return nil
 				})
 			}
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/kdf.go b/util/fipstools/acvp/acvptool/subprocess/kdf.go
index e61d26b..e27fcaa 100644
--- a/util/fipstools/acvp/acvptool/subprocess/kdf.go
+++ b/util/fipstools/acvp/acvptool/subprocess/kdf.go
@@ -106,26 +106,30 @@
 			}
 
 			// Make the call to the crypto module.
-			resp, err := m.Transact("KDF-counter", 3, outputBytes, []byte(group.MACMode), []byte(group.CounterLocation), key, counterBits)
-			if err != nil {
-				return nil, fmt.Errorf("wrapper KDF operation failed: %s", err)
-			}
+			m.TransactAsync("KDF-counter", 3, [][]byte{outputBytes, []byte(group.MACMode), []byte(group.CounterLocation), key, counterBits}, func(result [][]byte) error {
+				testResp.ID = test.ID
+				if test.Deferred {
+					testResp.KeyIn = hex.EncodeToString(result[0])
+				}
+				testResp.FixedData = hex.EncodeToString(result[1])
+				testResp.KeyOut = hex.EncodeToString(result[2])
 
-			// Parse results.
-			testResp.ID = test.ID
-			if test.Deferred {
-				testResp.KeyIn = hex.EncodeToString(resp[0])
-			}
-			testResp.FixedData = hex.EncodeToString(resp[1])
-			testResp.KeyOut = hex.EncodeToString(resp[2])
+				if !test.Deferred && !bytes.Equal(result[0], key) {
+					return fmt.Errorf("wrapper returned a different key for non-deferred KDF operation")
+				}
 
-			if !test.Deferred && !bytes.Equal(resp[0], key) {
-				return nil, fmt.Errorf("wrapper returned a different key for non-deferred KDF operation")
-			}
-
-			groupResp.Tests = append(groupResp.Tests, testResp)
+				groupResp.Tests = append(groupResp.Tests, testResp)
+				return nil
+			})
 		}
-		respGroups = append(respGroups, groupResp)
+
+		m.Barrier(func() {
+			respGroups = append(respGroups, groupResp)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return respGroups, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/keyedMac.go b/util/fipstools/acvp/acvptool/subprocess/keyedMac.go
index a722ac9..e43ab5d 100644
--- a/util/fipstools/acvp/acvptool/subprocess/keyedMac.go
+++ b/util/fipstools/acvp/acvptool/subprocess/keyedMac.go
@@ -122,17 +122,17 @@
 			}
 
 			if generate {
-				result, err := m.Transact(k.algo, 1, outputBytes, key, msg)
-				if err != nil {
-					return nil, fmt.Errorf("wrapper %s operation failed: %s", k.algo, err)
-				}
+				expectedNumBytes := int(group.MACBits / 8)
 
-				calculatedMAC := result[0]
-				if len(calculatedMAC) != int(group.MACBits/8) {
-					return nil, fmt.Errorf("%s operation returned incorrect length value", k.algo)
-				}
+				m.TransactAsync(k.algo, 1, [][]byte{outputBytes, key, msg}, func(result [][]byte) error {
+					calculatedMAC := result[0]
+					if len(calculatedMAC) != expectedNumBytes {
+						return fmt.Errorf("%s operation returned incorrect length value", k.algo)
+					}
 
-				respTest.MACHex = hex.EncodeToString(calculatedMAC)
+					respTest.MACHex = hex.EncodeToString(calculatedMAC)
+					return nil
+				})
 			} else {
 				expectedMAC, err := hex.DecodeString(test.MACHex)
 				if err != nil {
@@ -142,23 +142,29 @@
 					return nil, fmt.Errorf("MACHex in test case %d/%d is %x, but should be %d bits", group.ID, test.ID, expectedMAC, group.MACBits)
 				}
 
-				result, err := m.Transact(k.algo+"/verify", 1, key, msg, expectedMAC)
-				if err != nil {
-					return nil, fmt.Errorf("wrapper %s operation failed: %s", k.algo, err)
-				}
+				m.TransactAsync(k.algo+"/verify", 1, [][]byte{key, msg, expectedMAC}, func(result [][]byte) error {
+					if len(result[0]) != 1 || (result[0][0]&0xfe) != 0 {
+						return fmt.Errorf("wrapper %s returned invalid success flag: %x", k.algo, result[0])
+					}
 
-				if len(result[0]) != 1 || (result[0][0]&0xfe) != 0 {
-					return nil, fmt.Errorf("wrapper %s returned invalid success flag: %x", k.algo, result[0])
-				}
-
-				ok := result[0][0] == 1
-				respTest.Passed = &ok
+					ok := result[0][0] == 1
+					respTest.Passed = &ok
+					return nil
+				})
 			}
 
-			respGroup.Tests = append(respGroup.Tests, respTest)
+			m.Barrier(func() {
+				respGroup.Tests = append(respGroup.Tests, respTest)
+			})
 		}
 
-		respGroups = append(respGroups, respGroup)
+		m.Barrier(func() {
+			respGroups = append(respGroups, respGroup)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return respGroups, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/rsa.go b/util/fipstools/acvp/acvptool/subprocess/rsa.go
index d29dd23..d975026 100644
--- a/util/fipstools/acvp/acvptool/subprocess/rsa.go
+++ b/util/fipstools/acvp/acvptool/subprocess/rsa.go
@@ -137,22 +137,26 @@
 		}
 
 		for _, test := range group.Tests {
-			results, err := m.Transact("RSA/keyGen", 5, uint32le(group.ModulusBits))
-			if err != nil {
-				return nil, err
-			}
-
-			response.Tests = append(response.Tests, rsaKeyGenTestResponse{
-				ID: test.ID,
-				E:  hex.EncodeToString(results[0]),
-				P:  hex.EncodeToString(results[1]),
-				Q:  hex.EncodeToString(results[2]),
-				N:  hex.EncodeToString(results[3]),
-				D:  hex.EncodeToString(results[4]),
+			m.TransactAsync("RSA/keyGen", 5, [][]byte{uint32le(group.ModulusBits)}, func(result [][]byte) error {
+				response.Tests = append(response.Tests, rsaKeyGenTestResponse{
+					ID: test.ID,
+					E:  hex.EncodeToString(result[0]),
+					P:  hex.EncodeToString(result[1]),
+					Q:  hex.EncodeToString(result[2]),
+					N:  hex.EncodeToString(result[3]),
+					D:  hex.EncodeToString(result[4]),
+				})
+				return nil
 			})
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
@@ -185,25 +189,29 @@
 				return nil, fmt.Errorf("test case %d/%d contains invalid hex: %s", group.ID, test.ID, err)
 			}
 
-			results, err := m.Transact(operation, 3, uint32le(group.ModulusBits), msg)
-			if err != nil {
-				return nil, err
-			}
+			m.TransactAsync(operation, 3, [][]byte{uint32le(group.ModulusBits), msg}, func(result [][]byte) error {
+				if len(response.N) == 0 {
+					response.N = hex.EncodeToString(result[0])
+					response.E = hex.EncodeToString(result[1])
+				} else if response.N != hex.EncodeToString(result[0]) {
+					return fmt.Errorf("module wrapper returned different RSA keys for the same SigGen configuration")
+				}
 
-			if len(response.N) == 0 {
-				response.N = hex.EncodeToString(results[0])
-				response.E = hex.EncodeToString(results[1])
-			} else if response.N != hex.EncodeToString(results[0]) {
-				return nil, fmt.Errorf("module wrapper returned different RSA keys for the same SigGen configuration")
-			}
-
-			response.Tests = append(response.Tests, rsaSigGenTestResponse{
-				ID:  test.ID,
-				Sig: hex.EncodeToString(results[2]),
+				response.Tests = append(response.Tests, rsaSigGenTestResponse{
+					ID:  test.ID,
+					Sig: hex.EncodeToString(result[2]),
+				})
+				return nil
 			})
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
@@ -249,18 +257,22 @@
 				return nil, fmt.Errorf("test case %d/%d contains invalid hex: %s", group.ID, test.ID, err)
 			}
 
-			results, err := m.Transact(operation, 1, n, e, msg, sig)
-			if err != nil {
-				return nil, err
-			}
-
-			response.Tests = append(response.Tests, rsaSigVerTestResponse{
-				ID:     test.ID,
-				Passed: len(results[0]) == 1 && results[0][0] == 1,
+			m.TransactAsync(operation, 1, [][]byte{n, e, msg, sig}, func(result [][]byte) error {
+				response.Tests = append(response.Tests, rsaSigVerTestResponse{
+					ID:     test.ID,
+					Passed: len(result[0]) == 1 && result[0][0] == 1,
+				})
+				return nil
 			})
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/subprocess.go b/util/fipstools/acvp/acvptool/subprocess/subprocess.go
index ee1cb87..9167b47 100644
--- a/util/fipstools/acvp/acvptool/subprocess/subprocess.go
+++ b/util/fipstools/acvp/acvptool/subprocess/subprocess.go
@@ -30,6 +30,9 @@
 // that don't call a server.
 type Transactable interface {
 	Transact(cmd string, expectedResults int, args ...[]byte) ([][]byte, error)
+	TransactAsync(cmd string, expectedResults int, args [][]byte, callback func([][]byte) error)
+	Barrier(callback func()) error
+	Flush() error
 }
 
 // Subprocess is a "middle" layer that interacts with a FIPS module via running
@@ -39,6 +42,26 @@
 	stdin      io.WriteCloser
 	stdout     io.ReadCloser
 	primitives map[string]primitive
+	// supportsFlush is true if the modulewrapper indicated that it wants to receive flush commands.
+	supportsFlush bool
+	// pendingReads is a queue of expected responses. `readerRoutine` reads each response and calls the callback in the matching pendingRead.
+	pendingReads chan pendingRead
+	// readerFinished is a channel that is closed if `readerRoutine` has finished (e.g. because of a read error).
+	readerFinished chan struct{}
+	// readerError is set iff readerFinished is closed. If non-nil then it is the read error that caused `readerRoutine` to finished.
+	readerError error
+}
+
+// pendingRead represents an expected response from the modulewrapper.
+type pendingRead struct {
+	// barrierCallback is called as soon as this pendingRead is the next in the queue, before any read from the modulewrapper.
+	barrierCallback func()
+
+	// callback is called with the result from the modulewrapper. If this is nil then no read is performed.
+	callback func(result [][]byte) error
+	// cmd is the command that requested this read for logging purposes.
+	cmd                string
+	expectedNumResults int
 }
 
 // New returns a new Subprocess middle layer that runs the given binary.
@@ -61,13 +84,18 @@
 	return NewWithIO(cmd, stdin, stdout), nil
 }
 
+// maxPending is the maximum number of requests that can be in the pipeline.
+const maxPending = 4096
+
 // NewWithIO returns a new Subprocess middle layer with the given ReadCloser and
 // WriteCloser. The returned Subprocess will call Wait on the Cmd when closed.
 func NewWithIO(cmd *exec.Cmd, in io.WriteCloser, out io.ReadCloser) *Subprocess {
 	m := &Subprocess{
-		cmd:    cmd,
-		stdin:  in,
-		stdout: out,
+		cmd:            cmd,
+		stdin:          in,
+		stdout:         out,
+		pendingReads:   make(chan pendingRead, maxPending),
+		readerFinished: make(chan struct{}),
 	}
 
 	m.primitives = map[string]primitive{
@@ -116,6 +144,7 @@
 	}
 	m.primitives["ECDSA"] = &ecdsa{"ECDSA", map[string]bool{"P-224": true, "P-256": true, "P-384": true, "P-521": true}, m.primitives}
 
+	go m.readerRoutine()
 	return m
 }
 
@@ -124,10 +153,57 @@
 	m.stdout.Close()
 	m.stdin.Close()
 	m.cmd.Wait()
+	<-m.readerFinished
 }
 
-// Transact performs a single request--response pair with the subprocess.
-func (m *Subprocess) Transact(cmd string, expectedResults int, args ...[]byte) ([][]byte, error) {
+func (m *Subprocess) flush() error {
+	if !m.supportsFlush {
+		return nil
+	}
+
+	const cmd = "flush"
+	buf := make([]byte, 8, 8+len(cmd))
+	binary.LittleEndian.PutUint32(buf, 1)
+	binary.LittleEndian.PutUint32(buf[4:], uint32(len(cmd)))
+	buf = append(buf, []byte(cmd)...)
+
+	if _, err := m.stdin.Write(buf); err != nil {
+		return err
+	}
+	return nil
+}
+
+func (m *Subprocess) enqueueRead(pending pendingRead) error {
+	select {
+	case <-m.readerFinished:
+		return m.readerError
+	default:
+	}
+
+	select {
+	case m.pendingReads <- pending:
+		break
+	default:
+		// `pendingReads` is full. Ensure that the modulewrapper will process
+		// some outstanding requests to free up space in the queue.
+		if err := m.flush(); err != nil {
+			return err
+		}
+		m.pendingReads <- pending
+	}
+
+	return nil
+}
+
+// TransactAsync performs a single request--response pair with the subprocess.
+// The callback will run at some future point, in a separate goroutine. All
+// callbacks will, however, be run in the order that TransactAsync was called.
+// Use Flush to wait for all outstanding callbacks.
+func (m *Subprocess) TransactAsync(cmd string, expectedNumResults int, args [][]byte, callback func(result [][]byte) error) {
+	if err := m.enqueueRead(pendingRead{nil, callback, cmd, expectedNumResults}); err != nil {
+		panic(err)
+	}
+
 	argLength := len(cmd)
 	for _, arg := range args {
 		argLength += len(arg)
@@ -145,17 +221,90 @@
 	}
 
 	if _, err := m.stdin.Write(buf); err != nil {
+		panic(err)
+	}
+}
+
+// Flush tells the subprocess to complete all outstanding requests and waits
+// for all outstanding TransactAsync callbacks to complete.
+func (m *Subprocess) Flush() error {
+	if m.supportsFlush {
+		m.flush()
+	}
+
+	done := make(chan struct{})
+	if err := m.enqueueRead(pendingRead{barrierCallback: func() {
+		close(done)
+	}}); err != nil {
+		return err
+	}
+
+	<-done
+	return nil
+}
+
+// Barrier runs callback after all outstanding TransactAsync callbacks have
+// been run.
+func (m *Subprocess) Barrier(callback func()) error {
+	return m.enqueueRead(pendingRead{barrierCallback: callback})
+}
+
+func (m *Subprocess) Transact(cmd string, expectedNumResults int, args ...[]byte) ([][]byte, error) {
+	done := make(chan struct{})
+	var result [][]byte
+	m.TransactAsync(cmd, expectedNumResults, args, func(r [][]byte) error {
+		result = r
+		close(done)
+		return nil
+	})
+
+	if err := m.flush(); err != nil {
 		return nil, err
 	}
 
-	buf = buf[:4]
+	select {
+	case <-done:
+		return result, nil
+	case <-m.readerFinished:
+		return nil, m.readerError
+	}
+}
+
+func (m *Subprocess) readerRoutine() {
+	defer close(m.readerFinished)
+
+	for pendingRead := range m.pendingReads {
+		if pendingRead.barrierCallback != nil {
+			pendingRead.barrierCallback()
+		}
+
+		if pendingRead.callback == nil {
+			continue
+		}
+
+		result, err := m.readResult(pendingRead.cmd, pendingRead.expectedNumResults)
+		if err != nil {
+			m.readerError = err
+			return
+		}
+
+		if err := pendingRead.callback(result); err != nil {
+			m.readerError = err
+			return
+		}
+	}
+}
+
+func (m *Subprocess) readResult(cmd string, expectedNumResults int) ([][]byte, error) {
+	buf := make([]byte, 4)
+
 	if _, err := io.ReadFull(m.stdout, buf); err != nil {
 		return nil, err
 	}
 
 	numResults := binary.LittleEndian.Uint32(buf)
-	if int(numResults) != expectedResults {
-		return nil, fmt.Errorf("expected %d results from %q but got %d", expectedResults, cmd, numResults)
+	if int(numResults) != expectedNumResults {
+		return nil, fmt.Errorf("expected %d results from %q but got %d", expectedNumResults, cmd, numResults)
 	}
 
 	buf = make([]byte, 4*numResults)
@@ -197,16 +346,25 @@
 		return nil, err
 	}
 	var config []struct {
-		Algorithm string `json:"algorithm"`
+		Algorithm string   `json:"algorithm"`
+		Features  []string `json:"features"`
 	}
 	if err := json.Unmarshal(results[0], &config); err != nil {
 		return nil, errors.New("failed to parse config response from wrapper: " + err.Error())
 	}
 	for _, algo := range config {
-		if _, ok := m.primitives[algo.Algorithm]; !ok {
+		if algo.Algorithm == "acvptool" {
+			for _, feature := range algo.Features {
+				switch feature {
+				case "batch":
+					m.supportsFlush = true
+				}
+			}
+		} else if _, ok := m.primitives[algo.Algorithm]; !ok {
 			return nil, fmt.Errorf("wrapper config advertises support for unknown algorithm %q", algo.Algorithm)
 		}
 	}
+
 	return results[0], nil
 }
 
diff --git a/util/fipstools/acvp/acvptool/subprocess/tls13.go b/util/fipstools/acvp/acvptool/subprocess/tls13.go
index 6e14942..af2aae8 100644
--- a/util/fipstools/acvp/acvptool/subprocess/tls13.go
+++ b/util/fipstools/acvp/acvptool/subprocess/tls13.go
@@ -36,7 +36,7 @@
 }
 
 type tls13Test struct {
-	ID                uint64 `json:"tcId"`
+	ID uint64 `json:"tcId"`
 	// Although ACVP refers to these as client and server randoms, these
 	// fields are misnamed and really contain portions of the handshake
 	// transcript. Concatenated in order, they give the transcript up to
diff --git a/util/fipstools/acvp/acvptool/subprocess/tlskdf.go b/util/fipstools/acvp/acvptool/subprocess/tlskdf.go
index 5b28e3f..3a0d7ce 100644
--- a/util/fipstools/acvp/acvptool/subprocess/tlskdf.go
+++ b/util/fipstools/acvp/acvptool/subprocess/tlskdf.go
@@ -118,19 +118,23 @@
 			binary.LittleEndian.PutUint32(outLenBytes[:], uint32(group.KeyBlockBits/8))
 			// TLS 1.0, 1.1, and 1.2 use a different order for the client and server
 			// randoms when computing the key block.
-			result2, err := m.Transact(method, 1, outLenBytes[:], result[0], []byte(keyBlockLabel), serverRandom, clientRandom)
-			if err != nil {
-				return nil, err
-			}
-
-			response.Tests = append(response.Tests, tlsKDFTestResponse{
-				ID:              test.ID,
-				MasterSecretHex: hex.EncodeToString(result[0]),
-				KeyBlockHex:     hex.EncodeToString(result2[0]),
+			m.TransactAsync(method, 1, [][]byte{outLenBytes[:], result[0], []byte(keyBlockLabel), serverRandom, clientRandom}, func(result2 [][]byte) error {
+				response.Tests = append(response.Tests, tlsKDFTestResponse{
+					ID:              test.ID,
+					MasterSecretHex: hex.EncodeToString(result[0]),
+					KeyBlockHex:     hex.EncodeToString(result2[0]),
+				})
+				return nil
 			})
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/subprocess/xts.go b/util/fipstools/acvp/acvptool/subprocess/xts.go
index 50eb6fd..e813409 100644
--- a/util/fipstools/acvp/acvptool/subprocess/xts.go
+++ b/util/fipstools/acvp/acvptool/subprocess/xts.go
@@ -126,22 +126,26 @@
 				return nil, fmt.Errorf("failed to decode hex in test case %d/%d: %s", group.ID, test.ID, err)
 			}
 
-			result, err := m.Transact(funcName, 1, key, msg, tweak[:])
-			if err != nil {
-				return nil, fmt.Errorf("submodule failed on test case %d/%d: %s", group.ID, test.ID, err)
-			}
+			m.TransactAsync(funcName, 1, [][]byte{key, msg, tweak[:]}, func(result [][]byte) error {
+				testResponse := xtsTestResponse{ID: test.ID}
+				if decrypt {
+					testResponse.PlaintextHex = hex.EncodeToString(result[0])
+				} else {
+					testResponse.CiphertextHex = hex.EncodeToString(result[0])
+				}
 
-			testResponse := xtsTestResponse{ID: test.ID}
-			if decrypt {
-				testResponse.PlaintextHex = hex.EncodeToString(result[0])
-			} else {
-				testResponse.CiphertextHex = hex.EncodeToString(result[0])
-			}
-
-			response.Tests = append(response.Tests, testResponse)
+				response.Tests = append(response.Tests, testResponse)
+				return nil
+			})
 		}
 
-		ret = append(ret, response)
+		m.Barrier(func() {
+			ret = append(ret, response)
+		})
+	}
+
+	if err := m.Flush(); err != nil {
+		return nil, err
 	}
 
 	return ret, nil
diff --git a/util/fipstools/acvp/acvptool/testmodulewrapper/testmodulewrapper.go b/util/fipstools/acvp/acvptool/testmodulewrapper/testmodulewrapper.go
index 8c4c97a..4cf5069 100644
--- a/util/fipstools/acvp/acvptool/testmodulewrapper/testmodulewrapper.go
+++ b/util/fipstools/acvp/acvptool/testmodulewrapper/testmodulewrapper.go
@@ -35,7 +35,13 @@
 	"golang.org/x/crypto/xts"
 )
 
+var (
+	output       io.Writer
+	outputBuffer *bytes.Buffer
+)
+
 var handlers = map[string]func([][]byte) error{
+	"flush":                    flush,
 	"getConfig":                getConfig,
 	"KDF-counter":              kdfCounter,
 	"AES-XTS/encrypt":          xtsEncrypt,
@@ -47,13 +53,29 @@
 	"AES-CBC-CS3/decrypt":      ctsDecrypt,
 }
 
+func flush(args [][]byte) error {
+	if outputBuffer == nil {
+		return nil
+	}
+
+	if _, err := os.Stdout.Write(outputBuffer.Bytes()); err != nil {
+		return err
+	}
+	outputBuffer = new(bytes.Buffer)
+	output = outputBuffer
+	return nil
+}
+
 func getConfig(args [][]byte) error {
 	if len(args) != 0 {
 		return fmt.Errorf("getConfig received %d args", len(args))
 	}
 
-	return reply([]byte(`[
+	if err := reply([]byte(`[
 	{
+		"algorithm": "acvptool",
+		"features": ["batch"]
+	}, {
 		"algorithm": "KDF",
 		"revision": "1.0",
 		"capabilities": [{
@@ -146,7 +168,11 @@
 		  256
 		]
 	}
-]`))
+]`)); err != nil {
+		return err
+	}
+
+	return flush(nil)
 }
 
 func kdfCounter(args [][]byte) error {
@@ -207,12 +233,12 @@
 	}
 
 	lengthsLength := (1 + len(responses)) * 4
-	if n, err := os.Stdout.Write(lengths[:lengthsLength]); n != lengthsLength || err != nil {
+	if n, err := output.Write(lengths[:lengthsLength]); n != lengthsLength || err != nil {
 		return fmt.Errorf("write failed: %s", err)
 	}
 
 	for _, response := range responses {
-		if n, err := os.Stdout.Write(response); n != len(response) || err != nil {
+		if n, err := output.Write(response); n != len(response) || err != nil {
 			return fmt.Errorf("write failed: %s", err)
 		}
 	}
@@ -460,6 +486,10 @@
 }
 
 func do() error {
+	// In order to exercise pipelining, all output is buffered until a "flush".
+	outputBuffer = new(bytes.Buffer)
+	output = outputBuffer
+
 	var nums [4 * (1 + maxArgs)]byte
 	var argLengths [maxArgs]uint32
 	var args [maxArgs][]byte
diff --git a/util/fipstools/acvp/modulewrapper/main.cc b/util/fipstools/acvp/modulewrapper/main.cc
index 03aead5..546091f 100644
--- a/util/fipstools/acvp/modulewrapper/main.cc
+++ b/util/fipstools/acvp/modulewrapper/main.cc
@@ -23,6 +23,10 @@
 #include "modulewrapper.h"
 
 
+static bool EqString(bssl::Span<const uint8_t> cmd, const char *str) {
+  return cmd.size() == strlen(str) && memcmp(str, cmd.data(), cmd.size()) == 0;
+}
+
 int main(int argc, char **argv) {
   if (argc == 2 && strcmp(argv[1], "--version") == 0) {
     printf("Built for architecture: ");
@@ -46,10 +50,14 @@
     return 4;
   }
 
+  // modulewrapper buffers responses to the greatest degree allowed in order to
+  // fully exercise the async handling in acvptool.
   std::unique_ptr<bssl::acvp::RequestBuffer> buffer =
       bssl::acvp::RequestBuffer::New();
   const bssl::acvp::ReplyCallback write_reply = std::bind(
       bssl::acvp::WriteReplyToFd, STDOUT_FILENO, std::placeholders::_1);
+  const bssl::acvp::ReplyCallback buffer_reply =
+      std::bind(bssl::acvp::WriteReplyToBuffer, std::placeholders::_1);
 
   for (;;) {
     const bssl::Span<const bssl::Span<const uint8_t>> args =
@@ -58,12 +66,21 @@
       return 1;
     }
 
+    if (EqString(args[0], "flush")) {
+      if (!bssl::acvp::FlushBuffer(STDOUT_FILENO)) {
+        abort();
+      }
+      continue;
+    }
+
     const bssl::acvp::Handler handler = bssl::acvp::FindHandler(args);
     if (!handler) {
       return 2;
     }
 
-    if (!handler(args.subspan(1).data(), write_reply)) {
+    auto &reply_callback =
+        EqString(args[0], "getConfig") ? write_reply : buffer_reply;
+    if (!handler(args.subspan(1).data(), reply_callback)) {
       const std::string name(reinterpret_cast<const char *>(args[0].data()),
                              args[0].size());
       fprintf(stderr, "\'%s\' operation failed.\n", name.c_str());
diff --git a/util/fipstools/acvp/modulewrapper/modulewrapper.cc b/util/fipstools/acvp/modulewrapper/modulewrapper.cc
index f417b64..0c1359e 100644
--- a/util/fipstools/acvp/modulewrapper/modulewrapper.cc
+++ b/util/fipstools/acvp/modulewrapper/modulewrapper.cc
@@ -165,8 +165,51 @@
   return Span<const Span<const uint8_t>>(buffer->args, num_args);
 }
 
+// g_reply_buffer contains buffered replies which will be flushed when acvp
+// requests.
+static std::vector<uint8_t> g_reply_buffer;
+
+bool WriteReplyToBuffer(const std::vector<Span<const uint8_t>> &spans) {
+  if (spans.size() > kMaxArgs) {
+    abort();
+  }
+
+  uint8_t buf[4];
+  CRYPTO_store_u32_le(buf, spans.size());
+  g_reply_buffer.insert(g_reply_buffer.end(), buf, buf + sizeof(buf));
+  for (const auto &span : spans) {
+    CRYPTO_store_u32_le(buf, span.size());
+    g_reply_buffer.insert(g_reply_buffer.end(), buf, buf + sizeof(buf));
+  }
+  for (const auto &span : spans) {
+    g_reply_buffer.insert(g_reply_buffer.end(), span.begin(), span.end());
+  }
+
+  return true;
+}
+
+bool FlushBuffer(int fd) {
+  size_t done = 0;
+
+  while (done < g_reply_buffer.size()) {
+    ssize_t n;
+    do {
+      n = write(fd, g_reply_buffer.data() + done, g_reply_buffer.size() - done);
+    } while (n < 0 && errno == EINTR);
+
+    if (n < 0) {
+      return false;
+    }
+    done += static_cast<size_t>(n);
+  }
+
+  g_reply_buffer.clear();
+
+  return true;
+}
+
 bool WriteReplyToFd(int fd, const std::vector<Span<const uint8_t>> &spans) {
-  if (spans.empty() || spans.size() > kMaxArgs) {
+  if (spans.size() > kMaxArgs) {
     abort();
   }
 
@@ -228,6 +271,10 @@
   static constexpr char kConfig[] =
       R"([
       {
+        "algorithm": "acvptool",
+        "features": ["batch"]
+      },
+      {
         "algorithm": "SHA2-224",
         "revision": "1.0",
         "messageLength": [{
@@ -930,6 +977,14 @@
       reinterpret_cast<const uint8_t *>(kConfig), sizeof(kConfig) - 1)});
 }
 
+static bool Flush(const Span<const uint8_t> args[], ReplyCallback write_reply) {
+  fprintf(
+      stderr,
+      "modulewrapper code processed a `flush` command but this must be handled "
+      "at a higher-level. See the example in main.cc in BoringSSL\n");
+  abort();
+}
+
 template <uint8_t *(*OneShotHash)(const uint8_t *, size_t, uint8_t *),
           size_t DigestLength>
 static bool Hash(const Span<const uint8_t> args[], ReplyCallback write_reply) {
@@ -2047,6 +2102,7 @@
   bool (*handler)(const Span<const uint8_t> args[], ReplyCallback write_reply);
 } kFunctions[] = {
     {"getConfig", 0, GetConfig},
+    {"flush", 0, Flush},
     {"SHA-1", 1, Hash<SHA1, SHA_DIGEST_LENGTH>},
     {"SHA2-224", 1, Hash<SHA224, SHA224_DIGEST_LENGTH>},
     {"SHA2-256", 1, Hash<SHA256, SHA256_DIGEST_LENGTH>},
diff --git a/util/fipstools/acvp/modulewrapper/modulewrapper.h b/util/fipstools/acvp/modulewrapper/modulewrapper.h
index cb8f9f3..8a5bdb8 100644
--- a/util/fipstools/acvp/modulewrapper/modulewrapper.h
+++ b/util/fipstools/acvp/modulewrapper/modulewrapper.h
@@ -49,6 +49,13 @@
 // WriteReplyToFd writes a reply to the given file descriptor.
 bool WriteReplyToFd(int fd, const std::vector<Span<const uint8_t>> &spans);
 
+// WriteReplyToBuffer writes a reply to an internal buffer that may be flushed
+// with |FlushBuffer|.
+bool WriteReplyToBuffer(const std::vector<Span<const uint8_t>> &spans);
+
+// FlushBuffer writes the buffer that |WriteReplyToBuffer| fills, to |fd|.
+bool FlushBuffer(int fd);
+
 // ReplyCallback is the type of a callback that writes a reply to an ACVP
 // request.
 typedef std::function<bool(const std::vector<Span<const uint8_t>> &)>