Finish porting dsa_test.cc to GTest.

Along the way, this fixes some size_t truncations.

Bug: 516
Change-Id: Iff0cf6ced0b7deb4c48c268e051a4da433088056
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/55453
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: Bob Beck <bbe@google.com>

crypto/dsa/dsa_test.cc

diff --git a/crypto/dsa/dsa_test.cc b/crypto/dsa/dsa_test.cc
index cee783d..dc1956e 100644
--- a/crypto/dsa/dsa_test.cc
+++ b/crypto/dsa/dsa_test.cc
@@ -69,8 +69,9 @@
 #include <openssl/bn.h>
 #include <openssl/crypto.h>
 #include <openssl/err.h>
+#include <openssl/span.h>
 
-#include "../internal.h"
+#include "../test/test_util.h"
 
 
 // The following values are taken from the updated Appendix 5 to FIPS PUB 186
@@ -169,153 +170,79 @@
 };
 
 static bssl::UniquePtr<DSA> GetFIPSDSA(void) {
-  bssl::UniquePtr<DSA>  dsa(DSA_new());
+  bssl::UniquePtr<DSA> dsa(DSA_new());
   if (!dsa) {
     return nullptr;
   }
-  dsa->p = BN_bin2bn(fips_p, sizeof(fips_p), nullptr);
-  dsa->q = BN_bin2bn(fips_q, sizeof(fips_q), nullptr);
-  dsa->g = BN_bin2bn(fips_g, sizeof(fips_g), nullptr);
-  dsa->pub_key = BN_bin2bn(fips_y, sizeof(fips_y), nullptr);
-  dsa->priv_key = BN_bin2bn(fips_x, sizeof(fips_x), nullptr);
-  if (dsa->p == nullptr || dsa->q == nullptr || dsa->g == nullptr ||
-      dsa->pub_key == nullptr || dsa->priv_key == nullptr) {
+  bssl::UniquePtr<BIGNUM> p(BN_bin2bn(fips_p, sizeof(fips_p), nullptr));
+  bssl::UniquePtr<BIGNUM> q(BN_bin2bn(fips_q, sizeof(fips_q), nullptr));
+  bssl::UniquePtr<BIGNUM> g(BN_bin2bn(fips_g, sizeof(fips_g), nullptr));
+  if (!p || !q || !g || !DSA_set0_pqg(dsa.get(), p.get(), q.get(), g.get())) {
     return nullptr;
   }
+  // |DSA_set0_pqg| takes ownership.
+  p.release();
+  q.release();
+  g.release();
+  bssl::UniquePtr<BIGNUM> pub_key(BN_bin2bn(fips_y, sizeof(fips_y), nullptr));
+  bssl::UniquePtr<BIGNUM> priv_key(BN_bin2bn(fips_x, sizeof(fips_x), nullptr));
+  if (!pub_key || !priv_key ||
+      !DSA_set0_key(dsa.get(), pub_key.get(), priv_key.get())) {
+    return nullptr;
+  }
+  // |DSA_set0_key| takes ownership.
+  pub_key.release();
+  priv_key.release();
   return dsa;
 }
 
-struct GenerateContext {
-  FILE *out = nullptr;
-  int ok = 0;
-  int num = 0;
-};
-
-static int GenerateCallback(int p, int n, BN_GENCB *arg) {
-  GenerateContext *ctx = reinterpret_cast<GenerateContext *>(arg->arg);
-  char c = '*';
-  switch (p) {
-    case 0:
-      c = '.';
-      ctx->num++;
-      break;
-    case 1:
-      c = '+';
-      break;
-    case 2:
-      c = '*';
-      ctx->ok++;
-      break;
-    case 3:
-      c = '\n';
-  }
-  fputc(c, ctx->out);
-  fflush(ctx->out);
-  if (!ctx->ok && p == 0 && ctx->num > 1) {
-    fprintf(stderr, "error in dsatest\n");
-    return 0;
-  }
-  return 1;
-}
-
-static int TestGenerate(FILE *out) {
-  BN_GENCB cb;
-  int counter, i, j;
-  uint8_t buf[256];
-  unsigned long h;
-  uint8_t sig[256];
-  unsigned int siglen;
-
-  fprintf(out, "test generation of DSA parameters\n");
-
-  GenerateContext ctx;
-  ctx.out = out;
-  BN_GENCB_set(&cb, GenerateCallback, &ctx);
+TEST(DSATest, Generate) {
   bssl::UniquePtr<DSA> dsa(DSA_new());
-  if (!dsa ||
-      !DSA_generate_parameters_ex(dsa.get(), 512, seed, 20, &counter, &h,
-                                  &cb)) {
-    return false;
-  }
+  ASSERT_TRUE(dsa);
+  int counter;
+  unsigned long h;
+  ASSERT_TRUE(DSA_generate_parameters_ex(dsa.get(), 512, seed, 20, &counter, &h,
+                                         nullptr));
+  EXPECT_EQ(counter, 105);
+  EXPECT_EQ(h, 2u);
 
-  fprintf(out, "seed\n");
-  for (i = 0; i < 20; i += 4) {
-    fprintf(out, "%02X%02X%02X%02X ", seed[i], seed[i + 1], seed[i + 2],
-            seed[i + 3]);
-  }
-  fprintf(out, "\ncounter=%d h=%lu\n", counter, h);
+  auto expect_bn_bytes = [](const char *msg, const BIGNUM *bn,
+                            bssl::Span<const uint8_t> bytes) {
+    std::vector<uint8_t> buf(BN_num_bytes(bn));
+    BN_bn2bin(bn, buf.data());
+    EXPECT_EQ(Bytes(buf), Bytes(bytes)) << msg;
+  };
+  expect_bn_bytes("q value is wrong", DSA_get0_q(dsa.get()), fips_q);
+  expect_bn_bytes("p value is wrong", DSA_get0_p(dsa.get()), fips_p);
+  expect_bn_bytes("g value is wrong", DSA_get0_g(dsa.get()), fips_g);
 
-  if (counter != 105) {
-    fprintf(stderr, "counter should be 105\n");
-    return false;
-  }
-  if (h != 2) {
-    fprintf(stderr, "h should be 2\n");
-    return false;
-  }
+  ASSERT_TRUE(DSA_generate_key(dsa.get()));
 
-  i = BN_bn2bin(dsa->q, buf);
-  j = sizeof(fips_q);
-  if (i != j || OPENSSL_memcmp(buf, fips_q, i) != 0) {
-    fprintf(stderr, "q value is wrong\n");
-    return false;
-  }
+  std::vector<uint8_t> sig(DSA_size(dsa.get()));
+  unsigned sig_len;
+  ASSERT_TRUE(DSA_sign(0, fips_digest, sizeof(fips_digest), sig.data(),
+                       &sig_len, dsa.get()));
 
-  i = BN_bn2bin(dsa->p, buf);
-  j = sizeof(fips_p);
-  if (i != j || OPENSSL_memcmp(buf, fips_p, i) != 0) {
-    fprintf(stderr, "p value is wrong\n");
-    return false;
-  }
-
-  i = BN_bn2bin(dsa->g, buf);
-  j = sizeof(fips_g);
-  if (i != j || OPENSSL_memcmp(buf, fips_g, i) != 0) {
-    fprintf(stderr, "g value is wrong\n");
-    return false;
-  }
-
-  if (!DSA_generate_key(dsa.get()) ||
-      !DSA_sign(0, fips_digest, sizeof(fips_digest), sig, &siglen, dsa.get())) {
-    return false;
-  }
-  if (DSA_verify(0, fips_digest, sizeof(fips_digest), sig, siglen, dsa.get()) !=
-      1) {
-    fprintf(stderr, "verification failure\n");
-    return false;
-  }
-
-  return true;
+  EXPECT_EQ(1, DSA_verify(0, fips_digest, sizeof(fips_digest), sig.data(),
+                          sig_len, dsa.get()));
 }
 
-static bool TestVerify(const uint8_t *sig, size_t sig_len, int expect) {
+TEST(DSATest, Verify) {
   bssl::UniquePtr<DSA> dsa = GetFIPSDSA();
-  if (!dsa) {
-    return false;
-  }
+  ASSERT_TRUE(dsa);
 
-  int ret =
-      DSA_verify(0, fips_digest, sizeof(fips_digest), sig, sig_len, dsa.get());
-  if (ret != expect) {
-    fprintf(stderr, "DSA_verify returned %d, want %d\n", ret, expect);
-    return false;
-  }
-
-  // Clear any errors from a test with expected failure.
-  ERR_clear_error();
-  return true;
-}
-
-// TODO(davidben): Convert this file to GTest properly.
-TEST(DSATest, AllTests) {
-  if (!TestGenerate(stdout) ||
-      !TestVerify(fips_sig, sizeof(fips_sig), 1) ||
-      !TestVerify(fips_sig_negative, sizeof(fips_sig_negative), -1) ||
-      !TestVerify(fips_sig_extra, sizeof(fips_sig_extra), -1) ||
-      !TestVerify(fips_sig_bad_length, sizeof(fips_sig_bad_length), -1) ||
-      !TestVerify(fips_sig_bad_r, sizeof(fips_sig_bad_r), 0)) {
-    ADD_FAILURE() << "Tests failed";
-  }
+  EXPECT_EQ(1, DSA_verify(0, fips_digest, sizeof(fips_digest), fips_sig,
+                          sizeof(fips_sig), dsa.get()));
+  EXPECT_EQ(-1,
+            DSA_verify(0, fips_digest, sizeof(fips_digest), fips_sig_negative,
+                       sizeof(fips_sig_negative), dsa.get()));
+  EXPECT_EQ(-1, DSA_verify(0, fips_digest, sizeof(fips_digest), fips_sig_extra,
+                           sizeof(fips_sig_extra), dsa.get()));
+  EXPECT_EQ(-1,
+            DSA_verify(0, fips_digest, sizeof(fips_digest), fips_sig_bad_length,
+                       sizeof(fips_sig_bad_length), dsa.get()));
+  EXPECT_EQ(0, DSA_verify(0, fips_digest, sizeof(fips_digest), fips_sig_bad_r,
+                          sizeof(fips_sig_bad_r), dsa.get()));
 }
 
 TEST(DSATest, InvalidGroup) {