Pretty-print large INTEGERs and ENUMERATEDs in hex.

This avoids taking quadratic time to pretty-print certificates with
excessively large integer fields. Very large integers aren't any more
readable in decimal than hexadecimal anyway, and the i2s_* functions
will parse either form.

Found by libFuzzer.

Change-Id: Id586cd1b0eef8936d38ff50433ae7c819f0054f3
Reviewed-on: https://boringssl-review.googlesource.com/23424
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/x509/x509_test.cc b/crypto/x509/x509_test.cc
index cd4e61d..b4cecca 100644
--- a/crypto/x509/x509_test.cc
+++ b/crypto/x509/x509_test.cc
@@ -26,6 +26,7 @@
 #include <openssl/pem.h>
 #include <openssl/pool.h>
 #include <openssl/x509.h>
+#include <openssl/x509v3.h>
 
 #include "../internal.h"
 
@@ -996,3 +997,41 @@
               std::string(reinterpret_cast<const char *>(contents), len));
   }
 }
+
+TEST(X509Test, PrettyPrintIntegers) {
+  static const char *kTests[] = {
+      // Small numbers are pretty-printed in decimal.
+      "0",
+      "-1",
+      "1",
+      "42",
+      "-42",
+      "256",
+      "-256",
+      // Large numbers are pretty-printed in hex to avoid taking quadratic time.
+      "0x0123456789",
+      "-0x0123456789",
+  };
+  for (const char *in : kTests) {
+    SCOPED_TRACE(in);
+    BIGNUM *bn = nullptr;
+    ASSERT_TRUE(BN_asc2bn(&bn, in));
+    bssl::UniquePtr<BIGNUM> free_bn(bn);
+
+    {
+      bssl::UniquePtr<ASN1_INTEGER> asn1(BN_to_ASN1_INTEGER(bn, nullptr));
+      ASSERT_TRUE(asn1);
+      bssl::UniquePtr<char> out(i2s_ASN1_INTEGER(nullptr, asn1.get()));
+      ASSERT_TRUE(out.get());
+      EXPECT_STREQ(in, out.get());
+    }
+
+    {
+      bssl::UniquePtr<ASN1_ENUMERATED> asn1(BN_to_ASN1_ENUMERATED(bn, nullptr));
+      ASSERT_TRUE(asn1);
+      bssl::UniquePtr<char> out(i2s_ASN1_ENUMERATED(nullptr, asn1.get()));
+      ASSERT_TRUE(out.get());
+      EXPECT_STREQ(in, out.get());
+    }
+  }
+}
diff --git a/crypto/x509v3/v3_utl.c b/crypto/x509v3/v3_utl.c
index feb3dc6..7d109ee 100644
--- a/crypto/x509v3/v3_utl.c
+++ b/crypto/x509v3/v3_utl.c
@@ -155,6 +155,45 @@
     return 1;
 }
 
+static char *bignum_to_string(const BIGNUM *bn)
+{
+    char *tmp, *ret;
+    size_t len;
+
+    /*
+     * Display large numbers in hex and small numbers in decimal. Converting to
+     * decimal takes quadratic time and is no more useful than hex for large
+     * numbers.
+     */
+    if (BN_num_bits(bn) < 32) {
+        return BN_bn2dec(bn);
+    }
+
+    tmp = BN_bn2hex(bn);
+    if (tmp == NULL) {
+        return NULL;
+    }
+
+    len = strlen(tmp) + 3;
+    ret = OPENSSL_malloc(len);
+    if (ret == NULL) {
+        OPENSSL_PUT_ERROR(X509V3, ERR_R_MALLOC_FAILURE);
+        OPENSSL_free(tmp);
+        return NULL;
+    }
+
+    /* Prepend "0x", but place it after the "-" if negative. */
+    if (tmp[0] == '-') {
+        BUF_strlcpy(ret, "-0x", len);
+        BUF_strlcat(ret, tmp + 1, len);
+    } else {
+        BUF_strlcpy(ret, "0x", len);
+        BUF_strlcat(ret, tmp, len);
+    }
+    OPENSSL_free(tmp);
+    return ret;
+}
+
 char *i2s_ASN1_ENUMERATED(X509V3_EXT_METHOD *method, ASN1_ENUMERATED *a)
 {
     BIGNUM *bntmp = NULL;
@@ -162,7 +201,7 @@
     if (!a)
         return NULL;
     if (!(bntmp = ASN1_ENUMERATED_to_BN(a, NULL)) ||
-        !(strtmp = BN_bn2dec(bntmp)))
+        !(strtmp = bignum_to_string(bntmp)))
         OPENSSL_PUT_ERROR(X509V3, ERR_R_MALLOC_FAILURE);
     BN_free(bntmp);
     return strtmp;
@@ -175,7 +214,7 @@
     if (!a)
         return NULL;
     if (!(bntmp = ASN1_INTEGER_to_BN(a, NULL)) ||
-        !(strtmp = BN_bn2dec(bntmp)))
+        !(strtmp = bignum_to_string(bntmp)))
         OPENSSL_PUT_ERROR(X509V3, ERR_R_MALLOC_FAILURE);
     BN_free(bntmp);
     return strtmp;