size_t SSL*_use_*_ASN1.

So long as we're not getting rid of them (the certificate variants may
be useful when we decouple from crypto/x509 anyway), get the types and
bounds checks right.

Also reject trailing data and require the input be a single element.
Note: this is a slight compatibility risk, but we did it for
SSL*_use_RSAPrivateKey_ASN1 previously and I think it's probably worth
seeing if anything breaks here.

Change-Id: I64fa3fc6249021ccf59584d68e56ff424a190082
Reviewed-on: https://boringssl-review.googlesource.com/6490
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/include/openssl/ssl.h b/include/openssl/ssl.h
index 29adcb3..e5ab007 100644
--- a/include/openssl/ssl.h
+++ b/include/openssl/ssl.h
@@ -877,15 +877,16 @@
  * input DER-encoded structures. They return one on success and zero on
  * failure. */
 
-OPENSSL_EXPORT int SSL_CTX_use_certificate_ASN1(SSL_CTX *ctx, int len,
-                                                const uint8_t *d);
+OPENSSL_EXPORT int SSL_CTX_use_certificate_ASN1(SSL_CTX *ctx, size_t der_len,
+                                                const uint8_t *der);
 OPENSSL_EXPORT int SSL_use_certificate_ASN1(SSL *ssl, const uint8_t *der,
-                                            int len);
+                                            size_t der_len);
 
 OPENSSL_EXPORT int SSL_CTX_use_PrivateKey_ASN1(int pk, SSL_CTX *ctx,
-                                               const uint8_t *d, long len);
+                                               const uint8_t *der,
+                                               size_t der_len);
 OPENSSL_EXPORT int SSL_use_PrivateKey_ASN1(int type, SSL *ssl,
-                                           const uint8_t *d, long len);
+                                           const uint8_t *der, size_t der_len);
 
 OPENSSL_EXPORT int SSL_CTX_use_RSAPrivateKey_ASN1(SSL_CTX *ctx,
                                                   const uint8_t *der,
diff --git a/ssl/ssl_rsa.c b/ssl/ssl_rsa.c
index 512a41f..b6ae370 100644
--- a/ssl/ssl_rsa.c
+++ b/ssl/ssl_rsa.c
@@ -56,6 +56,8 @@
 
 #include <openssl/ssl.h>
 
+#include <limits.h>
+
 #include <openssl/err.h>
 #include <openssl/evp.h>
 #include <openssl/mem.h>
@@ -79,18 +81,22 @@
   return ssl_set_cert(ssl->cert, x);
 }
 
-int SSL_use_certificate_ASN1(SSL *ssl, const uint8_t *d, int len) {
-  X509 *x;
-  int ret;
-
-  x = d2i_X509(NULL, &d, (long)len);
-  if (x == NULL) {
-    OPENSSL_PUT_ERROR(SSL, ERR_R_ASN1_LIB);
+int SSL_use_certificate_ASN1(SSL *ssl, const uint8_t *der, size_t der_len) {
+  if (der_len > LONG_MAX) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_OVERFLOW);
     return 0;
   }
 
-  ret = SSL_use_certificate(ssl, x);
-  X509_free(x);
+  const uint8_t *p = der;
+  X509 *x509 = d2i_X509(NULL, &p, (long)der_len);
+  if (x509 == NULL || p != der + der_len) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_ASN1_LIB);
+    X509_free(x509);
+    return 0;
+  }
+
+  int ret = SSL_use_certificate(ssl, x509);
+  X509_free(x509);
   return ret;
 }
 
@@ -165,19 +171,22 @@
   return ret;
 }
 
-int SSL_use_PrivateKey_ASN1(int type, SSL *ssl, const uint8_t *d, long len) {
-  int ret;
-  const uint8_t *p;
-  EVP_PKEY *pkey;
-
-  p = d;
-  pkey = d2i_PrivateKey(type, NULL, &p, (long)len);
-  if (pkey == NULL) {
-    OPENSSL_PUT_ERROR(SSL, ERR_R_ASN1_LIB);
+int SSL_use_PrivateKey_ASN1(int type, SSL *ssl, const uint8_t *der,
+                            size_t der_len) {
+  if (der_len > LONG_MAX) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_OVERFLOW);
     return 0;
   }
 
-  ret = SSL_use_PrivateKey(ssl, pkey);
+  const uint8_t *p = der;
+  EVP_PKEY *pkey = d2i_PrivateKey(type, NULL, &p, (long)der_len);
+  if (pkey == NULL || p != der + der_len) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_ASN1_LIB);
+    EVP_PKEY_free(pkey);
+    return 0;
+  }
+
+  int ret = SSL_use_PrivateKey(ssl, pkey);
   EVP_PKEY_free(pkey);
   return ret;
 }
@@ -227,18 +236,23 @@
   return 1;
 }
 
-int SSL_CTX_use_certificate_ASN1(SSL_CTX *ctx, int len, const uint8_t *d) {
-  X509 *x;
-  int ret;
-
-  x = d2i_X509(NULL, &d, (long)len);
-  if (x == NULL) {
-    OPENSSL_PUT_ERROR(SSL, ERR_R_ASN1_LIB);
+int SSL_CTX_use_certificate_ASN1(SSL_CTX *ctx, size_t der_len,
+                                 const uint8_t *der) {
+  if (der_len > LONG_MAX) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_OVERFLOW);
     return 0;
   }
 
-  ret = SSL_CTX_use_certificate(ctx, x);
-  X509_free(x);
+  const uint8_t *p = der;
+  X509 *x509 = d2i_X509(NULL, &p, (long)der_len);
+  if (x509 == NULL || p != der + der_len) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_ASN1_LIB);
+    X509_free(x509);
+    return 0;
+  }
+
+  int ret = SSL_CTX_use_certificate(ctx, x509);
+  X509_free(x509);
   return ret;
 }
 
@@ -287,20 +301,22 @@
   return ssl_set_pkey(ctx->cert, pkey);
 }
 
-int SSL_CTX_use_PrivateKey_ASN1(int type, SSL_CTX *ctx, const uint8_t *d,
-                                long len) {
-  int ret;
-  const uint8_t *p;
-  EVP_PKEY *pkey;
-
-  p = d;
-  pkey = d2i_PrivateKey(type, NULL, &p, (long)len);
-  if (pkey == NULL) {
-    OPENSSL_PUT_ERROR(SSL, ERR_R_ASN1_LIB);
+int SSL_CTX_use_PrivateKey_ASN1(int type, SSL_CTX *ctx, const uint8_t *der,
+                                size_t der_len) {
+  if (der_len > LONG_MAX) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_OVERFLOW);
     return 0;
   }
 
-  ret = SSL_CTX_use_PrivateKey(ctx, pkey);
+  const uint8_t *p = der;
+  EVP_PKEY *pkey = d2i_PrivateKey(type, NULL, &p, (long)der_len);
+  if (pkey == NULL || p != der + der_len) {
+    OPENSSL_PUT_ERROR(SSL, ERR_R_ASN1_LIB);
+    EVP_PKEY_free(pkey);
+    return 0;
+  }
+
+  int ret = SSL_CTX_use_PrivateKey(ctx, pkey);
   EVP_PKEY_free(pkey);
   return ret;
 }