Account for the MTU BIO_ctrls returning negative or overly large numbers.

BIO_ctrls do not have terribly well-defined return values on error. (Though the
existing ones seem to all return 0, not -1, on nonexistant operation.)

Change-Id: I08497f023ce3257c253aa71517a98b2fe73c3f74
Reviewed-on: https://boringssl-review.googlesource.com/2829
Reviewed-by: Adam Langley <agl@google.com>

ssl/d1_both.c

diff --git a/ssl/d1_both.c b/ssl/d1_both.c
index 156c38e..2604466 100644
--- a/ssl/d1_both.c
+++ b/ssl/d1_both.c
@@ -251,11 +251,10 @@
   /* AHA!  Figure out the MTU, and stick to the right size */
   if (s->d1->mtu < dtls1_min_mtu() &&
       !(SSL_get_options(s) & SSL_OP_NO_QUERY_MTU)) {
-    s->d1->mtu = BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_QUERY_MTU, 0, NULL);
-
-    /* I've seen the kernel return bogus numbers when it doesn't know
-     * (initial write), so just make sure we have a reasonable number */
-    if (s->d1->mtu < dtls1_min_mtu()) {
+    long mtu = BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_QUERY_MTU, 0, NULL);
+    if (mtu >= 0 && mtu <= (1 << 30) && (unsigned)mtu >= dtls1_min_mtu()) {
+      s->d1->mtu = (unsigned)mtu;
+    } else {
       s->d1->mtu = kDefaultMTU;
       BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_SET_MTU, s->d1->mtu, NULL);
     }

ssl/d1_lib.c

diff --git a/ssl/d1_lib.c b/ssl/d1_lib.c
index d08b6bd..8244cb9 100644
--- a/ssl/d1_lib.c
+++ b/ssl/d1_lib.c
@@ -56,6 +56,7 @@
 
 #include <openssl/base.h>
 
+#include <limits.h>
 #include <stdio.h>
 
 #if defined(OPENSSL_WINDOWS)
@@ -358,8 +359,11 @@
   /* Reduce MTU after 2 unsuccessful retransmissions */
   if (s->d1->timeout.num_alerts > 2 &&
       !(SSL_get_options(s) & SSL_OP_NO_QUERY_MTU)) {
-    s->d1->mtu =
-        BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_GET_FALLBACK_MTU, 0, NULL);
+    long mtu = BIO_ctrl(SSL_get_wbio(s), BIO_CTRL_DGRAM_GET_FALLBACK_MTU, 0,
+                        NULL);
+    if (mtu >= 0 && mtu <= (1 << 30) && (unsigned)mtu >= dtls1_min_mtu()) {
+      s->d1->mtu = (unsigned)mtu;
+    }
   }
 
   if (s->d1->timeout.num_alerts > DTLS1_TMO_ALERT_COUNT) {