commit d9e27021e171d5f8c790b282315a07bde735da97
author Adam Langley <agl@google.com> Thu Nov 12 12:24:35 2015 -0800
committer Adam Langley <agl@google.com> Thu Nov 12 23:52:37 2015 +0000
tree fb3d07a6c460ae6c3992208d1fa24beb16002a2b
parent e7806fd47771476facb71e0a3cc045f4756229a5

Don't encode or decode ∞.

|EC_POINT_point2oct| would encode ∞, which is surprising, and
|EC_POINT_oct2point| would decode ∞, which is insane. This change
removes both behaviours.

Thanks to Brian Smith for pointing it out.

Change-Id: Ia89f257dc429a69b9ea7b7b15f75454ccc9c3bdd
Reviewed-on: https://boringssl-review.googlesource.com/6488
Reviewed-by: Adam Langley <agl@google.com>

crypto/ec/oct.c

diff --git a/crypto/ec/oct.c b/crypto/ec/oct.c
index 4f9a159..e39337d 100644
--- a/crypto/ec/oct.c
+++ b/crypto/ec/oct.c
@@ -90,18 +90,10 @@
   }
 
   if (EC_POINT_is_at_infinity(group, point)) {
-    /* encodes to a single 0 octet */
-    if (buf != NULL) {
-      if (len < 1) {
-        OPENSSL_PUT_ERROR(EC, EC_R_BUFFER_TOO_SMALL);
-        return 0;
-      }
-      buf[0] = 0;
-    }
-    return 1;
+    OPENSSL_PUT_ERROR(EC, EC_R_POINT_AT_INFINITY);
+    goto err;
   }
 
-
   /* ret := required output buffer length */
   field_len = BN_num_bytes(&group->field);
   ret =
@@ -117,7 +109,7 @@
     if (ctx == NULL) {
       ctx = new_ctx = BN_CTX_new();
       if (ctx == NULL) {
-        return 0;
+        goto err;
       }
     }
 
@@ -193,24 +185,12 @@
   form = buf[0];
   y_bit = form & 1;
   form = form & ~1U;
-  if ((form != 0) && (form != POINT_CONVERSION_COMPRESSED) &&
-      (form != POINT_CONVERSION_UNCOMPRESSED)) {
+  if ((form != POINT_CONVERSION_COMPRESSED &&
+       form != POINT_CONVERSION_UNCOMPRESSED) ||
+      (form == POINT_CONVERSION_UNCOMPRESSED && y_bit)) {
     OPENSSL_PUT_ERROR(EC, EC_R_INVALID_ENCODING);
     return 0;
   }
-  if ((form == 0 || form == POINT_CONVERSION_UNCOMPRESSED) && y_bit) {
-    OPENSSL_PUT_ERROR(EC, EC_R_INVALID_ENCODING);
-    return 0;
-  }
-
-  if (form == 0) {
-    if (len != 1) {
-      OPENSSL_PUT_ERROR(EC, EC_R_INVALID_ENCODING);
-      return 0;
-    }
-
-    return EC_POINT_set_to_infinity(group, point);
-  }
 
   field_len = BN_num_bytes(&group->field);
   enc_len =