Don't encode or decode ∞. |EC_POINT_point2oct| would encode ∞, which is surprising, and |EC_POINT_oct2point| would decode ∞, which is insane. This change removes both behaviours. Thanks to Brian Smith for pointing it out. Change-Id: Ia89f257dc429a69b9ea7b7b15f75454ccc9c3bdd Reviewed-on: https://boringssl-review.googlesource.com/6488 Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/ec/oct.c b/crypto/ec/oct.c index 4f9a159..e39337d 100644 --- a/crypto/ec/oct.c +++ b/crypto/ec/oct.c
@@ -90,18 +90,10 @@ } if (EC_POINT_is_at_infinity(group, point)) { - /* encodes to a single 0 octet */ - if (buf != NULL) { - if (len < 1) { - OPENSSL_PUT_ERROR(EC, EC_R_BUFFER_TOO_SMALL); - return 0; - } - buf[0] = 0; - } - return 1; + OPENSSL_PUT_ERROR(EC, EC_R_POINT_AT_INFINITY); + goto err; } - /* ret := required output buffer length */ field_len = BN_num_bytes(&group->field); ret = @@ -117,7 +109,7 @@ if (ctx == NULL) { ctx = new_ctx = BN_CTX_new(); if (ctx == NULL) { - return 0; + goto err; } } @@ -193,24 +185,12 @@ form = buf[0]; y_bit = form & 1; form = form & ~1U; - if ((form != 0) && (form != POINT_CONVERSION_COMPRESSED) && - (form != POINT_CONVERSION_UNCOMPRESSED)) { + if ((form != POINT_CONVERSION_COMPRESSED && + form != POINT_CONVERSION_UNCOMPRESSED) || + (form == POINT_CONVERSION_UNCOMPRESSED && y_bit)) { OPENSSL_PUT_ERROR(EC, EC_R_INVALID_ENCODING); return 0; } - if ((form == 0 || form == POINT_CONVERSION_UNCOMPRESSED) && y_bit) { - OPENSSL_PUT_ERROR(EC, EC_R_INVALID_ENCODING); - return 0; - } - - if (form == 0) { - if (len != 1) { - OPENSSL_PUT_ERROR(EC, EC_R_INVALID_ENCODING); - return 0; - } - - return EC_POINT_set_to_infinity(group, point); - } field_len = BN_num_bytes(&group->field); enc_len =