Base EC_FELEM conversions on bytes rather than BIGNUMs.

It is tricky to create EC_FELEMs right now. This will simplify making them.

Change-Id: Icde518efed61381004e2e5569a45d653af48ca2a
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/40585
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/crypto/fipsmodule/ec/ec_montgomery.c b/crypto/fipsmodule/ec/ec_montgomery.c
index 22251e0..1e281a5 100644
--- a/crypto/fipsmodule/ec/ec_montgomery.c
+++ b/crypto/fipsmodule/ec/ec_montgomery.c
@@ -139,30 +139,21 @@
                               group->mont);
 }
 
-int ec_GFp_mont_bignum_to_felem(const EC_GROUP *group, EC_FELEM *out,
-                                const BIGNUM *in) {
-  if (group->mont == NULL) {
-    OPENSSL_PUT_ERROR(EC, EC_R_NOT_INITIALIZED);
-    return 0;
-  }
-
-  if (!bn_copy_words(out->words, group->field.width, in)) {
-    return 0;
-  }
-  ec_GFp_mont_felem_to_montgomery(group, out, out);
-  return 1;
-}
-
-int ec_GFp_mont_felem_to_bignum(const EC_GROUP *group, BIGNUM *out,
-                                const EC_FELEM *in) {
-  if (group->mont == NULL) {
-    OPENSSL_PUT_ERROR(EC, EC_R_NOT_INITIALIZED);
-    return 0;
-  }
-
+void ec_GFp_mont_felem_to_bytes(const EC_GROUP *group, uint8_t *out,
+                                size_t *out_len, const EC_FELEM *in) {
   EC_FELEM tmp;
   ec_GFp_mont_felem_from_montgomery(group, &tmp, in);
-  return bn_set_words(out, tmp.words, group->field.width);
+  ec_GFp_simple_felem_to_bytes(group, out, out_len, &tmp);
+}
+
+int ec_GFp_mont_felem_from_bytes(const EC_GROUP *group, EC_FELEM *out,
+                                 const uint8_t *in, size_t len) {
+  if (!ec_GFp_simple_felem_from_bytes(group, out, in, len)) {
+    return 0;
+  }
+
+  ec_GFp_mont_felem_to_montgomery(group, out, out);
+  return 1;
 }
 
 static int ec_GFp_mont_point_get_affine_coordinates(const EC_GROUP *group,
@@ -460,8 +451,8 @@
   out->mul_public = ec_GFp_mont_mul_public;
   out->felem_mul = ec_GFp_mont_felem_mul;
   out->felem_sqr = ec_GFp_mont_felem_sqr;
-  out->bignum_to_felem = ec_GFp_mont_bignum_to_felem;
-  out->felem_to_bignum = ec_GFp_mont_felem_to_bignum;
+  out->felem_to_bytes = ec_GFp_mont_felem_to_bytes;
+  out->felem_from_bytes = ec_GFp_mont_felem_from_bytes;
   out->scalar_inv0_montgomery = ec_simple_scalar_inv0_montgomery;
   out->scalar_to_montgomery_inv_vartime =
       ec_simple_scalar_to_montgomery_inv_vartime;
diff --git a/crypto/fipsmodule/ec/felem.c b/crypto/fipsmodule/ec/felem.c
index 9be9f8c..713b6b8 100644
--- a/crypto/fipsmodule/ec/felem.c
+++ b/crypto/fipsmodule/ec/felem.c
@@ -23,15 +23,34 @@
 
 
 int ec_bignum_to_felem(const EC_GROUP *group, EC_FELEM *out, const BIGNUM *in) {
-  if (BN_is_negative(in) || BN_cmp(in, &group->field) >= 0) {
+  uint8_t bytes[EC_MAX_BYTES];
+  size_t len = BN_num_bytes(&group->field);
+  assert(sizeof(bytes) >= len);
+  if (BN_is_negative(in) ||
+      BN_cmp(in, &group->field) >= 0 ||
+      !BN_bn2bin_padded(bytes, len, in)) {
     OPENSSL_PUT_ERROR(EC, EC_R_COORDINATES_OUT_OF_RANGE);
     return 0;
   }
-  return group->meth->bignum_to_felem(group, out, in);
+
+  return ec_felem_from_bytes(group, out, bytes, len);
 }
 
 int ec_felem_to_bignum(const EC_GROUP *group, BIGNUM *out, const EC_FELEM *in) {
-  return group->meth->felem_to_bignum(group, out, in);
+  uint8_t bytes[EC_MAX_BYTES];
+  size_t len;
+  ec_felem_to_bytes(group, bytes, &len, in);
+  return BN_bin2bn(bytes, len, out) != NULL;
+}
+
+void ec_felem_to_bytes(const EC_GROUP *group, uint8_t *out, size_t *out_len,
+                       const EC_FELEM *in) {
+  group->meth->felem_to_bytes(group, out, out_len, in);
+}
+
+int ec_felem_from_bytes(const EC_GROUP *group, EC_FELEM *out, const uint8_t *in,
+                        size_t len) {
+  return group->meth->felem_from_bytes(group, out, in, len);
 }
 
 void ec_felem_neg(const EC_GROUP *group, EC_FELEM *out, const EC_FELEM *a) {
diff --git a/crypto/fipsmodule/ec/internal.h b/crypto/fipsmodule/ec/internal.h
index d076c91..f98b1dd 100644
--- a/crypto/fipsmodule/ec/internal.h
+++ b/crypto/fipsmodule/ec/internal.h
@@ -187,6 +187,17 @@
 // zero on allocation failure.
 int ec_felem_to_bignum(const EC_GROUP *group, BIGNUM *out, const EC_FELEM *in);
 
+// ec_felem_to_bytes serializes |in| as a big-endian bytestring to |out| and
+// sets |*out_len| to the number of bytes written. The number of bytes written
+// is |BN_num_bytes(&group->order)|, which is at most |EC_MAX_BYTES|.
+void ec_felem_to_bytes(const EC_GROUP *group, uint8_t *out, size_t *out_len,
+                       const EC_FELEM *in);
+
+// ec_felem_from_bytes deserializes |in| and stores the resulting field element
+// to |out|. It returns one on success and zero if |in| is invalid.
+int ec_felem_from_bytes(const EC_GROUP *group, EC_FELEM *out, const uint8_t *in,
+                        size_t len);
+
 // ec_felem_neg sets |out| to -|a|.
 void ec_felem_neg(const EC_GROUP *group, EC_FELEM *out, const EC_FELEM *a);
 
@@ -317,10 +328,10 @@
                     const EC_FELEM *b);
   void (*felem_sqr)(const EC_GROUP *, EC_FELEM *r, const EC_FELEM *a);
 
-  int (*bignum_to_felem)(const EC_GROUP *group, EC_FELEM *out,
-                         const BIGNUM *in);
-  int (*felem_to_bignum)(const EC_GROUP *group, BIGNUM *out,
+  void (*felem_to_bytes)(const EC_GROUP *group, uint8_t *out, size_t *out_len,
                          const EC_FELEM *in);
+  int (*felem_from_bytes)(const EC_GROUP *group, EC_FELEM *out,
+                          const uint8_t *in, size_t len);
 
   // scalar_inv0_montgomery implements |ec_scalar_inv0_montgomery|.
   void (*scalar_inv0_montgomery)(const EC_GROUP *group, EC_SCALAR *out,
@@ -445,6 +456,11 @@
 int ec_GFp_simple_cmp_x_coordinate(const EC_GROUP *group, const EC_RAW_POINT *p,
                                    const EC_SCALAR *r);
 
+void ec_GFp_simple_felem_to_bytes(const EC_GROUP *group, uint8_t *out,
+                                  size_t *out_len, const EC_FELEM *in);
+int ec_GFp_simple_felem_from_bytes(const EC_GROUP *group, EC_FELEM *out,
+                                   const uint8_t *in, size_t len);
+
 // method functions in montgomery.c
 int ec_GFp_mont_group_init(EC_GROUP *);
 int ec_GFp_mont_group_set_curve(EC_GROUP *, const BIGNUM *p, const BIGNUM *a,
@@ -454,10 +470,10 @@
                            const EC_FELEM *b);
 void ec_GFp_mont_felem_sqr(const EC_GROUP *, EC_FELEM *r, const EC_FELEM *a);
 
-int ec_GFp_mont_bignum_to_felem(const EC_GROUP *group, EC_FELEM *out,
-                                const BIGNUM *in);
-int ec_GFp_mont_felem_to_bignum(const EC_GROUP *group, BIGNUM *out,
-                                const EC_FELEM *in);
+void ec_GFp_mont_felem_to_bytes(const EC_GROUP *group, uint8_t *out,
+                                size_t *out_len, const EC_FELEM *in);
+int ec_GFp_mont_felem_from_bytes(const EC_GROUP *group, EC_FELEM *out,
+                                 const uint8_t *in, size_t len);
 
 void ec_GFp_nistp_recode_scalar_bits(uint8_t *sign, uint8_t *digit, uint8_t in);
 
diff --git a/crypto/fipsmodule/ec/p224-64.c b/crypto/fipsmodule/ec/p224-64.c
index fa7ede1..c106de0 100644
--- a/crypto/fipsmodule/ec/p224-64.c
+++ b/crypto/fipsmodule/ec/p224-64.c
@@ -1154,16 +1154,6 @@
   p224_felem_to_generic(r, felem);
 }
 
-static int ec_GFp_nistp224_bignum_to_felem(const EC_GROUP *group, EC_FELEM *out,
-                                           const BIGNUM *in) {
-  return bn_copy_words(out->words, group->field.width, in);
-}
-
-static int ec_GFp_nistp224_felem_to_bignum(const EC_GROUP *group, BIGNUM *out,
-                                           const EC_FELEM *in) {
-  return bn_set_words(out, in->words, group->field.width);
-}
-
 DEFINE_METHOD_FUNCTION(EC_METHOD, EC_GFp_nistp224_method) {
   out->group_init = ec_GFp_simple_group_init;
   out->group_finish = ec_GFp_simple_group_finish;
@@ -1177,8 +1167,8 @@
   out->mul_public = ec_GFp_nistp224_point_mul_public;
   out->felem_mul = ec_GFp_nistp224_felem_mul;
   out->felem_sqr = ec_GFp_nistp224_felem_sqr;
-  out->bignum_to_felem = ec_GFp_nistp224_bignum_to_felem;
-  out->felem_to_bignum = ec_GFp_nistp224_felem_to_bignum;
+  out->felem_to_bytes = ec_GFp_simple_felem_to_bytes;
+  out->felem_from_bytes = ec_GFp_simple_felem_from_bytes;
   out->scalar_inv0_montgomery = ec_simple_scalar_inv0_montgomery;
   out->scalar_to_montgomery_inv_vartime =
       ec_simple_scalar_to_montgomery_inv_vartime;
diff --git a/crypto/fipsmodule/ec/p256-x86_64.c b/crypto/fipsmodule/ec/p256-x86_64.c
index 3993ff4..9f5bf87 100644
--- a/crypto/fipsmodule/ec/p256-x86_64.c
+++ b/crypto/fipsmodule/ec/p256-x86_64.c
@@ -640,8 +640,8 @@
   out->mul_public = ecp_nistz256_points_mul_public;
   out->felem_mul = ec_GFp_mont_felem_mul;
   out->felem_sqr = ec_GFp_mont_felem_sqr;
-  out->bignum_to_felem = ec_GFp_mont_bignum_to_felem;
-  out->felem_to_bignum = ec_GFp_mont_felem_to_bignum;
+  out->felem_to_bytes = ec_GFp_mont_felem_to_bytes;
+  out->felem_from_bytes = ec_GFp_mont_felem_from_bytes;
   out->scalar_inv0_montgomery = ecp_nistz256_inv0_mod_ord;
   out->scalar_to_montgomery_inv_vartime =
       ecp_nistz256_scalar_to_montgomery_inv_vartime;
diff --git a/crypto/fipsmodule/ec/simple.c b/crypto/fipsmodule/ec/simple.c
index af91da9..fe8386f 100644
--- a/crypto/fipsmodule/ec/simple.c
+++ b/crypto/fipsmodule/ec/simple.c
@@ -344,3 +344,32 @@
   return ec_get_x_coordinate_as_scalar(group, &x, p) &&
          ec_scalar_equal_vartime(group, &x, r);
 }
+
+void ec_GFp_simple_felem_to_bytes(const EC_GROUP *group, uint8_t *out,
+                                  size_t *out_len, const EC_FELEM *in) {
+  size_t len = BN_num_bytes(&group->field);
+  for (size_t i = 0; i < len; i++) {
+    out[i] = in->bytes[len - 1 - i];
+  }
+  *out_len = len;
+}
+
+int ec_GFp_simple_felem_from_bytes(const EC_GROUP *group, EC_FELEM *out,
+                                   const uint8_t *in, size_t len) {
+  if (len != BN_num_bytes(&group->field)) {
+    OPENSSL_PUT_ERROR(EC, EC_R_DECODE_ERROR);
+    return 0;
+  }
+
+  OPENSSL_memset(out, 0, sizeof(EC_FELEM));
+  for (size_t i = 0; i < len; i++) {
+    out->bytes[i] = in[len - 1 - i];
+  }
+
+  if (!bn_less_than_words(out->words, group->field.d, group->field.width)) {
+    OPENSSL_PUT_ERROR(EC, EC_R_DECODE_ERROR);
+    return 0;
+  }
+
+  return 1;
+}
diff --git a/third_party/fiat/p256.c b/third_party/fiat/p256.c
index dd113c4..4a63ef5 100644
--- a/third_party/fiat/p256.c
+++ b/third_party/fiat/p256.c
@@ -1053,8 +1053,8 @@
   out->mul_public = ec_GFp_nistp256_point_mul_public;
   out->felem_mul = ec_GFp_mont_felem_mul;
   out->felem_sqr = ec_GFp_mont_felem_sqr;
-  out->bignum_to_felem = ec_GFp_mont_bignum_to_felem;
-  out->felem_to_bignum = ec_GFp_mont_felem_to_bignum;
+  out->felem_to_bytes = ec_GFp_mont_felem_to_bytes;
+  out->felem_from_bytes = ec_GFp_mont_felem_from_bytes;
   out->scalar_inv0_montgomery = ec_simple_scalar_inv0_montgomery;
   out->scalar_to_montgomery_inv_vartime =
       ec_simple_scalar_to_montgomery_inv_vartime;