rust: bssl-tls: Introduce non-X.509 mode

This allows the clients to use a more memory-efficient connection handle
at the cost of implementing its own certificate verifier.

Signed-off-by: Xiangfei Ding <xfding@google.com>
Change-Id: I3a86dd9d975239d3b6807c808328eb936a6a6964
Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/98807
Reviewed-by: Adam Langley <agl@google.com>
diff --git a/rust/bssl-tls/src/connection/methods.rs b/rust/bssl-tls/src/connection/methods.rs
index c22aae3..dc227ab 100644
--- a/rust/bssl-tls/src/connection/methods.rs
+++ b/rust/bssl-tls/src/connection/methods.rs
@@ -35,9 +35,11 @@
     VerifyCertificateMethods,
     abort_on_panic,
     context::{
+        DtlsExternalVerifierMode,
         DtlsMode,
         QuicMode,
-        TlsMode, //
+        TlsExternalVerifierMode, //
+        TlsMode,
     },
     credentials::{
         PrivateKeyDelegate,
@@ -300,63 +302,54 @@
     fn registration() -> c_int;
 }
 
-impl HasTlsConnectionMethod for TlsMode {
-    #[inline(always)]
-    fn registration() -> c_int {
-        static TLS_CONTEXT_METHOD: Lazy<c_int> =
-            Lazy::new(register_tls_connection_vtable::<TlsMode>);
-        *TLS_CONTEXT_METHOD
-    }
+macro_rules! impl_has_tls_connection_method {
+    ($($mode:ty),+ $(,)?) => {
+        $(
+            impl HasTlsConnectionMethod for $mode {
+                #[inline(always)]
+                fn registration() -> c_int {
+                    static TLS_CONTEXT_METHOD: Lazy<c_int> =
+                        Lazy::new(register_tls_connection_vtable::<$mode>);
+                    *TLS_CONTEXT_METHOD
+                }
+            }
+        )+
+    };
 }
 
-impl HasTlsConnectionMethod for DtlsMode {
-    #[inline(always)]
-    fn registration() -> c_int {
-        static TLS_CONTEXT_METHOD: Lazy<c_int> =
-            Lazy::new(register_tls_connection_vtable::<DtlsMode>);
-        *TLS_CONTEXT_METHOD
-    }
-}
-
-impl HasTlsConnectionMethod for QuicMode {
-    #[inline(always)]
-    fn registration() -> c_int {
-        static TLS_CONTEXT_METHOD: Lazy<c_int> =
-            Lazy::new(register_tls_connection_vtable::<QuicMode>);
-        *TLS_CONTEXT_METHOD
-    }
+impl_has_tls_connection_method! {
+    TlsMode,
+    TlsExternalVerifierMode,
+    DtlsMode,
+    DtlsExternalVerifierMode,
+    QuicMode,
 }
 
 pub(super) trait HasPrivateKeyMethods {
     const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD;
 }
 
-impl HasPrivateKeyMethods for TlsMode {
-    const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD = {
-        &bssl_sys::SSL_PRIVATE_KEY_METHOD {
-            sign: Some(sign::<RustConnectionMethods<TlsMode>>),
-            decrypt: Some(decrypt::<RustConnectionMethods<TlsMode>>),
-            complete: Some(complete::<RustConnectionMethods<TlsMode>>),
-        } as _
+macro_rules! impl_private_key_methods {
+    ($wrapper:ident, $($mode:ty),+ $(,)?) => {
+        $(
+            impl HasPrivateKeyMethods for $mode {
+                const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD = {
+                    &bssl_sys::SSL_PRIVATE_KEY_METHOD {
+                        sign: Some(sign::<$wrapper<$mode>>),
+                        decrypt: Some(decrypt::<$wrapper<$mode>>),
+                        complete: Some(complete::<$wrapper<$mode>>),
+                    } as _
+                };
+            }
+        )+
     };
 }
 
-impl HasPrivateKeyMethods for DtlsMode {
-    const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD = {
-        &bssl_sys::SSL_PRIVATE_KEY_METHOD {
-            sign: Some(sign::<RustConnectionMethods<DtlsMode>>),
-            decrypt: Some(decrypt::<RustConnectionMethods<DtlsMode>>),
-            complete: Some(complete::<RustConnectionMethods<DtlsMode>>),
-        } as _
-    };
-}
-
-impl HasPrivateKeyMethods for QuicMode {
-    const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD = {
-        &bssl_sys::SSL_PRIVATE_KEY_METHOD {
-            sign: Some(sign::<RustConnectionMethods<QuicMode>>),
-            decrypt: Some(decrypt::<RustConnectionMethods<QuicMode>>),
-            complete: Some(complete::<RustConnectionMethods<QuicMode>>),
-        } as _
-    };
+impl_private_key_methods! {
+    RustConnectionMethods,
+    TlsMode,
+    DtlsMode,
+    QuicMode,
+    TlsExternalVerifierMode,
+    DtlsExternalVerifierMode,
 }
diff --git a/rust/bssl-tls/src/context.rs b/rust/bssl-tls/src/context.rs
index 29088c6..88c0774 100644
--- a/rust/bssl-tls/src/context.rs
+++ b/rust/bssl-tls/src/context.rs
@@ -62,8 +62,28 @@
 /// QUIC mode
 pub enum QuicMode {}
 
+/// TLS mode without built-in X.509 support.
+///
+/// This mode uses `TLS_with_buffers_method`, which avoids all use of crypto/x509.
+/// All client connections will fail unless a certificate verifier is installed with
+/// [`TlsContextBuilder::with_certificate_verifier`].
+pub enum TlsExternalVerifierMode {}
+
+/// DTLS mode without built-in X.509 support.
+///
+/// This mode uses `DTLS_with_buffers_method`, which avoids all use of crypto/x509.
+/// All client connections will fail unless a certificate verifier is installed with
+/// [`TlsContextBuilder::with_certificate_verifier`].
+pub enum DtlsExternalVerifierMode {}
+
 pub(crate) trait HasBasicIo {}
 
+/// A marker trait for modes that have built-in X.509 support.
+pub trait UseBuiltinX509 {}
+
+impl UseBuiltinX509 for TlsMode {}
+impl UseBuiltinX509 for DtlsMode {}
+
 /// A collection of supported mode of operations.
 pub trait SupportedMode:
     HasTlsContextMethod + HasTlsConnectionMethod + HasPrivateKeyMethods
@@ -73,9 +93,13 @@
 impl SupportedMode for TlsMode {}
 impl SupportedMode for DtlsMode {}
 impl SupportedMode for QuicMode {}
+impl SupportedMode for TlsExternalVerifierMode {}
+impl SupportedMode for DtlsExternalVerifierMode {}
 
 impl HasBasicIo for TlsMode {}
 impl HasBasicIo for DtlsMode {}
+impl HasBasicIo for TlsExternalVerifierMode {}
+impl HasBasicIo for DtlsExternalVerifierMode {}
 
 /// General TLS configuration
 ///
@@ -142,6 +166,34 @@
     }
 }
 
+/// # Make a TLS context builder without built-in X.509 support
+impl TlsContextBuilder<TlsExternalVerifierMode> {
+    /// Creates a new TLS context builder without X.509 support.
+    ///
+    /// All client connections will fail unless a certificate verifier is installed
+    /// with [`TlsContextBuilder::with_certificate_verifier`].
+    pub fn new_tls_no_x509() -> Self {
+        Self::new_inner(unsafe {
+            // Safety: this call returns a static immutable data
+            bssl_sys::TLS_with_buffers_method()
+        })
+    }
+}
+
+/// # Make a DTLS context builder without built-in X.509 support
+impl TlsContextBuilder<DtlsExternalVerifierMode> {
+    /// Creates a new DTLS context builder without X.509 support.
+    ///
+    /// All client connections will fail unless a certificate verifier is installed
+    /// with [`TlsContextBuilder::with_certificate_verifier`].
+    pub fn new_dtls_no_x509() -> Self {
+        Self::new_inner(unsafe {
+            // Safety: this call returns a static immutable data
+            bssl_sys::DTLS_with_buffers_method()
+        })
+    }
+}
+
 /// # Configure the context through a context builder
 impl<M> TlsContextBuilder<M>
 where
diff --git a/rust/bssl-tls/src/context/credentials.rs b/rust/bssl-tls/src/context/credentials.rs
index f9f951b..c304218 100644
--- a/rust/bssl-tls/src/context/credentials.rs
+++ b/rust/bssl-tls/src/context/credentials.rs
@@ -240,8 +240,15 @@
     }
 }
 
-/// # Certificate verification
-impl<M> TlsContextBuilder<M> {
+/// # Certificate verification, X.509
+///
+/// These methods require built-in X.509 support and are not available for
+/// [`TlsNoX509Mode`](super::TlsNoX509Mode) or
+/// [`DtlsNoX509Mode`](super::DtlsNoX509Mode).
+impl<M> TlsContextBuilder<M>
+where
+    M: super::UseBuiltinX509,
+{
     /// Set certificate verification parameters.
     pub fn with_certificate_verification_params(
         &mut self,
@@ -283,7 +290,10 @@
         }
         self
     }
+}
 
+/// # Signature algorithm preferences
+impl<M> TlsContextBuilder<M> {
     /// Set a preference list of signature algorithms for verification.
     ///
     /// This method returns [`ConfigurationError::InvalidParameters`] if the list of algorithms
diff --git a/rust/bssl-tls/src/context/methods.rs b/rust/bssl-tls/src/context/methods.rs
index 49c4669..8bce0a0 100644
--- a/rust/bssl-tls/src/context/methods.rs
+++ b/rust/bssl-tls/src/context/methods.rs
@@ -26,9 +26,11 @@
     PrivateKeyMethods,
     VerifyCertificateMethods,
     context::{
+        DtlsExternalVerifierMode,
         DtlsMode,
         QuicMode,
-        TlsMode, //
+        TlsExternalVerifierMode, //
+        TlsMode,
     },
     credentials::{
         PrivateKeyDelegate,
@@ -119,58 +121,54 @@
     fn registration() -> c_int;
 }
 
-impl HasTlsContextMethod for TlsMode {
-    #[inline(always)]
-    fn registration() -> c_int {
-        static TLS_CONTEXT_METHOD: Lazy<c_int> = Lazy::new(register_tls_context_vtable::<TlsMode>);
-        *TLS_CONTEXT_METHOD
-    }
+macro_rules! impl_has_tls_context_method {
+    ($($mode:ty),+ $(,)?) => {
+        $(
+            impl HasTlsContextMethod for $mode {
+                #[inline(always)]
+                fn registration() -> c_int {
+                    static TLS_CONTEXT_METHOD: Lazy<c_int> =
+                        Lazy::new(register_tls_context_vtable::<$mode>);
+                    *TLS_CONTEXT_METHOD
+                }
+            }
+        )+
+    };
 }
 
-impl HasTlsContextMethod for DtlsMode {
-    #[inline(always)]
-    fn registration() -> c_int {
-        static TLS_CONTEXT_METHOD: Lazy<c_int> = Lazy::new(register_tls_context_vtable::<DtlsMode>);
-        *TLS_CONTEXT_METHOD
-    }
-}
-
-impl HasTlsContextMethod for QuicMode {
-    #[inline(always)]
-    fn registration() -> c_int {
-        static TLS_CONTEXT_METHOD: Lazy<c_int> = Lazy::new(register_tls_context_vtable::<QuicMode>);
-        *TLS_CONTEXT_METHOD
-    }
+impl_has_tls_context_method! {
+    TlsMode,
+    TlsExternalVerifierMode,
+    DtlsMode,
+    DtlsExternalVerifierMode,
+    QuicMode,
 }
 
 pub(super) trait HasPrivateKeyMethods {
     const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD;
 }
 
-impl HasPrivateKeyMethods for TlsMode {
-    const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD = {
-        &bssl_sys::SSL_PRIVATE_KEY_METHOD {
-            sign: Some(sign::<RustContextMethods<TlsMode>>),
-            decrypt: Some(decrypt::<RustContextMethods<TlsMode>>),
-            complete: Some(complete::<RustContextMethods<TlsMode>>),
-        } as _
+macro_rules! impl_private_key_methods {
+    ($wrapper:ident, $($mode:ty),+ $(,)?) => {
+        $(
+            impl HasPrivateKeyMethods for $mode {
+                const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD = {
+                    &bssl_sys::SSL_PRIVATE_KEY_METHOD {
+                        sign: Some(sign::<$wrapper<$mode>>),
+                        decrypt: Some(decrypt::<$wrapper<$mode>>),
+                        complete: Some(complete::<$wrapper<$mode>>),
+                    } as _
+                };
+            }
+        )+
     };
 }
-impl HasPrivateKeyMethods for DtlsMode {
-    const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD = {
-        &bssl_sys::SSL_PRIVATE_KEY_METHOD {
-            sign: Some(sign::<RustContextMethods<DtlsMode>>),
-            decrypt: Some(decrypt::<RustContextMethods<DtlsMode>>),
-            complete: Some(complete::<RustContextMethods<DtlsMode>>),
-        } as _
-    };
-}
-impl HasPrivateKeyMethods for QuicMode {
-    const METHODS: *const bssl_sys::SSL_PRIVATE_KEY_METHOD = {
-        &bssl_sys::SSL_PRIVATE_KEY_METHOD {
-            sign: Some(sign::<RustContextMethods<QuicMode>>),
-            decrypt: Some(decrypt::<RustContextMethods<QuicMode>>),
-            complete: Some(complete::<RustContextMethods<QuicMode>>),
-        } as _
-    };
+
+impl_private_key_methods! {
+    RustContextMethods,
+    TlsMode,
+    TlsExternalVerifierMode,
+    DtlsMode,
+    DtlsExternalVerifierMode,
+    QuicMode,
 }