feat(s2n-quic): allow s2n-quic to use s2n-tls on Windows with MinGW

.github/workflows/ci.yml

@@ -343,6 +343,90 @@ jobs:
         run: |
           ${{ matrix.target != 'native' && 'cross' || 'cargo' }} test --workspace ${{ matrix.exclude }} ${{ matrix.target != 'native' && format('--target {0}', matrix.target) || '' }} ${{ matrix.args }}
 
+  # The previous build runs on MSVC. s2n-quic can use s2n-tls, but only on MinGW, so we need a new job to test that.
+  test-windows-gnu:
+    runs-on: windows-latest
+    name: Windows s2n-tls (${{ matrix.sys }})
+    env:
+      # msys2 ships clang, not gcc, but cc-rs defaults to gcc for *-gnu.
+      CC: clang
+      # gnullvm doesn't auto-link winpthreads, so s2n's pthread/clock_gettime/nanosleep symbols
+      # are undefined; -lpthread links it explicitly. Both vars are needed: RUSTFLAGS for the
+      # normal link, RUSTDOCFLAGS for doctests.
+      CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_RUSTFLAGS: -Clink-arg=-lpthread
+      CARGO_TARGET_X86_64_PC_WINDOWS_GNULLVM_RUSTDOCFLAGS: -Clink-arg=-lpthread
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - sys: ucrt64
+            pkg-prefix: mingw-w64-ucrt-x86_64
+            env-path: /ucrt64
+            rust-host: x86_64-pc-windows-gnu
+          - sys: mingw64
+            pkg-prefix: mingw-w64-x86_64
+            env-path: /mingw64
+            rust-host: x86_64-pc-windows-gnu
+          - sys: clang64
+            pkg-prefix: mingw-w64-clang-x86_64
+            env-path: /clang64
+            # clang64 targets gnullvm, which has no host toolchain, so cross-compile to it from
+            # the gnu host. rust-host is the host we install; cargo-target is what we build for.
+            rust-host: x86_64-pc-windows-gnu
+            cargo-target: x86_64-pc-windows-gnullvm
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          lfs: true
+          submodules: true
+
+      - name: Setup MSYS2
+        uses: msys2/setup-msys2@v2
+        with:
+          msystem: ${{ matrix.sys }}
+          update: true
+          install: >-
+            ${{ matrix.pkg-prefix }}-clang
+            ${{ matrix.pkg-prefix }}-clang-libs
+            ${{ matrix.pkg-prefix }}-cmake
+            ${{ matrix.pkg-prefix }}-ninja
+            ${{ matrix.pkg-prefix }}-nasm
+            ${{ matrix.pkg-prefix }}-rustup
+            diffutils
+            jq
+
+      - name: Install Rust toolchain
+        shell: msys2 {0}
+        run: |
+          set -eu
+          # Force the MinGW gnu host; otherwise rustup picks the runner's MSVC host, which can't build the C dependencies.
+          rustup set default-host ${{ matrix.rust-host }}
+          rustup default stable
+          rustup show active-toolchain
+
+          # clang64 has no gnullvm host toolchain, so cross-compile to gnullvm from the gnu host.
+          if [ -n "${{ matrix.cargo-target }}" ]; then
+            rustup target add ${{ matrix.cargo-target }}
+            echo "CARGO_BUILD_TARGET=${{ matrix.cargo-target }}" >> "$GITHUB_ENV"
+            # aws-lc-sys has no prebuilt bindings for gnullvm, so it runs bindgen at build time.
+            # Install bindgen-cli (on PATH) built for the gnullvm environment.
+            cargo install --locked --target ${{ matrix.cargo-target }} \
+              --root "${{ matrix.env-path }}" bindgen-cli
+          fi
+
+      - name: Restore fuzz corpus
+        shell: msys2 {0}
+        run: |
+          find . -name 'corpus.tar.gz' -exec dirname {} ';' | xargs -L 1 bash -c 'cd "$0" && rm -rf corpus && tar xf corpus.tar.gz'
+
+      # Tests the workspace `default-members`, which excludes the `dc/` crates. Crates and tests
+      # that can't build on the MinGW toolchains are gated in their own manifests.
+      - name: Run cargo test
+        shell: msys2 {0}
+        run: |
+          set -eu
+          cargo test
+
   asan:
     runs-on: ubuntu-latest
     steps:
@@ -1036,7 +1120,7 @@ jobs:
   ci-status-report:
     runs-on: ubuntu-latest
     if: ${{ always() }}
-    needs: [env, rustfmt, clippy, udeps, doc, test, asan, fips, miri, no_std, compliance, coverage, crates, examples, recovery-simulations, sims, copyright, s2n-events, generate-events, snapshots, timing, typos, kani, dhat, loom, xdp, dc-wireshark]
+    needs: [env, rustfmt, clippy, udeps, doc, test, test-windows-gnu, asan, fips, miri, no_std, compliance, coverage, crates, examples, recovery-simulations, sims, copyright, s2n-events, generate-events, snapshots, timing, typos, kani, dhat, loom, xdp, dc-wireshark]
     steps:
       - uses: aws-actions/configure-aws-credentials@v6.2.3
         if: github.event_name == 'push' || github.event_name == 'schedule' || github.repository == github.event.pull_request.head.repo.full_name

quic/s2n-quic-tests/Cargo.toml

@@ -14,7 +14,6 @@ publish = false
 bach = "0.1.0"
 bytes = { version = "1", default-features = false }
 futures = { version = "0.3", default-features = false, features = ["std"] }
-quiche = "0.29"
 rand = { version = "0.10", features = ["chacha"] }
 s2n-codec = { path = "../../common/s2n-codec" }
 s2n-quic-core = { path = "../s2n-quic-core", features = ["branch-tracing", "event-tracing", "probe-tracing", "testing"] }
@@ -26,9 +25,9 @@ tracing = { version = "0.1" }
 tracing-subscriber = { version = "0.3", features = ["env-filter"] }
 zerocopy = { version = "0.8", features = ["derive"] }
 
-# s2n-tls is required by ch_callback_server_local_address_test and doesn't build on Windows
-[target.'cfg(not(target_os = "windows"))'.dependencies]
-s2n-tls = "0.3.31"
+# quiche depends on BoringSSL, which builds on unix and on Windows MSVC, but not with the Windows MinGW-family toolchains.
+[target.'cfg(not(all(target_os = "windows", not(target_env = "msvc"))))'.dependencies]
+quiche = "0.29"
 
-[target.'cfg(unix)'.dependencies]
-s2n-quic = { path = "../s2n-quic", features = ["provider-event-tracing", "provider-tls-s2n", "unstable-provider-io-testing", "unstable-provider-dc", "unstable-provider-packet-interceptor", "unstable-provider-random", "unstable-offload-tls", "unstable_client_hello"] }
+[target.'cfg(any(unix, all(target_os = "windows", target_env = "gnu")))'.dependencies]
+s2n-quic = { path = "../s2n-quic", features = ["provider-tls-s2n", "unstable_client_hello"] }

quic/s2n-quic-tests/build.rs

@@ -0,0 +1,36 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Emits cfgs describing which optional test dependencies are available for the
+//! current target, so the tests can be gated on the reason rather than on a
+//! repeated platform matrix.
+//!
+//! These must stay in sync with the corresponding `[target.'cfg(..)'.dependencies]`
+//! sections in `Cargo.toml`; Cargo cannot select dependencies based on a cfg emitted
+//! by a build script.
+
+fn main() {
+    println!("cargo::rustc-check-cfg=cfg(s2n_tls_provider)");
+    println!("cargo::rustc-check-cfg=cfg(boringssl)");
+
+    let target_os = env("CARGO_CFG_TARGET_OS");
+    let target_env = env("CARGO_CFG_TARGET_ENV");
+    let is_unix = std::env::var_os("CARGO_CFG_UNIX").is_some();
+    let is_windows = target_os == "windows";
+
+    // s2n-tls builds on unix and on Windows with the GNU/MinGW toolchain, but not with MSVC.
+    if is_unix || (is_windows && target_env == "gnu") {
+        println!("cargo::rustc-cfg=s2n_tls_provider");
+    }
+
+    // quiche depends on BoringSSL, which builds on unix and on Windows MSVC, but not with the
+    // Windows MinGW-family toolchains.
+    if !is_windows || target_env == "msvc" {
+        println!("cargo::rustc-cfg=boringssl");
+    }
+}
+
+fn env(name: &str) -> String {
+    std::env::var(name)
+        .unwrap_or_else(|_| panic!("build script missing {name:?} environment variable"))
+}

quic/s2n-quic-tests/src/lib.rs

@@ -363,7 +363,13 @@ impl s2n_quic::provider::random::Generator for Random {
     }
 }
 
-#[cfg(not(target_os = "windows"))]
+// mTLS is only wired up for the s2n-tls provider. On the `s2n_tls_provider` targets (unix and
+// Windows GNU/MinGW) `tls::default` resolves to s2n-tls, so gate on that cfg and go through the
+// default provider.
+//
+// TODO: https://github.com/aws/s2n-quic/issues/1726
+// Build the rustls provider with mTLS enabled so these tests can run against either provider.
+#[cfg(s2n_tls_provider)]
 mod mtls {
     use super::*;
     use s2n_quic::provider::tls;
@@ -414,7 +420,8 @@ mod slow_tls {
     }
 }
 
-#[cfg(unix)]
+// Session resumption is only wired up for the s2n-tls provider.
+#[cfg(s2n_tls_provider)]
 pub mod resumption {
     use super::*;
     use s2n_quic::provider::tls::{
@@ -493,7 +500,7 @@ pub mod resumption {
     }
 }
 
-#[cfg(not(target_os = "windows"))]
+#[cfg(s2n_tls_provider)]
 pub use mtls::*;
 
 pub use slow_tls::SlowTlsProvider;

quic/s2n-quic-tests/src/tests.rs

@@ -49,27 +49,27 @@ mod self_test;
 mod skip_packets;
 mod slow_tls;
 mod tls_context;
+// This test uses quiche, which depends on BoringSSL. See the `boringssl` cfg in build.rs.
+#[cfg(boringssl)]
 mod zero_length_cid_client_connection_migration;
 
-// The ClientHelloCallback trait is only available with s2n-tls
-#[cfg(not(target_os = "windows"))]
+// These tests use the s2n-tls provider specifically (the ClientHelloCallback trait, mTLS
+// providers). See the `s2n_tls_provider` cfg in build.rs.
+#[cfg(s2n_tls_provider)]
 mod ch_callback_connection_info;
-// TODO: https://github.com/aws/s2n-quic/issues/1726
-//
-// The rustls tls provider is used on windows and has different
-// build options than s2n-tls. We should build the rustls provider with
-// mTLS enabled and remove the `cfg(target_os("windows"))`.
-#[cfg(not(target_os = "windows"))]
+#[cfg(s2n_tls_provider)]
 mod chain;
-#[cfg(not(target_os = "windows"))]
+#[cfg(s2n_tls_provider)]
 mod client_handshake_confirm;
-#[cfg(not(target_os = "windows"))]
+#[cfg(s2n_tls_provider)]
 mod dc;
-#[cfg(not(target_os = "windows"))]
+#[cfg(s2n_tls_provider)]
 mod dc_connection_close;
-#[cfg(not(target_os = "windows"))]
+// The s2n-tls `fips` feature depends on aws-lc-fips-sys, which can't be built on Windows with the
+// MinGW toolchain. See: https://github.com/aws/aws-lc/issues/3207
+#[cfg(unix)]
 mod fips;
-#[cfg(not(target_os = "windows"))]
+#[cfg(s2n_tls_provider)]
 mod mtls;
 // This test uses real OS sockets, which conflicts with bach's simulated time scope on Windows.
 #[cfg(not(target_os = "windows"))]

quic/s2n-quic-tests/src/tests/buffer_limit.rs

@@ -98,7 +98,10 @@ fn buffer_limit_test() {
 
     // Rustls emits INTERNAL_ERROR and S2N-TLS emits UNEXPECTED_MESSAGE error
     // when the server close the connection due to large Client Hello.
-    let expected_error = if cfg!(target_os = "windows") {
+    //
+    // rustls is only the default TLS provider on Windows with the MSVC toolchain. On unix and on
+    // Windows with the GNU/MinGW toolchain (target_env = "gnu"), s2n-tls is the default.
+    let expected_error = if cfg!(all(target_os = "windows", target_env = "msvc")) {
         TlsError::INTERNAL_ERROR
     } else {
         TlsError::UNEXPECTED_MESSAGE

quic/s2n-quic-tests/src/tests/ch_callback_connection_info.rs

@@ -2,10 +2,13 @@
 // SPDX-License-Identifier: Apache-2.0
 
 use super::*;
-use s2n_quic::provider::tls::ConnectionInfo;
-use s2n_tls::{
-    callbacks::{ClientHelloCallback, ConnectionFuture},
-    error::Error as S2nError,
+use s2n_quic::provider::tls::{
+    s2n_tls::{
+        callbacks::{ClientHelloCallback, ConnectionFuture},
+        connection::Connection,
+        error::Error as S2nError,
+    },
+    ConnectionInfo,
 };
 use std::{
     pin::Pin,
@@ -28,7 +31,7 @@ impl TestClientHelloHandle {
 impl ClientHelloCallback for TestClientHelloHandle {
     fn on_client_hello(
         &self,
-        connection: &mut s2n_tls::connection::Connection,
+        connection: &mut Connection,
     ) -> Result<Option<Pin<Box<dyn ConnectionFuture>>>, S2nError> {
         let connection_info = connection.application_context::<ConnectionInfo>();
 

quic/s2n-quic-tests/src/tests/mtu.rs

@@ -30,7 +30,7 @@ macro_rules! mtu_test {
                 $impl
             }
 
-            #[cfg(not(target_os = "windows"))]
+            #[cfg(s2n_tls_provider)]
             #[test]
             fn mutual_auth() {
                 let $client = build_client_mtls_provider(certificates::MTLS_CA_CERT).unwrap();

quic/s2n-quic-tests/src/tests/offload.rs

@@ -152,7 +152,7 @@ fn failed_tls_handshake() {
 }
 
 #[test]
-#[cfg(unix)]
+#[cfg(s2n_tls_provider)]
 fn mtls() {
     let model = Model::default();
     test(model.clone(), |handle| {
@@ -190,7 +190,7 @@ fn mtls() {
 }
 
 #[test]
-#[cfg(unix)]
+#[cfg(s2n_tls_provider)]
 fn async_client_hello() {
     use futures::{ready, FutureExt};
     use s2n_quic::provider::tls::s2n_tls::{

quic/s2n-quic-tests/src/tests/resumption.rs

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 /// Tests resumption handshake
-#[cfg(unix)]
+#[cfg(s2n_tls_provider)]
 #[test]
 fn resumption_handshake() {
     use super::*;

quic/s2n-quic-tls-default/Cargo.toml

@@ -26,8 +26,8 @@ fips = ["s2n-quic-tls?/fips"]
 [dependencies]
 s2n-quic-tls = { version = "=0.85.0", path = "../s2n-quic-tls", optional = true }
 
-[target.'cfg(unix)'.dependencies]
+[target.'cfg(any(unix, all(target_os = "windows", target_env = "gnu")))'.dependencies]
 s2n-quic-tls = { version = "=0.85.0", path = "../s2n-quic-tls" }
 
-[target.'cfg(not(unix))'.dependencies]
+[target.'cfg(not(any(unix, all(target_os = "windows", target_env = "gnu"))))'.dependencies]
 s2n-quic-rustls = { version = "=0.85.0", path = "../s2n-quic-rustls" }

quic/s2n-quic-tls-default/src/lib.rs

@@ -1,7 +1,11 @@
 // Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
 
-#[cfg(not(unix))]
+// s2n-tls builds on unix and on Windows with the GNU/MinGW toolchain, but not with MSVC. Use it as
+// the default provider wherever it builds; fall back to rustls elsewhere (notably Windows MSVC).
+//
+// Keep in sync with the target-specific dependencies in Cargo.toml.
+#[cfg(not(any(unix, all(target_os = "windows", target_env = "gnu"))))]
 pub use s2n_quic_rustls::*;
-#[cfg(unix)]
+#[cfg(any(unix, all(target_os = "windows", target_env = "gnu")))]
 pub use s2n_quic_tls::*;

quic/s2n-quic-tls/Cargo.toml

@@ -24,7 +24,8 @@ s2n-quic-core = { version = "=0.85.0", path = "../s2n-quic-core", default-featur
 s2n-quic-crypto = { version = "=0.85.0", path = "../s2n-quic-crypto", default-features = false }
 # the `on_key_exchange_group` event relies on an API added in 0.3.15
 # s2n-tls v0.3.31 and beyond allows setting multiple application context to a TLS connection
-s2n-tls = { version = "0.3.31", features = ["quic"] }
+# s2n-tls v0.3.40 and beyond builds on Windows (via the GNU/MinGW toolchain)
+s2n-tls = { version = "0.3.40", features = ["quic"] }
 
 [dev-dependencies]
 checkers = "0.7"

quic/s2n-quic-tls/src/callback.rs

@@ -23,6 +23,32 @@ use s2n_tls::{connection::Connection, error::Fallible, ffi::*};
 /// s2n-tls sends small chunks
 const SEND_BUFFER_CAPACITY: usize = 2048;
 
+/// Signals a "would block" to s2n-tls's C IO layer by setting the CRT `errno`
+/// to `EWOULDBLOCK`. `s2n_io.c` reads `errno` to distinguish a retriable blocked
+/// read from a fatal IO error.
+fn set_recv_would_block() {
+    // On non-Windows targets the `errno` crate writes the CRT `errno`, which is
+    // what s2n-tls reads.
+    #[cfg(not(target_os = "windows"))]
+    errno::set_errno(errno::Errno(libc::EWOULDBLOCK));
+
+    // On Windows the `errno` crate writes the Win32 last-error (via
+    // `SetLastError`), not the CRT `errno` that s2n-tls reads, so set the CRT
+    // `errno` directly. s2n-tls and this crate share one statically linked CRT,
+    // so `_set_errno` and the `errno` s2n-tls reads hit the same thread-local
+    // variable.
+    #[cfg(target_os = "windows")]
+    {
+        extern "C" {
+            fn _set_errno(value: core::ffi::c_int) -> core::ffi::c_int;
+        }
+        // SAFETY: `_set_errno` only writes the thread-local CRT errno.
+        unsafe {
+            let _ = _set_errno(libc::EWOULDBLOCK);
+        }
+    }
+}
+
 /// Handles all callback contexts for each session
 pub struct Callback<'a, T, C> {
     pub context: &'a mut T,
@@ -349,7 +375,7 @@ where
             0 => {
                 // https://github.com/aws/s2n-tls/blob/main/docs/USAGE-GUIDE.md#s2n_connection_set_send_cb
                 // s2n-tls wants us to set the global errno to signal blocked
-                errno::set_errno(errno::Errno(libc::EWOULDBLOCK));
+                set_recv_would_block();
                 -1
             }
             len => len as _,

quic/s2n-quic-tls/src/lib.rs

@@ -4,7 +4,9 @@
 use s2n_quic_core::application::ServerName;
 
 /// Ensure memory is correctly managed in tests
-#[cfg(test)]
+/// `checkers` doesn't work well on Windows. It's only used for these
+/// allocation-sanity checks, so we gate it to non-Windows targets.
+#[cfg(all(test, not(target_os = "windows")))]
 #[global_allocator]
 static ALLOCATOR: checkers::Allocator = checkers::Allocator::system();