From 1ab20d3cc24a28a4d6cd0e2f62f54f7df0415d0a Mon Sep 17 00:00:00 2001 From: Jack Luo Date: Sat, 1 Aug 2026 19:42:53 -0400 Subject: [PATCH] fix(ci): Run LocalStack-backed Rust tests on self-hosted runners. `tests:rust-all` starts LocalStack with `--publish :4566` and connects to it over loopback. Our self-hosted runners mount the host's Docker socket, so the container `start.py` starts is a sibling on the host, and the port is published in the host's network namespace rather than the runner's, where it is unreachable from the job. The port `get-free-port.py` picks is also probed in the runner's namespace while being reserved in the host's, so it can collide with an unrelated listener or with another runner on the same machine. `clp-rust-checks` therefore targeted `GH_RUNNER_LINUX_LIGHT_X64` (`["ubuntu-24.04"]`, GitHub-hosted) to stay off the self-hosted pool. `start.py` now detects whether it is running inside a container the Docker daemon can resolve (`/.dockerenv` plus `docker inspect $(hostname)`) and, if so, starts LocalStack with `--network container:` and `GATEWAY_LISTEN=0.0.0.0:` instead of publishing. LocalStack then shares the job's network namespace, so loopback reaches it, nothing is published to the host, and the chosen port is probed in the namespace it is bound in. No elevated privileges are needed: `--network container:` is an ordinary network mode requiring only the socket access the runners already have. With that, `clp-rust-checks` moves to `GH_RUNNER_LINUX_HEAVY_X64` (`["self-hosted","x64","ubuntu-noble","docker"]`), which also suits the job better: `tests:rust-all` builds the workspace in release mode. GitHub-hosted runners and dev machines fail the detection and keep the existing publishing behaviour, so `AWS_ENDPOINT_URL` and the developer instructions in `components/log-ingestor/README.md` are unchanged. Address LocalStack by `127.0.0.1` rather than `localhost`: inside a container `localhost` can resolve to `::1` alone, which LocalStack's IPv4 gateway doesn't bind, so the name form fails with a connection refusal indistinguishable from the bug above. `tests/aws_config.rs` already defaulted to `127.0.0.1`; this aligns the taskfile and `create-bucket.py` with it. Also wait for `/_localstack/health` before returning from `start.py`. Nothing waited for readiness previously; `create-bucket.py` happened to paper over it via boto3 retries. Verified against a runner simulated with the host socket mounted: LocalStack was reachable on loopback with no host port published; two such runners ran concurrent LocalStacks on the same port without interfering; and the runner retained DNS and egress with LocalStack in its namespace. Publishing mode was re-verified end to end on the host through `create-bucket.py`. `ruff check`, `ruff format`, and `yamllint --strict` are clean. Co-Authored-By: Claude --- .github/workflows/clp-rust-checks.yaml | 4 +- taskfiles/tests/main.yaml | 2 +- tools/scripts/localstack/create-bucket.py | 4 +- tools/scripts/localstack/start.py | 117 ++++++++++++++++++++-- 4 files changed, 112 insertions(+), 15 deletions(-) diff --git a/.github/workflows/clp-rust-checks.yaml b/.github/workflows/clp-rust-checks.yaml index f0c4942bad..a722355859 100644 --- a/.github/workflows/clp-rust-checks.yaml +++ b/.github/workflows/clp-rust-checks.yaml @@ -28,10 +28,8 @@ concurrency: jobs: rust-checks: - # Stays on GH_RUNNER_LINUX_LIGHT_X64: tests:rust-all uses LocalStack on - # 127.0.0.1, broken on self-hosted. runs-on: >- - ${{ fromJSON(vars.GH_RUNNER_LINUX_LIGHT_X64 || '["ubuntu-24.04"]') }} + ${{ fromJSON(vars.GH_RUNNER_LINUX_HEAVY_X64 || '["ubuntu-24.04"]') }} steps: - uses: "actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd" # v6.0.2 with: diff --git a/taskfiles/tests/main.yaml b/taskfiles/tests/main.yaml index 6b157e9594..153f7ffea2 100644 --- a/taskfiles/tests/main.yaml +++ b/taskfiles/tests/main.yaml @@ -31,7 +31,7 @@ tasks: --port "{{.LOCALSTACK_PORT}}" - |- . "{{.G_RUST_TOOLCHAIN_ENV_FILE}}" - AWS_ENDPOINT_URL="http://localhost:{{.LOCALSTACK_PORT}}" \ + AWS_ENDPOINT_URL="http://127.0.0.1:{{.LOCALSTACK_PORT}}" \ CLP_LOG_INGESTOR_S3_BUCKET="{{.LOG_INGESTOR_BUCKET}}" \ CLP_LOG_INGESTOR_SQS_QUEUE="{{.LOG_INGESTOR_QUEUE}}" \ cargo nextest run --all --all-features --run-ignored all --release diff --git a/tools/scripts/localstack/create-bucket.py b/tools/scripts/localstack/create-bucket.py index 59dd366d31..73dcb17398 100755 --- a/tools/scripts/localstack/create-bucket.py +++ b/tools/scripts/localstack/create-bucket.py @@ -149,7 +149,9 @@ def main() -> int: ) args = parser.parse_args() - localstack_endpoint = f"http://localhost:{args.port}" + # `localhost` can resolve to `::1` alone, which LocalStack doesn't bind, so address it by its + # IPv4 loopback address instead. + localstack_endpoint = f"http://127.0.0.1:{args.port}" logger.info("Using LocalStack endpoint: %s", localstack_endpoint) session = boto3.session.Session( diff --git a/tools/scripts/localstack/start.py b/tools/scripts/localstack/start.py index 1e6793926a..f57e1dde07 100755 --- a/tools/scripts/localstack/start.py +++ b/tools/scripts/localstack/start.py @@ -6,12 +6,29 @@ import argparse import logging +import socket import subprocess import sys +import time +import urllib.request +from http import HTTPStatus +from pathlib import Path # Lock `localstack` image version to 4.14 as a workaround for #2118. _LOCALSTACK_IMAGE: str = "localstack/localstack:4.14" +# Docker creates this marker in every container it starts, so its presence means this script is +# itself running inside a container. +_DOCKER_ENV_MARKER: Path = Path("/.dockerenv") + +# Silence Ruff S607: the absolute path of the Docker binary may vary depending on the installation +# method. +_DOCKER_EXECUTABLE: str = "docker" + +_READINESS_TIMEOUT_SECS: float = 120 +_READINESS_POLL_INTERVAL_SECS: float = 1 +_READINESS_REQUEST_TIMEOUT_SECS: float = 5 + logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(message)s", @@ -20,6 +37,69 @@ logger = logging.getLogger(__name__) +def _get_enclosing_container_id() -> str | None: + """ + Resolves the ID of the container this script is running in, if any. + + On a runner that mounts the host's Docker socket, containers started by this script are siblings + on the host rather than children of the runner, so a published port lands in the host's network + namespace instead of the runner's and is unreachable from here. Joining the runner's network + namespace avoids that, but requires identifying the runner's container. + + :return: The container ID, or None if this script isn't running in a container that the Docker + daemon can resolve. + """ + if not _DOCKER_ENV_MARKER.exists(): + return None + + # Docker defaults a container's hostname to its ID, and Compose sets it to the container's name; + # the daemon resolves either. + hostname = socket.gethostname() + result = subprocess.run( + [_DOCKER_EXECUTABLE, "inspect", "-f", "{{.Id}}", hostname], + capture_output=True, + text=True, + check=False, + ) + if result.returncode != 0: + logger.warning( + "Running inside a container, but the Docker daemon can't resolve it from hostname" + " '%s'. Falling back to publishing a port, which is unreachable from here if the" + " daemon is the host's. Docker error:\n%s", + hostname, + result.stderr.strip(), + ) + return None + + return result.stdout.strip() + + +def _wait_until_ready(port: int) -> bool: + """ + Waits until LocalStack answers on the loopback address. + + :param port: The port LocalStack is expected to listen on. + :return: Whether LocalStack became ready before timing out. + """ + # `localhost` can resolve to `::1` alone, which LocalStack doesn't bind, so address it by its + # IPv4 loopback address instead. + health_url = f"http://127.0.0.1:{port}/_localstack/health" + deadline = time.monotonic() + _READINESS_TIMEOUT_SECS + while time.monotonic() < deadline: + try: + # Silence Ruff S310: `health_url` is built above from a literal `http` scheme. + with urllib.request.urlopen( # noqa: S310 + health_url, timeout=_READINESS_REQUEST_TIMEOUT_SECS + ) as response: + if HTTPStatus.OK == response.status: + return True + except OSError as e: + logger.debug("LocalStack isn't ready yet: %s", e) + time.sleep(_READINESS_POLL_INTERVAL_SECS) + + return False + + def main() -> int: """Main.""" parser = argparse.ArgumentParser(description="Start LocalStack Docker container.") @@ -37,12 +117,8 @@ def main() -> int: ) args = parser.parse_args() - # Silence Ruff S607: the absolute path of the Docker binary may vary depending on the - # installation method. - docker_executable = "docker" - result = subprocess.run( - [docker_executable, "inspect", "-f", "{{.State.Running}}", args.name], + [_DOCKER_EXECUTABLE, "inspect", "-f", "{{.State.Running}}", args.name], capture_output=True, text=True, check=False, @@ -54,7 +130,7 @@ def main() -> int: logger.info("Starting LocalStack container '%s' on port %d", args.name, args.port) logger.info("Pulling LocalStack image.") result = subprocess.run( - [docker_executable, "pull", _LOCALSTACK_IMAGE], capture_output=True, text=True, check=False + [_DOCKER_EXECUTABLE, "pull", _LOCALSTACK_IMAGE], capture_output=True, text=True, check=False ) if result.returncode != 0: logger.error("Failed to pull LocalStack image:\n%s", result.stderr) @@ -62,22 +138,43 @@ def main() -> int: logger.info("Successfully pulled LocalStack image.") localstack_start_cmd = [ - "docker", + _DOCKER_EXECUTABLE, "run", "--rm", "--detach", "--name", args.name, - "--publish", - f"{args.port}:4566", - _LOCALSTACK_IMAGE, ] + enclosing_container_id = _get_enclosing_container_id() + if enclosing_container_id is None: + localstack_start_cmd += ["--publish", f"{args.port}:4566"] + else: + # Publishing a port is incompatible with this network mode, so bind LocalStack's gateway + # directly to `--port` instead. + logger.info("Joining the network namespace of container '%s'.", enclosing_container_id) + localstack_start_cmd += [ + "--network", + f"container:{enclosing_container_id}", + "--env", + f"GATEWAY_LISTEN=0.0.0.0:{args.port}", + ] + localstack_start_cmd.append(_LOCALSTACK_IMAGE) result = subprocess.run(localstack_start_cmd, capture_output=True, text=True, check=False) if result.returncode != 0: logger.error("Failed to start LocalStack container:\n%s", result.stderr) return result.returncode logger.info("LocalStack container started successfully with ID: %s", result.stdout.strip()) + + if not _wait_until_ready(args.port): + logger.error( + "LocalStack didn't become ready on port %d within %g seconds.", + args.port, + _READINESS_TIMEOUT_SECS, + ) + return 1 + logger.info("LocalStack is ready on port %d.", args.port) + return 0