From f99b038a8264fdecb04c2ca5567f7e1832d27558 Mon Sep 17 00:00:00 2001 From: "sm.wu" Date: Wed, 24 Jun 2026 11:26:23 +0800 Subject: [PATCH] avoid token unset and replace with short mint token --- ci/gpu-runner/README.md | 16 +++++++++----- ci/gpu-runner/entrypoint.sh | 31 ++++++++++++-------------- ci/gpu-runner/runner.env.example | 3 ++- ci/gpu-runner/start-runner.sh | 38 +++++++++++++++++++++++++++++--- 4 files changed, 61 insertions(+), 27 deletions(-) diff --git a/ci/gpu-runner/README.md b/ci/gpu-runner/README.md index 0558e6478..c6cdcc2f1 100644 --- a/ci/gpu-runner/README.md +++ b/ci/gpu-runner/README.md @@ -5,8 +5,9 @@ Docker image and kept alive by a cron watchdog. - **Ephemeral**: the runner exits cleanly after each job, so no state leaks between CI runs. The watchdog immediately brings up a fresh one. -- **Self-healing tokens**: the container mints a fresh registration token from - the GitHub API on every start, so restarts never fail on an expired token. +- **Self-healing tokens**: the host mints a fresh registration token from the + GitHub API on every start, so restarts never fail on an expired token. The + long-lived PAT is never passed into the runner container. - **Cron watchdog**: restarts the container if it stops *or* if the GPU becomes unreachable inside it (your stated failure mode). @@ -153,15 +154,18 @@ The watchdog checks every minute and restarts on stop or GPU-unreachable. | file | purpose | |-----------------------|----------------------------------------------------------------| | `Dockerfile` | CUDA-devel image + actions runner + Rust toolchain + git-lfs | -| `entrypoint.sh` | mint token → configure ephemeral runner → run → de-register | -| `start-runner.sh` | (re)launch the container with `--gpus all`; idempotent | +| `entrypoint.sh` | configure ephemeral runner → run → de-register | +| `start-runner.sh` | mint token → launch container with `--gpus all`; idempotent | | `watchdog.sh` | cron health check + restart | -| `runner.env.example` | template for `runner.env` (PAT + repo URL; gitignored) | +| `runner.env.example` | host-only template for `runner.env` (PAT + repo URL; gitignored) | ## Notes & gotchas - **PAT scope**: repo-level runner needs `repo` (classic) or fine-grained - "Administration: Read and write" on this repo. + "Administration: Read and write" on this repo. `start-runner.sh` uses the PAT + on the host only to mint a short-lived runner registration token, then starts + the container with that short-lived token mounted from `/run/runner-registration-token`. + The long-lived PAT is never present in the container environment. - **Warm builds across ephemeral restarts**: two named volumes persist between containers — `ceno-gpu-runner-cargo` (the cargo registry, so deps aren't re-downloaded) and `ceno-gpu-runner-target` (mounted at `/cache/target`, with diff --git a/ci/gpu-runner/entrypoint.sh b/ci/gpu-runner/entrypoint.sh index cf04853b4..5d6c7d831 100755 --- a/ci/gpu-runner/entrypoint.sh +++ b/ci/gpu-runner/entrypoint.sh @@ -3,41 +3,38 @@ # # Differs from the ceno-reth-benchmark reference entrypoint in one way: instead # of taking a hard-coded RUNNER_TOKEN (which GitHub expires after ~1h and would -# make an auto-restart fail), it mints a FRESH registration token from the -# GitHub API on every start using a stored PAT. That is what lets the cron -# watchdog restart this container indefinitely. It also runs an EPHEMERAL runner -# (exits cleanly after one job) so no state leaks between CI runs. +# make an auto-restart fail), start-runner.sh mints a FRESH registration token +# from the GitHub API on every start. The long-lived PAT stays on the host; this +# container receives only the short-lived registration token. That is what lets +# the cron watchdog restart this container indefinitely. It also runs an +# EPHEMERAL runner (exits cleanly after one job) so no state leaks between CI +# runs. # # Required env: -# GITHUB_PAT - classic PAT with `repo` scope, or fine-grained token with -# "Administration: read/write" on the repo. # REPO_URL - e.g. https://github.com/scroll-tech/ceno # Optional env: # RUNNER_NAME - defaults to gpu- # RUNNER_LABELS- defaults to "self-hosted,Linux,X64,gpu" set -euo pipefail -: "${GITHUB_PAT:?set GITHUB_PAT}" : "${REPO_URL:?set REPO_URL, e.g. https://github.com/scroll-tech/ceno}" RUNNER_NAME="${RUNNER_NAME:-gpu-$(hostname | cut -c1-12)}" RUNNER_LABELS="${RUNNER_LABELS:-self-hosted,Linux,X64,gpu}" +TOKEN_FILE="/run/runner-registration-token/token" RUNNER_DIR="/home/docker/actions-runner" cd "${RUNNER_DIR}" -# REPO_URL -> owner/repo -REPO_PATH="$(echo "${REPO_URL}" | sed -E 's#https?://[^/]+/##; s#\.git$##')" -API="https://api.github.com/repos/${REPO_PATH}/actions/runners" +if [[ ! -f "${TOKEN_FILE}" ]]; then + echo "[entrypoint] ERROR: registration token file not found at ${TOKEN_FILE}" >&2 + exit 1 +fi -echo "[entrypoint] requesting registration token for ${REPO_PATH} ..." -REG_TOKEN="$(curl -fsSL -X POST \ - -H "Authorization: Bearer ${GITHUB_PAT}" \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "${API}/registration-token" | jq -r .token)" +REG_TOKEN="$(<"${TOKEN_FILE}")" +rm -f "${TOKEN_FILE}" || true if [[ -z "${REG_TOKEN}" || "${REG_TOKEN}" == "null" ]]; then - echo "[entrypoint] ERROR: could not obtain registration token (check PAT scope/REPO_URL)" >&2 + echo "[entrypoint] ERROR: registration token file was empty" >&2 exit 1 fi diff --git a/ci/gpu-runner/runner.env.example b/ci/gpu-runner/runner.env.example index 65c22a1db..e2fed9fe7 100644 --- a/ci/gpu-runner/runner.env.example +++ b/ci/gpu-runner/runner.env.example @@ -1,4 +1,5 @@ -# Copy to `runner.env` and fill in. runner.env is gitignored — never commit the PAT. +# Copy to `runner.env` and fill in. runner.env is gitignored — never commit it. +# start-runner.sh reads this file on the host; it is not passed to docker. # # PAT needs (repo-level runner): # - classic PAT: `repo` scope, OR diff --git a/ci/gpu-runner/start-runner.sh b/ci/gpu-runner/start-runner.sh index 0652df7ad..b839e777e 100755 --- a/ci/gpu-runner/start-runner.sh +++ b/ci/gpu-runner/start-runner.sh @@ -16,6 +16,14 @@ if [[ ! -f "${ENV_FILE}" ]]; then exit 1 fi +# Keep runner.env host-only. It contains the long-lived PAT used to mint a +# short-lived runner registration token; do not pass it to docker with --env-file. +# shellcheck source=/dev/null +source "${ENV_FILE}" + +: "${GITHUB_PAT:?set GITHUB_PAT in ${ENV_FILE}}" +: "${REPO_URL:?set REPO_URL in ${ENV_FILE}}" + # Build the image if it's missing (first run / after a host reboot+prune). # No secrets at build time — the SSH deploy key is injected per-job from GitHub # secrets via ssh-agent in the workflow. @@ -27,15 +35,39 @@ fi # Drop any previous instance (exited ephemeral runner, crashed container, etc.). docker rm -f "${CONTAINER_NAME}" >/dev/null 2>&1 || true +REPO_PATH="$(echo "${REPO_URL}" | sed -E 's#https?://[^/]+/##; s#\.git$##')" +API="https://api.github.com/repos/${REPO_PATH}/actions/runners" +TOKEN_DIR="${RUNNER_TOKEN_DIR:-${TMPDIR:-/tmp}/${CONTAINER_NAME}-registration-token}" +TOKEN_FILE="${TOKEN_DIR}/token" + +echo "[start] requesting registration token for ${REPO_PATH} ..." +REG_TOKEN="$(curl -fsSL -X POST \ + -H "Authorization: Bearer ${GITHUB_PAT}" \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${API}/registration-token" | jq -er .token)" + +rm -rf "${TOKEN_DIR}" +mkdir -p "${TOKEN_DIR}" +chmod 755 "${TOKEN_DIR}" +printf '%s' "${REG_TOKEN}" > "${TOKEN_FILE}" +chmod 644 "${TOKEN_FILE}" + echo "[start] launching ${CONTAINER_NAME} ..." -docker run -d \ +if ! docker run -d \ --name "${CONTAINER_NAME}" \ --gpus all \ - --env-file "${ENV_FILE}" \ --restart no \ + -e REPO_URL="${REPO_URL}" \ + -e RUNNER_NAME="${RUNNER_NAME:-}" \ + -e RUNNER_LABELS="${RUNNER_LABELS:-self-hosted,Linux,X64,gpu}" \ -e CARGO_TARGET_DIR=/cache/target \ + -v "${TOKEN_DIR}:/run/runner-registration-token:rw" \ -v ceno-gpu-runner-cargo:/home/docker/.cargo/registry \ -v ceno-gpu-runner-target:/cache/target \ - "${IMAGE_NAME}" + "${IMAGE_NAME}"; then + rm -rf "${TOKEN_DIR}" + exit 1 +fi echo "[start] done. logs: docker logs -f ${CONTAINER_NAME}"