-
Notifications
You must be signed in to change notification settings - Fork 1
feat(packaging): Add a busybox init-container installer image as a fourth distribution channel. #36
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 3 commits
124b2b1
fac30a9
65737b0
b57f9ec
5ae71ac
192de1c
2b50e41
801b5fb
6031345
4ec3ed2
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,8 @@ | |
| # resolve-version use the input version or derive it from pom.xml | ||
| # build-dependency-image ensure the hash-tagged build-env image exists | ||
| # build build .deb / .rpm / .tar.gz artifacts per architecture | ||
| # image build the busybox installer image per arch (push on main/tags) | ||
| # manifest combine per-arch images into a multi-arch tag (main/tags only) | ||
| # | ||
| # See tools/build-packages/README.md for package-build details. | ||
|
|
||
|
|
@@ -58,6 +60,7 @@ jobs: | |
| runs-on: "ubuntu-24.04" | ||
| outputs: | ||
| version: "${{steps.version.outputs.version}}" | ||
| publish: "${{steps.publish.outputs.publish}}" | ||
| steps: | ||
| - name: "Check out repository" | ||
| uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 | ||
|
|
@@ -88,6 +91,19 @@ jobs: | |
| echo "version=${version}" >> "$GITHUB_OUTPUT" | ||
| echo "::notice::Package version: ${version}" | ||
|
|
||
| - name: "Decide whether to publish images" | ||
| id: "publish" | ||
| env: | ||
| DEFAULT_BRANCH: "${{github.event.repository.default_branch}}" | ||
| run: |- | ||
| # Publish images only from the default branch and version tags so feature | ||
| # branches don't overwrite shared tags. | ||
| if [[ ${GITHUB_REF_NAME} == "${DEFAULT_BRANCH}" || ${GITHUB_REF_TYPE} == tag ]]; then | ||
| echo "publish=true" >> "$GITHUB_OUTPUT" | ||
| else | ||
| echo "publish=false" >> "$GITHUB_OUTPUT" | ||
| fi | ||
|
|
||
| build-dependency-image: | ||
| uses: "./.github/workflows/build-dependency-image.yaml" | ||
| permissions: | ||
|
|
@@ -182,3 +198,96 @@ jobs: | |
| path: "packages/${{steps.packages.outputs.tar_gz_filename}}" | ||
| if-no-files-found: "error" | ||
| retention-days: 14 | ||
|
|
||
| # Build the busybox installer image per architecture from the tarball artifact. This runs | ||
| # on the native runner (not inside the build-env container) so docker/buildx is available. | ||
| image: | ||
| needs: ["build", "resolve-version"] | ||
| permissions: | ||
| contents: "read" | ||
| packages: "write" | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| include: | ||
| - arch: "amd64" | ||
| runner: "ubuntu-24.04" | ||
| - arch: "arm64" | ||
| runner: "ubuntu-24.04-arm" | ||
| runs-on: "${{matrix.runner}}" | ||
| timeout-minutes: 30 | ||
| env: | ||
| ARCH: "${{matrix.arch}}" | ||
| steps: | ||
| - name: "Check out repository" | ||
| uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0 | ||
| with: | ||
| persist-credentials: false | ||
|
|
||
| - name: "Download tarball artifact" | ||
| uses: "actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53" # v6.0.0 | ||
| with: | ||
| pattern: "*-linux-${{matrix.arch}}.tar.gz" | ||
| path: "dist" | ||
| merge-multiple: true | ||
|
|
||
| - name: "Log in to GHCR" | ||
| if: "${{needs.resolve-version.outputs.publish == 'true'}}" | ||
| uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0 | ||
| with: | ||
| registry: "ghcr.io" | ||
| username: "${{github.actor}}" | ||
| password: "${{secrets.GITHUB_TOKEN}}" | ||
|
|
||
| - name: "Set up Docker Buildx" | ||
| uses: "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c" # v4.2.0 | ||
|
|
||
| - name: "Build the installer image (and push when publishing)" | ||
| env: | ||
| PUBLISH: "${{needs.resolve-version.outputs.publish}}" | ||
| run: |- | ||
| image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" | ||
| shopt -s nullglob | ||
| tarballs=(dist/*-linux-"${ARCH}".tar.gz) | ||
| if (( ${#tarballs[@]} != 1 )); then | ||
| echo "::error::Expected exactly one tarball for ${ARCH}, found ${#tarballs[@]}" | ||
| exit 1 | ||
| fi | ||
| # Push from the default branch / tags; otherwise just build to validate. | ||
| output="--load" | ||
| if [[ "${PUBLISH}" == "true" ]]; then | ||
| output="--push" | ||
| fi | ||
| bash tools/build-packages/build-installer-init-image.sh \ | ||
| --tarball "${tarballs[0]}" \ | ||
| --arch "${ARCH}" \ | ||
| --repo "${image_repo}" \ | ||
| "${output}" | ||
|
|
||
| # Combine the per-architecture installer images into one multi-arch version tag. | ||
| manifest: | ||
| needs: ["image", "resolve-version"] | ||
| if: "${{needs.resolve-version.outputs.publish == 'true'}}" | ||
| permissions: | ||
| packages: "write" | ||
| runs-on: "ubuntu-24.04" | ||
| timeout-minutes: 15 | ||
| env: | ||
| VERSION: "${{needs.resolve-version.outputs.version}}" | ||
| steps: | ||
| - name: "Log in to GHCR" | ||
| uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0 | ||
| with: | ||
| registry: "ghcr.io" | ||
| username: "${{github.actor}}" | ||
| password: "${{secrets.GITHUB_TOKEN}}" | ||
|
|
||
| - name: "Create multi-arch manifest" | ||
| run: |- | ||
| image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')" | ||
| # Docker tags allow only [A-Za-z0-9_.-]; sanitize to match the build script. | ||
| tag_version="${VERSION//[^A-Za-z0-9_.-]/_}" | ||
| docker buildx imagetools create \ | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. shall we consider this - the architecture tags are mutable shared names, so overlapping runs can interleave before this manifest resolves them. publish run-scoped staging tags, then promote only those references after both matrix legs succeed: # image job
staging_version="${VERSION}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
# pass --version "${staging_version}" and --push to the builder
# manifest job
source_tag="${tag_version}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
docker buildx imagetools create --prefer-index=false \
--tag "${image_repo}:${tag_version}-amd64" "${image_repo}:${source_tag}-amd64"
docker buildx imagetools create --prefer-index=false \
--tag "${image_repo}:${tag_version}-arm64" "${image_repo}:${source_tag}-arm64"
docker buildx imagetools create --tag "${image_repo}:${tag_version}" \
"${image_repo}:${source_tag}-amd64" "${image_repo}:${source_tag}-arm64"
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Good catch — the race is real: the per-arch tags are mutable shared names, so two overlapping publish runs (e.g. a re-run of an older attempt, or a tag build racing main) could produce a manifest mixing images from different runs. Rather than run-scoped staging tags, I went with referencing the images by digest: each image leg captures the pushed image's registry digest (buildx --metadata-file) and exposes it as a job output, and the manifest job does imagetools create --tag : @ @. Same guarantee — the manifest combines exactly the images this run built — but:
The per-arch :- tags are still pushed for debugging convenience; they just no longer carry correctness. The final : tag stays mutable by design — that's the SNAPSHOT contract (each publish is supposed to move it); the fix only ensures each move is internally consistent. |
||
| --tag "${image_repo}:${tag_version}" \ | ||
| "${image_repo}:${tag_version}-amd64" \ | ||
| "${image_repo}:${tag_version}-arm64" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,126 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| # Builds the busybox init-container installer image from a connector package tarball. | ||
| # | ||
| # The image bundles both plugins (coordinator JAR + native worker .so and lib/) and, when | ||
| # run, installs each into a mounted target directory. See tools/build-packages/README.md. | ||
| # | ||
| # Reusable by local builds (build-packages.sh, --load) and CI (--push). Prints the built | ||
| # image reference to stdout. | ||
| # | ||
| # Requires: docker (with buildx), git, tar. | ||
|
|
||
| set -o errexit | ||
| set -o nounset | ||
| set -o pipefail | ||
|
jackluo923 marked this conversation as resolved.
|
||
|
|
||
| script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" | ||
| image_dir="${script_dir}/image" | ||
|
|
||
| # Shared helpers: image_repo_from_origin (GHCR repo derivation) and _REPO_ROOT. | ||
| source "${script_dir}/dependency-image/utils.sh" | ||
|
|
||
| show_help() { | ||
| cat <<'EOF' | ||
| Usage: ./tools/build-packages/build-installer-init-image.sh --tarball FILE [OPTIONS] | ||
|
|
||
| Builds the busybox init-container installer image from a connector package tarball | ||
| (clp-plugin-presto-connector-<version>-linux-<arch>.tar.gz). | ||
|
|
||
| Options: | ||
| --tarball FILE Package tarball to build the image from (required) | ||
| --version VER Image version tag (default: parsed from the tarball name) | ||
| --arch ARCH amd64 or arm64 (default: parsed from the tarball name) | ||
| --repo REPO Image repository (default: derived from the git origin remote, | ||
| e.g. ghcr.io/y-scope/clp-plugin-presto-connector) | ||
| --push Push the image to the registry (default: --load into local docker) | ||
| --load Load the image into the local docker daemon (default) | ||
| --help Show this help | ||
|
|
||
| See tools/build-packages/README.md for details. | ||
| EOF | ||
| } | ||
|
|
||
| die() { | ||
|
jackluo923 marked this conversation as resolved.
Outdated
|
||
| echo >&2 "ERROR: $*" | ||
| exit 1 | ||
| } | ||
|
|
||
| require_value() { | ||
| [[ -n "${2:-}" ]] || die "$1 requires a value" | ||
| } | ||
|
|
||
| # ── Parse arguments ─────────────────────────────────────────────────────────── | ||
|
|
||
| tarball="" | ||
| version="" | ||
| arch="" | ||
| repo="" | ||
| output="--load" | ||
|
|
||
| while [[ $# -gt 0 ]]; do | ||
| case $1 in | ||
| --tarball) require_value "$1" "${2:-}"; tarball="$2"; shift 2 ;; | ||
| --version) require_value "$1" "${2:-}"; version="$2"; shift 2 ;; | ||
| --arch) require_value "$1" "${2:-}"; arch="$2"; shift 2 ;; | ||
| --repo) require_value "$1" "${2:-}"; repo="$2"; shift 2 ;; | ||
| --push) output="--push"; shift ;; | ||
| --load) output="--load"; shift ;; | ||
| --help) show_help; exit 0 ;; | ||
| *) die "unknown option: $1 (use --help for usage)" ;; | ||
| esac | ||
| done | ||
|
|
||
| [[ -n "${tarball}" ]] || die "--tarball is required (use --help for usage)" | ||
| [[ -f "${tarball}" ]] || die "tarball not found: ${tarball}" | ||
|
|
||
| command -v docker &>/dev/null || die "docker is required" | ||
| docker buildx version &>/dev/null || die "docker buildx is required" | ||
|
|
||
| # ── Resolve version and arch from the tarball name when not given ────────────── | ||
|
|
||
| # Tarball name format: clp-plugin-presto-connector-<version>-linux-<arch>.tar.gz | ||
| tar_base="$(basename "${tarball}")" | ||
| tar_base="${tar_base%.tar.gz}" | ||
| name_rest="${tar_base#clp-plugin-presto-connector-}" | ||
| if [[ "${name_rest}" == "${tar_base}" || "${name_rest}" != *-linux-* ]]; then | ||
| die "cannot parse tarball name '${tar_base}'; pass --version and --arch explicitly" | ||
| fi | ||
| [[ -n "${arch}" ]] || arch="${name_rest##*-linux-}" | ||
| [[ -n "${version}" ]] || version="${name_rest%-linux-"${arch}"}" | ||
|
|
||
| case "${arch}" in | ||
| amd64) platform="linux/amd64" ;; | ||
| arm64) platform="linux/arm64" ;; | ||
| *) die "unsupported arch: ${arch} (expected amd64 or arm64)" ;; | ||
| esac | ||
|
|
||
| [[ -n "${repo}" ]] || repo="$(image_repo_from_origin)" | ||
|
|
||
| # Docker tags allow only [A-Za-z0-9_.-]; sanitize any other version characters (e.g. '+'). | ||
| tag_version="${version//[^A-Za-z0-9_.-]/_}" | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this replacement is lossy: valid versions package_version_to_image_tag() {
local version="$1"
[[ "${version}" =~ ^[0-9][0-9A-Za-z.+~-]*$ ]] || return 1
local tag_version="${version//+/_plus_}"
tag_version="${tag_version//\~/_tilde_}"
printf '%s\n' "${tag_version}"
}
tag_version="$(package_version_to_image_tag "${version}")" ||
die "invalid package version: ${version}"
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Connector version shouldn't contain "+" or "~". Mistake on my part. I will switch to fail loudly if these versions are encountered. |
||
| image="${repo}:${tag_version}-${arch}" | ||
|
|
||
| # ── Assemble a self-contained build context and build ───────────────────────── | ||
|
|
||
| context_dir="$(mktemp -d)" | ||
| trap 'rm -rf "${context_dir}"' EXIT | ||
|
|
||
| # Extract the install tree so coordinator/ and worker/ sit at the context root, matching the | ||
| # Dockerfile's COPY paths. --strip-components=1 drops the versioned top-level directory. | ||
| tar -xzf "${tarball}" -C "${context_dir}" --strip-components=1 | ||
| [[ -d "${context_dir}/coordinator" && -d "${context_dir}/worker" ]] \ | ||
| || die "tarball did not contain coordinator/ and worker/ trees" | ||
|
|
||
| cp "${image_dir}/Dockerfile" "${image_dir}/entrypoint.sh" "${context_dir}/" | ||
|
|
||
| echo >&2 "==> Building installer image ${image} (${platform})..." | ||
| docker buildx build \ | ||
| --platform "${platform}" \ | ||
| --tag "${image}" \ | ||
| "${output}" \ | ||
| -f "${context_dir}/Dockerfile" \ | ||
| "${context_dir}" | ||
|
|
||
| echo >&2 "==> Built ${image}" | ||
| echo "${image}" | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,18 @@ | ||
| # syntax=docker/dockerfile:1 | ||
|
|
||
| # Init-container installer image for the CLP Presto connector. | ||
| # | ||
| # Bundles both plugins and, when run, copies each into a mounted target directory. The | ||
| # build context is an extracted package install tree (coordinator/ + worker/), assembled by | ||
| # build-installer-init-image.sh. See tools/build-packages/README.md. | ||
|
|
||
| # The installer only runs busybox's own sh/cp/mkdir to copy files; it never executes the | ||
| # plugin, so the libc flavor is irrelevant. Pin the small musl variant for a tiny image. | ||
| FROM busybox:1.37.0-musl | ||
|
jackluo923 marked this conversation as resolved.
|
||
|
|
||
| # Match the .deb/.rpm install root (PLUGIN_ROOT) so all channels share one layout. | ||
| COPY coordinator /opt/clp-plugin-presto-connector/coordinator | ||
| COPY worker /opt/clp-plugin-presto-connector/worker | ||
| COPY --chmod=0755 entrypoint.sh /usr/local/bin/install-clp-plugin | ||
|
|
||
| ENTRYPOINT ["/usr/local/bin/install-clp-plugin"] | ||
Uh oh!
There was an error while loading. Please reload this page.