Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 109 additions & 0 deletions .github/workflows/build-packages.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
# resolve-version use the input version or derive it from pom.xml
# build-dependency-image ensure the hash-tagged build-env image exists
# build build .deb / .rpm / .tar.gz artifacts per architecture
# image build the busybox installer image per arch (push on main/tags)
# manifest combine per-arch images into a multi-arch tag (main/tags only)
#
# See tools/build-packages/README.md for package-build details.

Expand Down Expand Up @@ -58,6 +60,7 @@ jobs:
runs-on: "ubuntu-24.04"
outputs:
version: "${{steps.version.outputs.version}}"
publish: "${{steps.publish.outputs.publish}}"
steps:
- name: "Check out repository"
uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0
Expand Down Expand Up @@ -88,6 +91,19 @@ jobs:
echo "version=${version}" >> "$GITHUB_OUTPUT"
echo "::notice::Package version: ${version}"

- name: "Decide whether to publish images"
id: "publish"
env:
DEFAULT_BRANCH: "${{github.event.repository.default_branch}}"
run: |-
# Publish images only from the default branch and version tags so feature
# branches don't overwrite shared tags.
if [[ ${GITHUB_REF_NAME} == "${DEFAULT_BRANCH}" || ${GITHUB_REF_TYPE} == tag ]]; then
echo "publish=true" >> "$GITHUB_OUTPUT"
else
echo "publish=false" >> "$GITHUB_OUTPUT"
fi

build-dependency-image:
uses: "./.github/workflows/build-dependency-image.yaml"
permissions:
Expand Down Expand Up @@ -182,3 +198,96 @@ jobs:
path: "packages/${{steps.packages.outputs.tar_gz_filename}}"
if-no-files-found: "error"
retention-days: 14

# Build the busybox installer image per architecture from the tarball artifact. This runs
# on the native runner (not inside the build-env container) so docker/buildx is available.
image:
needs: ["build", "resolve-version"]
permissions:
contents: "read"
packages: "write"
strategy:
fail-fast: false
matrix:
include:
- arch: "amd64"
runner: "ubuntu-24.04"
- arch: "arm64"
runner: "ubuntu-24.04-arm"
runs-on: "${{matrix.runner}}"
timeout-minutes: 30
env:
ARCH: "${{matrix.arch}}"
steps:
- name: "Check out repository"
uses: "actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0" # v7.0.0
with:
persist-credentials: false

- name: "Download tarball artifact"
uses: "actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53" # v6.0.0
Comment thread
jackluo923 marked this conversation as resolved.
Outdated
with:
pattern: "*-linux-${{matrix.arch}}.tar.gz"
path: "dist"
merge-multiple: true

- name: "Log in to GHCR"
if: "${{needs.resolve-version.outputs.publish == 'true'}}"
uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0
with:
registry: "ghcr.io"
username: "${{github.actor}}"
password: "${{secrets.GITHUB_TOKEN}}"

- name: "Set up Docker Buildx"
uses: "docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c" # v4.2.0

- name: "Build the installer image (and push when publishing)"
env:
PUBLISH: "${{needs.resolve-version.outputs.publish}}"
run: |-
image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')"
shopt -s nullglob
tarballs=(dist/*-linux-"${ARCH}".tar.gz)
if (( ${#tarballs[@]} != 1 )); then
echo "::error::Expected exactly one tarball for ${ARCH}, found ${#tarballs[@]}"
exit 1
fi
# Push from the default branch / tags; otherwise just build to validate.
output="--load"
if [[ "${PUBLISH}" == "true" ]]; then
output="--push"
fi
bash tools/build-packages/build-installer-init-image.sh \
--tarball "${tarballs[0]}" \
--arch "${ARCH}" \
--repo "${image_repo}" \
"${output}"

# Combine the per-architecture installer images into one multi-arch version tag.
manifest:
needs: ["image", "resolve-version"]
if: "${{needs.resolve-version.outputs.publish == 'true'}}"
permissions:
packages: "write"
runs-on: "ubuntu-24.04"
timeout-minutes: 15
env:
VERSION: "${{needs.resolve-version.outputs.version}}"
steps:
- name: "Log in to GHCR"
uses: "docker/login-action@af1e73f918a031802d376d3c8bbc3fe56130a9b0" # v4.4.0
with:
registry: "ghcr.io"
username: "${{github.actor}}"
password: "${{secrets.GITHUB_TOKEN}}"

- name: "Create multi-arch manifest"
run: |-
image_repo="ghcr.io/$(printf '%s' "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]')"
# Docker tags allow only [A-Za-z0-9_.-]; sanitize to match the build script.
tag_version="${VERSION//[^A-Za-z0-9_.-]/_}"
docker buildx imagetools create \

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shall we consider this -

the architecture tags are mutable shared names, so overlapping runs can interleave before this manifest resolves them. publish run-scoped staging tags, then promote only those references after both matrix legs succeed:

# image job
staging_version="${VERSION}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
# pass --version "${staging_version}" and --push to the builder

# manifest job
source_tag="${tag_version}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
docker buildx imagetools create --prefer-index=false \
  --tag "${image_repo}:${tag_version}-amd64" "${image_repo}:${source_tag}-amd64"
docker buildx imagetools create --prefer-index=false \
  --tag "${image_repo}:${tag_version}-arm64" "${image_repo}:${source_tag}-arm64"
docker buildx imagetools create --tag "${image_repo}:${tag_version}" \
  "${image_repo}:${source_tag}-amd64" "${image_repo}:${source_tag}-arm64"

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good catch — the race is real: the per-arch tags are mutable shared names, so two overlapping publish runs (e.g. a re-run of an older attempt, or a tag build racing main) could produce a manifest mixing images from different runs.

Rather than run-scoped staging tags, I went with referencing the images by digest: each image leg captures the pushed image's registry digest (buildx --metadata-file) and exposes it as a job output, and the manifest job does imagetools create --tag : @ @. Same guarantee — the manifest combines exactly the images this run built — but:

  • digests are immutable, so there's no window at all (staging tags are still mutable names, just less likely to collide);
  • no staging tags accumulating in the GHCR package (they'd need a cleanup step to avoid piling up per run);
  • no extra pushes — the digest is a side effect of the push we already do.

The per-arch :- tags are still pushed for debugging convenience; they just no longer carry correctness. The final : tag stays mutable by design — that's the SNAPSHOT contract (each publish is supposed to move it); the fix only ensures each move is internally consistent.

--tag "${image_repo}:${tag_version}" \
"${image_repo}:${tag_version}-amd64" \
"${image_repo}:${tag_version}-arm64"
22 changes: 20 additions & 2 deletions tools/build-packages/README.md
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
# CLP Presto connector packaging

This directory builds installable `.deb`, `.rpm`, and `.tar.gz` artifacts for the CLP
Presto connector (coordinator + worker) on `amd64` and `arm64`.
This directory builds installable `.deb`, `.rpm`, and `.tar.gz` artifacts, plus a busybox
init-container installer image, for the CLP Presto connector (coordinator + worker) on
`amd64` and `arm64`.

CI packaging runs `tools/build-packages/internal/container/build-artifacts.sh`
through `.github/workflows/build-packages.yaml`. Local builds use
Expand All @@ -25,6 +26,23 @@ task package

A thin wrapper over `./tools/build-packages/build-packages.sh` (call that directly if `go-task` isn't installed). Both accept `--output DIR`, `--version VER`, and `--with-ca-certs`; with the task, put `--` before the flags: `task package -- --output DIR`.

### Installer image

`task package` also builds and loads a busybox init-container image that bundles both plugins. Its entrypoint copies each component into a mounted volume named by `COORDINATOR_PLUGIN_INSTALL_PATH` / `WORKER_PLUGIN_INSTALL_PATH` (set either or both):

```bash
docker run --rm -e WORKER_PLUGIN_INSTALL_PATH=/plugins -v "$(pwd)/plugins:/plugins" \
ghcr.io/y-scope/clp-plugin-presto-connector:<version>-<arch>
```

Run `build-installer-init-image.sh --help` to build it standalone from any package tarball.
Comment thread
jackluo923 marked this conversation as resolved.
Outdated

In CI, `build-packages.yaml` builds the image per architecture on every run and
combines them into a multi-arch `:<version>` tag; pushes to GHCR happen only
from the default branch and version tags. The multi-arch tag exists only on the
registry (a manifest can't be loaded into a local daemon) — local builds always
load `:<version>-<arch>`.

The build runs inside a hash-tagged **build-env image** (`env-<hash>`) based on
`manylinux_2_28`. `build-dependency-image.sh` resolves it from the local Docker
cache, this repository's GHCR package, or a local build, reusing the cached
Expand Down
18 changes: 0 additions & 18 deletions tools/build-packages/build-dependency-image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,24 +19,6 @@ set -o pipefail
script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
source "${script_dir}/dependency-image/utils.sh"

# Derive this repo's GHCR namespace from its GitHub origin remote.
image_repo_from_origin() {
local remote_url owner_repo
remote_url="$(git -C "${_REPO_ROOT}" remote get-url origin)"
case "${remote_url}" in
https://github.com/*) owner_repo="${remote_url#https://github.com/}" ;;
git@github.com:*) owner_repo="${remote_url#git@github.com:}" ;;
ssh://git@github.com/*) owner_repo="${remote_url#ssh://git@github.com/}" ;;
*)
echo >&2 "ERROR: can't derive GHCR image repo from origin remote: ${remote_url}"
echo >&2 " Expected a github.com remote."
exit 1
;;
esac
owner_repo="${owner_repo%.git}"
printf 'ghcr.io/%s\n' "$(printf '%s' "${owner_repo}" | tr '[:upper:]' '[:lower:]')"
}

host_platform() {
case "$(uname -m)" in
x86_64) printf 'linux/amd64\n' ;;
Expand Down
126 changes: 126 additions & 0 deletions tools/build-packages/build-installer-init-image.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
#!/usr/bin/env bash

# Builds the busybox init-container installer image from a connector package tarball.
#
# The image bundles both plugins (coordinator JAR + native worker .so and lib/) and, when
# run, installs each into a mounted target directory. See tools/build-packages/README.md.
#
# Reusable by local builds (build-packages.sh, --load) and CI (--push). Prints the built
# image reference to stdout.
#
# Requires: docker (with buildx), git, tar.

set -o errexit
set -o nounset
set -o pipefail
Comment thread
jackluo923 marked this conversation as resolved.

script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
image_dir="${script_dir}/image"

# Shared helpers: image_repo_from_origin (GHCR repo derivation) and _REPO_ROOT.
source "${script_dir}/dependency-image/utils.sh"

show_help() {
cat <<'EOF'
Usage: ./tools/build-packages/build-installer-init-image.sh --tarball FILE [OPTIONS]

Builds the busybox init-container installer image from a connector package tarball
(clp-plugin-presto-connector-<version>-linux-<arch>.tar.gz).

Options:
--tarball FILE Package tarball to build the image from (required)
--version VER Image version tag (default: parsed from the tarball name)
--arch ARCH amd64 or arm64 (default: parsed from the tarball name)
--repo REPO Image repository (default: derived from the git origin remote,
e.g. ghcr.io/y-scope/clp-plugin-presto-connector)
--push Push the image to the registry (default: --load into local docker)
--load Load the image into the local docker daemon (default)
--help Show this help

See tools/build-packages/README.md for details.
EOF
}

die() {
Comment thread
jackluo923 marked this conversation as resolved.
Outdated
echo >&2 "ERROR: $*"
exit 1
}

require_value() {
[[ -n "${2:-}" ]] || die "$1 requires a value"
}

# ── Parse arguments ───────────────────────────────────────────────────────────

tarball=""
version=""
arch=""
repo=""
output="--load"

while [[ $# -gt 0 ]]; do
case $1 in
--tarball) require_value "$1" "${2:-}"; tarball="$2"; shift 2 ;;
--version) require_value "$1" "${2:-}"; version="$2"; shift 2 ;;
--arch) require_value "$1" "${2:-}"; arch="$2"; shift 2 ;;
--repo) require_value "$1" "${2:-}"; repo="$2"; shift 2 ;;
--push) output="--push"; shift ;;
--load) output="--load"; shift ;;
--help) show_help; exit 0 ;;
*) die "unknown option: $1 (use --help for usage)" ;;
esac
done

[[ -n "${tarball}" ]] || die "--tarball is required (use --help for usage)"
[[ -f "${tarball}" ]] || die "tarball not found: ${tarball}"

command -v docker &>/dev/null || die "docker is required"
docker buildx version &>/dev/null || die "docker buildx is required"

# ── Resolve version and arch from the tarball name when not given ──────────────

# Tarball name format: clp-plugin-presto-connector-<version>-linux-<arch>.tar.gz
tar_base="$(basename "${tarball}")"
tar_base="${tar_base%.tar.gz}"
name_rest="${tar_base#clp-plugin-presto-connector-}"
if [[ "${name_rest}" == "${tar_base}" || "${name_rest}" != *-linux-* ]]; then
die "cannot parse tarball name '${tar_base}'; pass --version and --arch explicitly"
fi
[[ -n "${arch}" ]] || arch="${name_rest##*-linux-}"
[[ -n "${version}" ]] || version="${name_rest%-linux-"${arch}"}"

case "${arch}" in
amd64) platform="linux/amd64" ;;
arm64) platform="linux/arm64" ;;
*) die "unsupported arch: ${arch} (expected amd64 or arm64)" ;;
esac

[[ -n "${repo}" ]] || repo="$(image_repo_from_origin)"

# Docker tags allow only [A-Za-z0-9_.-]; sanitize any other version characters (e.g. '+').
tag_version="${version//[^A-Za-z0-9_.-]/_}"

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this replacement is lossy: valid versions 1.0+rc and 1.0~rc both become 1.0_rc. define one shared encoding and use it here and in the manifest job:

package_version_to_image_tag() {
    local version="$1"
    [[ "${version}" =~ ^[0-9][0-9A-Za-z.+~-]*$ ]] || return 1
    local tag_version="${version//+/_plus_}"
    tag_version="${tag_version//\~/_tilde_}"
    printf '%s\n' "${tag_version}"
}

tag_version="$(package_version_to_image_tag "${version}")" ||
    die "invalid package version: ${version}"

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Connector version shouldn't contain "+" or "~". Mistake on my part. I will switch to fail loudly if these versions are encountered.

image="${repo}:${tag_version}-${arch}"

# ── Assemble a self-contained build context and build ─────────────────────────

context_dir="$(mktemp -d)"
trap 'rm -rf "${context_dir}"' EXIT

# Extract the install tree so coordinator/ and worker/ sit at the context root, matching the
# Dockerfile's COPY paths. --strip-components=1 drops the versioned top-level directory.
tar -xzf "${tarball}" -C "${context_dir}" --strip-components=1
[[ -d "${context_dir}/coordinator" && -d "${context_dir}/worker" ]] \
|| die "tarball did not contain coordinator/ and worker/ trees"

cp "${image_dir}/Dockerfile" "${image_dir}/entrypoint.sh" "${context_dir}/"

echo >&2 "==> Building installer image ${image} (${platform})..."
docker buildx build \
--platform "${platform}" \
--tag "${image}" \
"${output}" \
-f "${context_dir}/Dockerfile" \
"${context_dir}"

echo >&2 "==> Built ${image}"
echo "${image}"
13 changes: 13 additions & 0 deletions tools/build-packages/build-packages.sh
Original file line number Diff line number Diff line change
Expand Up @@ -164,3 +164,16 @@ if ! compgen -G "${artifact_stage}/*" > /dev/null; then
exit 1
fi
cp -f "${artifact_stage}"/* "${output_dir}/"

# Build the busybox installer image as a fourth distribution channel, from the tarball this
# run just produced. Source from artifact_stage (this run's fresh staging) rather than
# output_dir, which may hold tarballs from earlier or other-arch builds.
echo "==> Building busybox installer image..."
shopt -s nullglob
tarballs=("${artifact_stage}"/*.tar.gz)
shopt -u nullglob
if (( ${#tarballs[@]} != 1 )); then
echo >&2 "ERROR: expected exactly one .tar.gz in staging, found ${#tarballs[@]}"
exit 1
fi
"${src}/tools/build-packages/build-installer-init-image.sh" --tarball "${tarballs[0]}" --load
18 changes: 18 additions & 0 deletions tools/build-packages/dependency-image/utils.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,24 @@ image_ref() {
echo "$1/$2:env-$3"
}

# Derives this repo's GHCR namespace from its GitHub origin remote.
image_repo_from_origin() {
local remote_url owner_repo
remote_url="$(git -C "${_REPO_ROOT}" remote get-url origin)"
case "${remote_url}" in
https://github.com/*) owner_repo="${remote_url#https://github.com/}" ;;
git@github.com:*) owner_repo="${remote_url#git@github.com:}" ;;
ssh://git@github.com/*) owner_repo="${remote_url#ssh://git@github.com/}" ;;
*)
echo >&2 "ERROR: can't derive GHCR image repo from origin remote: ${remote_url}"
echo >&2 " Expected a github.com remote."
exit 1
;;
esac
owner_repo="${owner_repo%.git}"
printf 'ghcr.io/%s\n' "$(printf '%s' "${owner_repo}" | tr '[:upper:]' '[:lower:]')"
}

# Inputs that should change the build-env image tag.
_BUILD_ENV_HASH_INPUTS=(
".dockerignore"
Expand Down
18 changes: 18 additions & 0 deletions tools/build-packages/image/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
# syntax=docker/dockerfile:1

# Init-container installer image for the CLP Presto connector.
#
# Bundles both plugins and, when run, copies each into a mounted target directory. The
# build context is an extracted package install tree (coordinator/ + worker/), assembled by
# build-installer-init-image.sh. See tools/build-packages/README.md.

# The installer only runs busybox's own sh/cp/mkdir to copy files; it never executes the
# plugin, so the libc flavor is irrelevant. Pin the small musl variant for a tiny image.
FROM busybox:1.37.0-musl
Comment thread
jackluo923 marked this conversation as resolved.

# Match the .deb/.rpm install root (PLUGIN_ROOT) so all channels share one layout.
COPY coordinator /opt/clp-plugin-presto-connector/coordinator
COPY worker /opt/clp-plugin-presto-connector/worker
COPY --chmod=0755 entrypoint.sh /usr/local/bin/install-clp-plugin

ENTRYPOINT ["/usr/local/bin/install-clp-plugin"]
Loading
Loading