Skip to content

Commit 83b51f2

Browse files
committed
Simplify release publication automation
1 parent a83c5ac commit 83b51f2

16 files changed

Lines changed: 1163 additions & 967 deletions

‎.github/release-automation/release_automation/activities.py‎

Lines changed: 41 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,35 @@
11
import asyncio
2-
import json
32
import os
43
import signal
54
import subprocess
6-
import tempfile
7-
from collections.abc import Mapping
5+
from collections.abc import Awaitable, Callable, Mapping, Sequence
86
from pathlib import Path
9-
from typing import Any, TypeVar, cast
7+
from typing import TypeVar
108

119
from temporalio import activity
12-
from temporalio.converter import value_to_type
1310
from temporalio.exceptions import ApplicationError
1411

1512
from .models import (
1613
MavenInspection,
1714
PublicationInput,
1815
ReleaseResult,
1916
candidate_workflow_id,
20-
json_value,
2117
matches_maven_payload,
22-
maven_artifacts,
2318
publication_queue,
2419
)
20+
from .publication import PublicationFailure, Publisher
2521

26-
T = TypeVar("T")
2722
PASSTHROUGH_ENV = {"PATH", "HOME", "TMPDIR", "LANG", "LC_ALL", "JAVA_HOME", "CI"}
28-
COMMAND_FAILURES = {
29-
42: ("Immutable external identity or checksum conflict.", "ReleaseIdentityConflict"),
30-
44: ("Maven repository identity is not yet visible.", "MavenSubmissionAmbiguous"),
31-
45: ("Portal deployment failed validation.", "MavenDeploymentFailed"),
32-
46: ("GitHub artifact expired or was deleted.", "ArtifactUnavailable"),
33-
}
23+
Result = TypeVar("Result", ReleaseResult, MavenInspection)
3424

3525

36-
async def run_trusted_command(cwd: Path, script: Path, env: Mapping[str, str]) -> list[str]:
37-
"""Run trusted shell reconciliation with an explicit, credential-minimal environment.
26+
async def run_trusted_process(cwd: Path, command: Sequence[str], env: Mapping[str, str]) -> bytes:
27+
"""Run a trusted process with an explicit, credential-minimal environment.
3828
3929
The child gets only a small host-runtime allowlist plus values selected by the
4030
Activity. It runs in its own process group so Temporal cancellation can terminate
4131
the complete command tree, including curl, gh, Docker, Gradle, or GPG children.
4232
"""
43-
command = ["bash", str(script.absolute())]
4433
child_env = {key: value for key, value in os.environ.items() if key in PASSTHROUGH_ENV}
4534
child_env.update(env)
4635
process = await asyncio.create_subprocess_exec(
@@ -68,7 +57,13 @@ async def heartbeat() -> None:
6857
await asyncio.gather(beat, return_exceptions=True)
6958
if process.returncode:
7059
raise subprocess.CalledProcessError(process.returncode, command)
71-
return stdout.decode().splitlines()
60+
return stdout
61+
62+
63+
async def run_trusted_command(cwd: Path, script: Path, env: Mapping[str, str]) -> list[str]:
64+
"""Run a trusted shell script; retained as the small process-isolation test seam."""
65+
output = await run_trusted_process(cwd, ["bash", str(script.absolute())], env)
66+
return output.decode().splitlines()
7267

7368

7469
async def _terminate_tree(process: asyncio.subprocess.Process) -> None:
@@ -86,30 +81,33 @@ async def _terminate_tree(process: asyncio.subprocess.Process) -> None:
8681
class PublicationActivities:
8782
def __init__(
8883
self,
89-
trusted_root: Path,
9084
source_root: Path,
9185
environment: Mapping[str, str],
9286
) -> None:
93-
"""Capture trusted/source roots and a snapshot of the privileged job environment."""
94-
self.trusted_root, self.source_root = trusted_root, source_root
87+
"""Capture the source root and a snapshot of the privileged job environment."""
88+
self.source_root = source_root
9589
self.environment = dict(environment)
9690

9791
@activity.defn(name="publishRelease")
9892
async def publish_release(self, value: PublicationInput) -> ReleaseResult:
9993
"""Reconcile Maven and GitHub to the exact public release state."""
100-
return await self._command(value, "all", ReleaseResult)
94+
return await self._run(value, Publisher.publish_release)
10195

10296
@activity.defn(name="inspectMaven")
10397
async def inspect_maven(self, value: PublicationInput) -> MavenInspection:
10498
"""Inspect every durable Maven generation without creating or publishing state."""
105-
return await self._command(value, "inspect", MavenInspection)
106-
107-
async def _command(self, value: PublicationInput, stage: str, result_type: type[T]) -> T:
108-
"""Validate Activity routing, execute reconciliation, and decode its typed result.
99+
return await self._run(value, Publisher.inspect_maven)
109100

110-
Semantic conflicts and Maven ambiguity use stable exit codes that become typed,
111-
non-retryable ApplicationErrors. Transient process and network failures remain
112-
ordinary Activity failures and therefore use Temporal's configured retry policy.
101+
async def _run(
102+
self,
103+
value: PublicationInput,
104+
operation: Callable[[Publisher], Awaitable[Result]],
105+
) -> Result:
106+
"""Validate routing and execute one typed publication operation.
107+
108+
Durable semantic outcomes become non-retryable ApplicationErrors. Network,
109+
process, and visibility failures remain ordinary exceptions so Temporal applies
110+
the Workflow's retry policy.
113111
"""
114112
try:
115113
expected_commit = self.environment.get("EXPECTED_COMMIT")
@@ -121,39 +119,20 @@ async def _command(self, value: PublicationInput, stage: str, result_type: type[
121119
raise ApplicationError(
122120
str(error), type="InvalidPublicationInput", non_retryable=True
123121
) from error
124-
with tempfile.TemporaryDirectory(prefix="temporal-release-") as directory:
125-
input_file, output_file = (
126-
Path(directory) / name for name in ("input.json", "output.json")
127-
)
128-
data = json_value(value) | {
129-
"releaseDigest": value.release.digest(),
130-
"mavenArtifacts": maven_artifacts(value.release.candidate.mavenPolicy),
131-
}
132-
input_file.write_text(json.dumps(data, separators=(",", ":")))
133-
env = {
134-
"RELEASE_INPUT_FILE": str(input_file),
135-
"RELEASE_OUTPUT_FILE": str(output_file),
136-
"RELEASE_STAGE": stage,
137-
"TRUSTED_AUTOMATION_ROOT": str(self.trusted_root),
138-
}
139-
for name in ("GH_TOKEN", "RH_USER", "RH_PASSWORD"):
140-
if self.environment.get(name):
141-
env[name] = self.environment[name]
142-
try:
143-
output = await run_trusted_command(
144-
self.source_root,
145-
self.trusted_root / ".github/scripts/temporal-release/reconcile-publication.sh",
146-
env,
147-
)
148-
except subprocess.CalledProcessError as error:
149-
if error.returncode not in COMMAND_FAILURES:
150-
raise
151-
message, kind = COMMAND_FAILURES[error.returncode]
152-
raise ApplicationError(message, type=kind, non_retryable=True) from error
153-
if output:
154-
raise RuntimeError("Publication command wrote unexpected standard output.")
155-
raw: Any = json.loads(output_file.read_text())
156-
return cast(T, value_to_type(result_type, raw))
122+
123+
async def command(arguments: Sequence[str], env: Mapping[str, str]) -> bytes:
124+
"""Run publication tools through the Activity's cancellation-aware boundary."""
125+
return await run_trusted_process(self.source_root, arguments, env)
126+
127+
publisher: Publisher | None = None
128+
try:
129+
publisher = Publisher(value, self.source_root, self.environment, command)
130+
return await operation(publisher)
131+
except PublicationFailure as error:
132+
raise ApplicationError(str(error), type=error.error_type, non_retryable=True) from error
133+
finally:
134+
if publisher is not None:
135+
publisher.close()
157136

158137

159138
def validate_publication(

‎.github/release-automation/release_automation/cli.py‎

Lines changed: 24 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import subprocess
66
import sys
77
from collections.abc import Mapping
8+
from contextlib import AsyncExitStack
89
from pathlib import Path
910
from typing import Any, TypeVar, cast
1011

@@ -192,6 +193,14 @@ def workflow_queue(workflow_id: str) -> str:
192193
return candidate_queue_from_digest(workflow_id.removeprefix(prefix))
193194

194195

196+
def publication_worker_queues(queue: str) -> tuple[str, ...]:
197+
"""Return the current publication queue and its one bounded recovery successor."""
198+
match = re.fullmatch(r"(sdk-java-release-[0-9a-f]{32}-publication-g)([01])", queue)
199+
if match is None:
200+
raise ValueError("Refusing to poll a non-publication Task Queue.")
201+
return (queue, f"{match.group(1)}1") if match.group(2) == "0" else (queue,)
202+
203+
195204
def verify_candidate_origin(candidate: CandidateIdentity, run: Mapping[str, Any]) -> None:
196205
"""Reconfirm that GitHub authorized the candidate from the expected push workflow.
197206
@@ -293,33 +302,33 @@ async def output_jobs(client: Client, workflow_id: str, run_id: str) -> None:
293302

294303

295304
async def run_worker(client: Client, queue: str, env: Mapping[str, str]) -> None:
296-
"""Poll one release Workflow queue and one exact publication generation queue.
305+
"""Poll one release Workflow queue and the bounded publication generation queues.
297306
298307
Hosting both Workers in the publication job removes a separate Actions worker
299308
matrix. Activities still exist only on the privileged generation-specific queue,
300309
while deterministic Workflow code polls the candidate-specific queue.
301310
"""
302-
if not queue.startswith("sdk-java-release-") or "-publication-g" not in queue:
303-
raise ValueError("Refusing to poll a non-publication Task Queue.")
311+
queues = publication_worker_queues(queue)
304312
workflow_id = required(env, "EXPECTED_WORKFLOW_ID")
305313
source = Path(required(env, "RELEASE_SOURCE_DIR")).absolute()
306314
if not source.is_dir():
307315
raise ValueError("RELEASE_SOURCE_DIR is not a directory.")
308-
activities = PublicationActivities(ROOT, source, env)
316+
activities = PublicationActivities(source, env)
309317
release_worker = Worker(
310318
client, task_queue=workflow_queue(workflow_id), workflows=[ReleaseWorkflow]
311319
)
312-
publication_worker = Worker(
313-
client,
314-
task_queue=queue,
315-
activities=[activities.publish_release, activities.inspect_maven],
316-
)
317320
handle = client.get_workflow_handle(workflow_id, run_id=env.get("EXPECTED_RUN_ID"))
318-
async with release_worker, publication_worker:
319-
try:
320-
await asyncio.wait_for(asyncio.shield(handle.result()), 95 * 60)
321-
except TimeoutError:
322-
pass
321+
async with AsyncExitStack() as stack:
322+
await stack.enter_async_context(release_worker)
323+
for task_queue in queues:
324+
await stack.enter_async_context(
325+
Worker(
326+
client,
327+
task_queue=task_queue,
328+
activities=[activities.publish_release, activities.inspect_maven],
329+
)
330+
)
331+
await handle.result()
323332

324333

325334
async def async_main(argv: list[str], env: Mapping[str, str]) -> None:
@@ -357,7 +366,7 @@ async def async_main(argv: list[str], env: Mapping[str, str]) -> None:
357366

358367
def main() -> None:
359368
"""Validate the trusted checkout layout and run the asynchronous CLI."""
360-
if not (ROOT / ".github/scripts/temporal-release/reconcile-publication.sh").is_file():
369+
if not (ROOT / ".github/scripts/temporal-release/prepare-maven-payload.sh").is_file():
361370
raise RuntimeError("The trusted repository root has an unexpected layout.")
362371
asyncio.run(async_main(sys.argv[1:], os.environ))
363372

‎.github/release-automation/release_automation/maven_payload.py‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,13 @@
66
import sys
77
import tarfile
88
import xml.etree.ElementTree as ET
9+
from collections.abc import Iterable
910

1011

1112
def validate(
1213
root: pathlib.Path,
1314
manifest: pathlib.Path,
14-
policy: pathlib.Path,
15+
policy: Iterable[str],
1516
version: str,
1617
commit: str,
1718
exact: bool,
@@ -23,7 +24,7 @@ def validate(
2324
complete, preventing a partially generated payload from becoming durable release data.
2425
"""
2526
root = root.resolve()
26-
approved = set(policy.read_text().splitlines())
27+
approved = set(policy)
2728
records = []
2829
for line in manifest.read_text().splitlines():
2930
relative, digest, size = line.split("\t")
@@ -122,7 +123,8 @@ def main() -> None:
122123
manifest = pathlib.Path(source)
123124
else:
124125
raise ValueError("expected extract or validate")
125-
validate(root_path, manifest, pathlib.Path(policy), version, commit, mode == "validate")
126+
approved = pathlib.Path(policy).read_text().splitlines()
127+
validate(root_path, manifest, approved, version, commit, mode == "validate")
126128

127129

128130
if __name__ == "__main__":
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import gzip
2+
import io
3+
import stat
4+
import sys
5+
import tarfile
6+
import zipfile
7+
from pathlib import Path
8+
9+
10+
def package(source: Path, output: Path, root: str, name: str, windows: bool) -> None:
11+
"""Create a byte-reproducible native archive with one executable entry.
12+
13+
Fixed timestamps, ownership, permissions, and member order make two builds of
14+
identical native bytes produce the same public archive on every runner.
15+
"""
16+
data = source.read_bytes()
17+
if windows:
18+
entry = zipfile.ZipInfo(f"{root}/{name}", (1980, 1, 1, 0, 0, 0))
19+
entry.compress_type = zipfile.ZIP_DEFLATED
20+
entry.external_attr = (stat.S_IFREG | 0o755) << 16
21+
with zipfile.ZipFile(output, "w") as archive:
22+
archive.writestr(entry, data)
23+
return
24+
25+
tar_bytes = io.BytesIO()
26+
with tarfile.open(fileobj=tar_bytes, mode="w", format=tarfile.GNU_FORMAT) as archive:
27+
directory = tarfile.TarInfo(root)
28+
directory.type, directory.mode = tarfile.DIRTYPE, 0o755
29+
directory.uid = directory.gid = directory.mtime = 0
30+
directory.uname = directory.gname = ""
31+
archive.addfile(directory)
32+
binary = tarfile.TarInfo(f"{root}/{name}")
33+
binary.size, binary.mode = len(data), 0o755
34+
binary.uid = binary.gid = binary.mtime = 0
35+
binary.uname = binary.gname = ""
36+
archive.addfile(binary, io.BytesIO(data))
37+
with (
38+
output.open("wb") as raw,
39+
gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as compressed,
40+
):
41+
compressed.write(tar_bytes.getvalue())
42+
43+
44+
def main() -> None:
45+
"""Package the validated shell inputs without embedding Python in the script."""
46+
source, output, root, name, platform = sys.argv[1:]
47+
package(Path(source), Path(output), root, name, platform == "windows-amd64")
48+
49+
50+
if __name__ == "__main__":
51+
main()

0 commit comments

Comments
 (0)