From 4560d5bfbc323411c95ccc4c8bf01c65dd3901ef Mon Sep 17 00:00:00 2001 From: LuLo Date: Mon, 29 Jun 2026 11:28:10 +0200 Subject: [PATCH] feat(pyartcd): add golang-builder-shipment pipeline New artcd command to create shipment MRs in ocp-shipment-data for golang builder images. Auto-resolves Konflux image NVRs from golang RPM NVRs, detects prod vs ec lifecycle from ocp-build-data, and opens a draft MR for ERT approval. rh-pre-commit.version: 2.3.2 rh-pre-commit.check-secrets: ENABLED --- doozer/doozerlib/assembly_inspector.py | 38 +- doozer/doozerlib/backend/konflux_client.py | 3 +- doozer/doozerlib/backend/konflux_fbc.py | 3 +- doozer/doozerlib/backend/rebaser.py | 3 +- doozer/doozerlib/cli/images_streams.py | 151 ++--- doozer/doozerlib/cli/scan_sources_konflux.py | 58 +- .../doozerlib/lockfile_prototype/constants.py | 3 +- .../doozerlib/lockfile_prototype/generator.py | 292 ++------ .../lockfile_prototype/rebaser_hooks.py | 2 - .../doozerlib/lockfile_prototype/resolver.py | 23 +- .../lockfile_prototype/shell_parser.py | 54 +- doozer/tests/backend/test_konflux_client.py | 5 +- doozer/tests/backend/test_konflux_fbc.py | 13 +- doozer/tests/cli/test_images_streams.py | 66 +- .../lockfile_prototype/test_generator.py | 389 +---------- .../tests/lockfile_prototype/test_resolver.py | 98 --- .../lockfile_prototype/test_shell_parser.py | 64 +- doozer/tests/test_assembly_inspector.py | 23 - .../cli/process_release_from_fbc_bugs_cli.py | 76 +-- .../test_process_release_from_fbc_bugs_cli.py | 199 +----- .../tests/test_schema/test_group_schema.py | 23 - .../validator/json_schemas/repos.schema.json | 1 - pyartcd/pyartcd/__main__.py | 2 + pyartcd/pyartcd/pipelines/__init__.py | 2 + .../pipelines/build_microshift_bootc.py | 7 +- .../pipelines/golang_builder_shipment.py | 519 +++++++++++++++ pyartcd/pyartcd/pipelines/images_health.py | 37 +- pyartcd/pyartcd/pipelines/okd.py | 2 +- .../pyartcd/pipelines/quay_doomsday_backup.py | 10 +- pyartcd/pyartcd/pipelines/sync_ci_images.py | 49 +- .../pipelines/test_build_microshift_bootc.py | 53 -- .../pipelines/test_golang_builder_shipment.py | 627 ++++++++++++++++++ pyartcd/tests/pipelines/test_images_health.py | 74 +-- .../pipelines/test_quay_doomsday_backup.py | 59 -- .../tests/pipelines/test_sync_ci_images.py | 60 -- 35 files changed, 1410 insertions(+), 1678 deletions(-) create mode 100644 pyartcd/pyartcd/pipelines/golang_builder_shipment.py create mode 100644 pyartcd/tests/pipelines/test_golang_builder_shipment.py delete mode 100644 pyartcd/tests/pipelines/test_quay_doomsday_backup.py diff --git a/doozer/doozerlib/assembly_inspector.py b/doozer/doozerlib/assembly_inspector.py index 681a40554c..2a154b9748 100644 --- a/doozer/doozerlib/assembly_inspector.py +++ b/doozer/doozerlib/assembly_inspector.py @@ -202,20 +202,12 @@ async def check_rhcos_issues(self, rhcos_build: RHCOSBuildInspector) -> List[Ass build_dict = external_rpms.get(entry["tag"], {}).get(package_name) if not build_dict: continue - installed_rpm = installed_packages.get(package_name) - if installed_rpm and installed_rpm["nvr"] != build_dict["nvr"]: - if not self._is_installed_rpm_in_tag(installed_rpm, entry["tag"]): - self.runtime.logger.info( - "Installed rpm %s in RHCOS %s (%s) is not tagged in %s; skipping check_external_packages complaint", - installed_rpm["nvr"], - rhcos_build.build_id, - rhcos_build.brew_arch, - entry["tag"], - ) - continue if entry["condition"] == "match": desired_packages[package_name] = build_dict["nvr"] elif entry["condition"] == "greater_equal": + # installed_rpm = installed_packages.get(package_name) + # desired_packages[package_name] = build_dict["nvr"] if not installed_rpm or compare_nvr(build_dict, installed_rpm) > 0 else installed_rpm["nvr"] + installed_rpm = installed_packages.get(package_name) if not installed_rpm or compare_nvr(build_dict, installed_rpm) >= 0: desired_packages[package_name] = build_dict["nvr"] else: @@ -430,19 +422,10 @@ def check_group_image_consistency( build_dict = external_rpms.get(entry["tag"], {}).get(package_name) if not build_dict: continue - installed_rpm = installed_packages.get(package_name) - if installed_rpm and installed_rpm["nvr"] != build_dict["nvr"]: - if not self._is_installed_rpm_in_tag(installed_rpm, entry["tag"]): - image_meta.logger.info( - "Installed rpm %s in image %s is not tagged in %s; skipping check_external_packages complaint", - installed_rpm["nvr"], - dgk, - entry["tag"], - ) - continue if entry["condition"] == "match": desired_packages[package_name] = build_dict["nvr"] elif entry["condition"] == "greater_equal": + installed_rpm = installed_packages.get(package_name) if not installed_rpm or compare_nvr(build_dict, installed_rpm) >= 0: desired_packages[package_name] = build_dict["nvr"] else: @@ -586,19 +569,6 @@ def get_group_rpm_build_dicts(self, el_ver: int) -> Dict[str, Optional[Dict]]: return self._rpm_build_cache[el_ver] - def _is_installed_rpm_in_tag(self, installed_rpm: Optional[Dict], tag: str) -> bool: - """Check if an installed RPM build is tagged in the given Brew tag. - :param installed_rpm: installed package build dict (must have 'id' key), or None - :param tag: Brew tag name to check - :return: True if the installed RPM is tagged in the given tag - """ - if not installed_rpm: - return False - tag_names = { - t["name"] for t in self.brew_session.listTags(brew.KojiWrapperOpts(caching=True), build=installed_rpm["id"]) - } - return tag in tag_names - def get_external_rpm_build_dicts(self): """Get external rpm build dicts from rhaos candidate Brew tags. :return: a dict. key is Brew tag name, value is another (component_name, rpm_build) dict diff --git a/doozer/doozerlib/backend/konflux_client.py b/doozer/doozerlib/backend/konflux_client.py index 2b54f65675..6bbe713c88 100644 --- a/doozer/doozerlib/backend/konflux_client.py +++ b/doozer/doozerlib/backend/konflux_client.py @@ -1324,9 +1324,8 @@ def _modify_param(plr_params: List, name: str, value: Union[str, bool, list[str] _modify_param( task["params"], "refspec", - f"{commit_sha}:refs/remotes/origin/{target_branch}", + f"{commit_sha}:refs/remotes/origin/{target_branch} refs/tags/*:refs/tags/*", ) - _modify_param(task["params"], "fetchTags", "true") if build_params.enable_symlink_check is not None: _modify_param(task["params"], "enableSymlinkCheck", build_params.enable_symlink_check) case "sast-snyk-check": diff --git a/doozer/doozerlib/backend/konflux_fbc.py b/doozer/doozerlib/backend/konflux_fbc.py index ea72b5ba44..d8e1485e50 100644 --- a/doozer/doozerlib/backend/konflux_fbc.py +++ b/doozer/doozerlib/backend/konflux_fbc.py @@ -1574,8 +1574,7 @@ def _catagorize_catalog_blobs(self, blobs: List[Dict]): raise IOError(f"Found unsupported schema: {schema}") if not package_name: raise IOError(f"Couldn't determine package name for unknown schema: {schema}") - blob_key = package_name if schema == "olm.deprecations" else blob["name"] - categorized_blobs.setdefault(package_name, {}).setdefault(schema, {})[blob_key] = blob + categorized_blobs.setdefault(package_name, {}).setdefault(schema, {})[blob["name"]] = blob return categorized_blobs diff --git a/doozer/doozerlib/backend/rebaser.py b/doozer/doozerlib/backend/rebaser.py index b17f92dfae..e5b5ee1611 100644 --- a/doozer/doozerlib/backend/rebaser.py +++ b/doozer/doozerlib/backend/rebaser.py @@ -975,10 +975,9 @@ async def _update_build_dir( from doozerlib.lockfile_prototype.rebaser_hooks import apply_dockerfile_transforms lockfile_has_packages = bool(getattr(metadata, "lockfile_packages", None)) - upgrades_dropped = getattr(metadata, "lockfile_upgrades_dropped", False) apply_dockerfile_transforms( dest_dir, - strip_updates=not downstream_parents or not lockfile_has_packages or upgrades_dropped, + strip_updates=not downstream_parents or not lockfile_has_packages, logger=self._logger, ) diff --git a/doozer/doozerlib/cli/images_streams.py b/doozer/doozerlib/cli/images_streams.py index e022bf2466..5b3da7eef4 100644 --- a/doozer/doozerlib/cli/images_streams.py +++ b/doozer/doozerlib/cli/images_streams.py @@ -124,14 +124,6 @@ def images_streams(): multiple=True, help='If specified, only these stream names will be mirrored.', ) -@click.option( - '--image', - 'images', - metavar='IMAGE_NAME', - default=[], - multiple=True, - help='If specified, only these image distgit keys will be mirrored (must have ci_alignment.upstream_image).', -) @click.option( '--only-if-missing', default=False, @@ -153,7 +145,6 @@ def images_streams(): def images_streams_mirror( runtime, streams: Tuple[str, ...], - images: Tuple[str, ...], only_if_missing: bool, live_test_mode: bool, force: bool, @@ -172,7 +163,7 @@ def images_streams_mirror( elif runtime.registry_config_dir is not None: registry_config_file = get_docker_config_json(runtime.registry_config_dir) - def mirror_image(cmd_start: str, upstream_dest: str, source_digest: Optional[str] = None): + def mirror_image(cmd_start: str, upstream_dest: str): if upstream_dest.startswith('registry.ci.openshift.org/'): # Images targeting CI imagestreams must be mirrored to quay.io/openshift/ci (QCI) first, # then imagestreams updated to reference the QCI image by digest. @@ -206,12 +197,8 @@ def mirror_image(cmd_start: str, upstream_dest: str, source_digest: Optional[str f'Failed to mirror {upstream_entry_name}: {stderr}', (rc, stdout, stderr) ) - # Use source digest when available (oc image mirror preserves manifest digests). - # Falling back to get_image_digest for tag-based sources where digest isn't known. - if source_digest: - qci_digest = source_digest - else: - qci_digest = get_image_digest(floating_qci_dest, registry_config_file) + # Get digest of the newly mirrored image (handles manifest lists) + qci_digest = get_image_digest(floating_qci_dest, registry_config_file) if qci_digest: # Mirror to GC-prevention tag: art__ @@ -280,7 +267,7 @@ def mirror_image(cmd_start: str, upstream_dest: str, source_digest: Optional[str f'Failed to mirror {upstream_entry_name}: {stderr}', (rc, stdout, stderr) ) - upstreaming_entries = _get_upstreaming_entries(runtime, streams, image_names=images) + upstreaming_entries = _get_upstreaming_entries(runtime, streams) for upstream_entry_name, config in upstreaming_entries.items(): if config.mirror is True or force: @@ -357,13 +344,9 @@ def mirror_image(cmd_start: str, upstream_dest: str, source_digest: Optional[str if registry_config_file is not None: cmd += f" --registry-config={registry_config_file}" - # Extract digest from source pullspec when available (e.g. Konflux @sha256: refs). - # oc image mirror preserves manifest bytes, so source digest == destination digest. - src_digest = src_image_pullspec.split('@')[1] if '@' in src_image_pullspec else None - # Mirror to main destination only if not in only-if-missing mode OR destination doesn't exist if not only_if_missing or not destinations_to_check.get(upstream_dest, False): - mirror_image(cmd, upstream_dest, source_digest=src_digest) + mirror_image(cmd, upstream_dest) # mirror arm64 builder and base images for CI if mirror_arm: @@ -372,7 +355,6 @@ def mirror_image(cmd_start: str, upstream_dest: str, source_digest: Optional[str if registry_config_file is not None: arm_cmd += f" --registry-config={registry_config_file}" # Mirror ARM64 only if not in only-if-missing mode OR destination doesn't exist - # ARM64 filter produces a single-arch image, not the original manifest list, so don't pass source_digest if not only_if_missing or not destinations_to_check.get(f'{upstream_dest}-arm64', False): mirror_image(arm_cmd, f'{upstream_dest}-arm64') @@ -404,19 +386,11 @@ def mirror_image(cmd_start: str, upstream_dest: str, source_digest: Optional[str multiple=True, help='If specified, only check these streams', ) -@click.option( - '--image', - 'images', - metavar='IMAGE_NAME', - default=[], - multiple=True, - help='If specified, only check these image distgit keys (must have ci_alignment.upstream_image).', -) @click.option('--live-test-mode', default=False, is_flag=True, help='Scan for live-test mode buildconfigs') @click.option('--dry-run', default=False, is_flag=True, help='Do not actually mirror or update imagestreams') @option_registry_auth @pass_runtime -def images_streams_check_upstream(runtime, streams, images, live_test_mode, dry_run, registry_auth: Optional[str]): +def images_streams_check_upstream(runtime, streams, live_test_mode, dry_run, registry_auth: Optional[str]): runtime.initialize(clone_distgits=False, clone_source=False, prevent_cloning=True) # Determine which registry config to use @@ -432,7 +406,9 @@ def images_streams_check_upstream(runtime, streams, images, live_test_mode, dry_ istags_status = [] qci_status = [] - upstreaming_entries = _get_upstreaming_entries(runtime, streams, image_names=images) + if not streams: + streams = runtime.get_stream_names() + upstreaming_entries = _get_upstreaming_entries(runtime, streams) for upstream_entry_name, config in upstreaming_entries.items(): upstream_dest = config.upstream_image @@ -606,8 +582,8 @@ def images_streams_check_upstream(runtime, streams, images, live_test_mode, dry_ print() -def get_eligible_buildconfigs(runtime, streams, live_test_mode, images=()): - upstreaming_entries = _get_upstreaming_entries(runtime, streams, image_names=images) +def get_eligible_buildconfigs(runtime, streams, live_test_mode): + upstreaming_entries = _get_upstreaming_entries(runtime, streams) buildconfig_names = [] for upstream_entry_name, config in upstreaming_entries.items(): transform = config.transform @@ -637,14 +613,6 @@ def get_eligible_buildconfigs(runtime, streams, live_test_mode, images=()): multiple=True, help='If specified, only these stream names will be processed.', ) -@click.option( - '--image', - 'images', - metavar='IMAGE_NAME', - default=[], - multiple=True, - help='If specified, only these image distgit keys will be processed (must have ci_alignment.upstream_image).', -) @click.option( '--as', 'as_user', @@ -657,9 +625,7 @@ def get_eligible_buildconfigs(runtime, streams, live_test_mode, images=()): @click.option('--dry-run', default=False, is_flag=True, help='Do not build anything, but only print build operations.') @option_registry_auth @pass_runtime -def images_streams_start_buildconfigs( - runtime, streams, images, as_user, live_test_mode, dry_run, registry_auth: Optional[str] -): +def images_streams_start_buildconfigs(runtime, streams, as_user, live_test_mode, dry_run, registry_auth: Optional[str]): runtime.initialize(clone_distgits=False, clone_source=False, prevent_cloning=True) # Determine which registry config to use @@ -684,7 +650,7 @@ def images_streams_start_buildconfigs( return buildconfigs = json.loads(bc_stdout).get('items', []) - eligible_bc_names = get_eligible_buildconfigs(runtime, streams, live_test_mode, images=images) + eligible_bc_names = get_eligible_buildconfigs(runtime, streams, live_test_mode) # Track triggered builds for post-build preservation triggered_builds = [] @@ -1077,67 +1043,52 @@ def images_streams_start_buildconfigs( print('\n=== Post-Build Preservation Complete ===') -def _get_upstreaming_entry_for_image(runtime, image_meta): - """ - Build an upstreaming entry from an image meta's ci_alignment config. - Returns the entry Model, or None if the image should be skipped (e.g. no build yet). - """ - upstream_entry = Model(dict_to_model=image_meta.config.content.source.ci_alignment.primitive()) - - try: - upstream_entry['image'] = image_meta.pull_url() - except IOError as e: - runtime.logger.warning(f'Skipping {image_meta.distgit_key} for CI alignment: {e}') - return None - - if upstream_entry.final_user is Missing: - upstream_entry.final_user = image_meta.config.final_stage_user - return upstream_entry - - -def _get_upstreaming_entries(runtime, stream_names=None, image_names=None): +def _get_upstreaming_entries(runtime, stream_names=None): """ Looks through streams.yml entries and each image metadata for upstream transform information. :param runtime: doozer runtime :param stream_names: A list of streams to look for in streams.yml. If None or empty, all will be searched. - :param image_names: A list of image distgit keys to include. Each must have ci_alignment.upstream_image set. :return: Returns a map[name] => { transform: '...', upstream_image.... } where name is a stream name or distgit key. """ - fetch_all_image_metas = False - if not stream_names and not image_names: + fetch_image_metas = False + if not stream_names: + # If not specified, use all. stream_names = runtime.get_stream_names() - fetch_all_image_metas = True + fetch_image_metas = True upstreaming_entries = {} + streams_config = runtime.streams + for stream in stream_names: + config = streams_config[stream] + if config is Missing: + raise IOError(f'Did not find stream {stream} in streams.yml for this group') + if config.upstream_image is not Missing: + upstreaming_entries[stream] = streams_config[stream] + + if fetch_image_metas: + # Some images also have their own upstream information. This allows them to + # be mirrored out into upstream, optionally transformed, and made available as builder images for + # other images without being in streams.yml. - if stream_names: - streams_config = runtime.streams - for stream in stream_names: - config = streams_config[stream] - if config is Missing: - raise IOError(f'Did not find stream {stream} in streams.yml for this group') - if config.upstream_image is not Missing: - upstreaming_entries[stream] = streams_config[stream] - - if fetch_all_image_metas: for image_meta in runtime.ordered_image_metas(): if image_meta.config.content.source.ci_alignment.upstream_image is not Missing: - entry = _get_upstreaming_entry_for_image(runtime, image_meta) - if entry is not None: - upstreaming_entries[image_meta.distgit_key] = entry - - elif image_names: - for name in image_names: - meta = runtime.image_map.get(name) - if not meta: - raise IOError(f'Image {name} not found in group metadata') - if meta.config.content.source.ci_alignment.upstream_image is Missing: - raise IOError(f'Image {name} does not have ci_alignment.upstream_image set; not eligible for CI sync') - entry = _get_upstreaming_entry_for_image(runtime, meta) - if entry is not None: - upstreaming_entries[name] = entry + upstream_entry = Model( + dict_to_model=image_meta.config.content.source.ci_alignment.primitive() + ) # Make a copy + + # Use pull_url() which handles both Brew and Konflux build systems + try: + upstream_entry['image'] = image_meta.pull_url() + except IOError as e: + # Skip images that don't have builds yet (e.g., new images in Konflux) + runtime.logger.warning(f'Skipping {image_meta.distgit_key} for CI alignment: {e}') + continue + + if upstream_entry.final_user is Missing: + upstream_entry.final_user = image_meta.config.final_stage_user + upstreaming_entries[image_meta.distgit_key] = upstream_entry return upstreaming_entries @@ -1153,14 +1104,6 @@ def _get_upstreaming_entries(runtime, stream_names=None, image_names=None): multiple=True, help='If specified, only these stream names will be processed.', ) -@click.option( - '--image', - 'images', - metavar='IMAGE_NAME', - default=[], - multiple=True, - help='If specified, only these image distgit keys will be processed (must have ci_alignment.upstream_image).', -) @click.option( '-o', '--output', @@ -1174,7 +1117,7 @@ def _get_upstreaming_entries(runtime, stream_names=None, image_names=None): @click.option('--apply', default=False, is_flag=True, help='Apply the output if any buildconfigs are generated') @click.option('--live-test-mode', default=False, is_flag=True, help='Generate live-test mode buildconfigs') @pass_runtime -def images_streams_gen_buildconfigs(runtime, streams, images, output, as_user, apply, live_test_mode): +def images_streams_gen_buildconfigs(runtime, streams, output, as_user, apply, live_test_mode): """ ART has a mandate to make "ART equivalent" images available usptream for CI workloads. This enables CI to compile with the same golang version ART is using and use identical UBI8 images, etc. To accomplish @@ -1217,7 +1160,7 @@ def images_streams_gen_buildconfigs(runtime, streams, images, output, as_user, a buildconfig_definitions = [] - upstreaming_entries = _get_upstreaming_entries(runtime, streams, image_names=images) + upstreaming_entries = _get_upstreaming_entries(runtime, streams) for upstream_entry_name, config in upstreaming_entries.items(): transform = config.transform diff --git a/doozer/doozerlib/cli/scan_sources_konflux.py b/doozer/doozerlib/cli/scan_sources_konflux.py index 7f06dba72d..7cb37441b2 100644 --- a/doozer/doozerlib/cli/scan_sources_konflux.py +++ b/doozer/doozerlib/cli/scan_sources_konflux.py @@ -24,7 +24,7 @@ from artcommonlib.pushd import Dir from artcommonlib.release_util import isolate_timestamp_in_release from artcommonlib.rhcos import get_latest_layered_rhcos_build, get_primary_container_name -from artcommonlib.rpm_utils import parse_nvr, to_nevra +from artcommonlib.rpm_utils import parse_nvr from artcommonlib.util import deep_merge, fetch_slsa_attestation, uses_konflux_imagestream_override from artcommonlib.variants import BuildVariant from async_lru import alru_cache @@ -629,12 +629,11 @@ async def scan_image(self, image_meta: ImageMetadata): stage = 'external image checks' await self.scan_external_image_changes(image_meta) - # Check for changes in image arches - stage = 'arch checks' - await self.scan_arch_changes(image_meta) - # For OCP variant, perform additional checks that don't apply to OKD if self.variant != BuildVariant.OKD: + # Check for changes in image arches (skip for OKD - arch changes don't trigger OKD rebuilds) + stage = 'arch checks' + await self.scan_arch_changes(image_meta) # Check for changes in the network mode (skip for OKD - it always uses open network) stage = 'network mode checks' await self.scan_network_mode_changes(image_meta) @@ -1277,10 +1276,6 @@ async def scan_rpm_changes(self, image_meta: ImageMetadata): # Check for changes in non-ART RPMs build_record_inspector = KonfluxBuildRecordInspector(self.runtime, build_record) non_latest_rpms = await build_record_inspector.find_non_latest_rpms(self.package_rpm_finder) - - if non_latest_rpms: - non_latest_rpms = await self._filter_parent_inherited_rpms(image_meta, non_latest_rpms) - rebuild_hints = [ f"Outdated RPM {installed_rpm} installed in {build_record.nvr} ({arch}) when {latest_rpm} was available in repo {repo}" for arch, non_latest in non_latest_rpms.items() @@ -1293,51 +1288,6 @@ async def scan_rpm_changes(self, image_meta: ImageMetadata): else: self.logger.info('No package changes detected for %s', build_record.nvr) - async def _filter_parent_inherited_rpms( - self, - image_meta: ImageMetadata, - non_latest_rpms: dict[str, list[tuple[str, str, str]]], - ) -> dict[str, list[tuple[str, str, str]]]: - """ - Filter out outdated RPMs inherited from the parent image. - - If an outdated RPM is at the same version as in the parent image, - rebuilding this image cannot fix it — the parent must be updated - first. Suppressing these prevents pointless rebuild loops. - """ - parent_key = image_meta.config["from"].member - if not parent_key: - return non_latest_rpms - - parent_build = self.latest_image_build_records_map.get(parent_key) - if not parent_build: - return non_latest_rpms - - parent_rpms = self.package_rpm_finder.get_brew_rpms_from_build_record(parent_build) - parent_nevras = {to_nevra(rpm) for rpm in parent_rpms} - - filtered = {} - suppressed = 0 - for arch, rpm_list in non_latest_rpms.items(): - kept = [] - for installed_rpm, latest_rpm, repo in rpm_list: - if installed_rpm in parent_nevras: - suppressed += 1 - else: - kept.append((installed_rpm, latest_rpm, repo)) - if kept: - filtered[arch] = kept - - if suppressed: - self.logger.info( - "%s: suppressed %d outdated RPMs inherited from parent %s", - image_meta.distgit_key, - suppressed, - parent_key, - ) - - return filtered - @skip_check_if_changing async def scan_extra_packages(self, image_meta: ImageMetadata): """ diff --git a/doozer/doozerlib/lockfile_prototype/constants.py b/doozer/doozerlib/lockfile_prototype/constants.py index d86489a957..797f13917b 100644 --- a/doozer/doozerlib/lockfile_prototype/constants.py +++ b/doozer/doozerlib/lockfile_prototype/constants.py @@ -34,7 +34,7 @@ # RPM pseudo-packages that appear in rpmdb but are not installable via DNF RPM_PSEUDO_PACKAGES = frozenset({"gpg-pubkey"}) -VALID_PKG_NAME = re.compile(r"^[a-zA-Z0-9*][a-zA-Z0-9._+\-*]*$") +VALID_PKG_NAME = re.compile(r"^[a-zA-Z0-9][a-zA-Z0-9._+\-]*$") # rpm-lockfile-prototype stores extracted RPMDBs here. There is no env var @@ -47,7 +47,6 @@ RPMDB_CACHE_ERROR_PATTERNS = [ "database disk image is malformed", "failed loading RPMDB", - "No such file or directory", ] diff --git a/doozer/doozerlib/lockfile_prototype/generator.py b/doozer/doozerlib/lockfile_prototype/generator.py index d1db988773..00b4284036 100644 --- a/doozer/doozerlib/lockfile_prototype/generator.py +++ b/doozer/doozerlib/lockfile_prototype/generator.py @@ -45,18 +45,6 @@ from doozerlib.repos import Repos -def _is_local_rpm(token: str) -> bool: - """ - Return True if token is a local RPM file path or glob that can't - be resolved via lockfile repos (e.g. ``/path/*.rpm``, ``foo.rpm``). - """ - if token.endswith(".rpm"): - return True - if "/" in token and "*" in token: - return True - return False - - def build_rpms_in_yaml( repos: list[RepoEntry], arches: list[str], @@ -82,16 +70,6 @@ def build_rpms_in_yaml( Return Value(s): RpmsInConfig: Config ready for YAML serialization. """ - packages = [p for p in packages if not _is_local_rpm(p)] - if arch_specific_packages: - arch_specific_packages = { - arch: [p for p in pkgs if not _is_local_rpm(p)] for arch, pkgs in arch_specific_packages.items() - } - if reinstall_packages: - reinstall_packages = [p for p in reinstall_packages if not _is_local_rpm(p)] - if upgrade_packages: - upgrade_packages = [p for p in upgrade_packages if not _is_local_rpm(p)] - package_entries: list[str | ArchSpecificPackage] = list(packages) if arch_specific_packages: for arch, arch_pkgs in arch_specific_packages.items(): @@ -130,7 +108,6 @@ def __init__( self.fallback_installed: dict[int, list[str]] = {} self.parent_source_dirs: dict[int, Path] = {} self.logger = logger or logutil.get_logger(__name__) - self.upgrades_dropped = False self._container = container_helper or ContainerImageHelper(logger=self.logger) self._resolver = resolver or RpmResolver(working_dir=working_dir, logger=self.logger) @@ -460,20 +437,7 @@ async def _resolve_all_stages( reinstall_pkgs: list[str] | None = None strippable: set[str] | None = None - if stage_num == final_stage_num and not is_update_only and has_bare_update: - if image_pullspec and packages: - # Bare update + explicit installs in final stage: reinstall - # the Dockerfile packages so they appear in the lockfile - # even when already installed in the base image. Base image - # packages use upgrade semantics via upgrade_targets (set - # above), so we intentionally do NOT reinstall them here. - reinstall_pkgs = list(packages) - strippable = set() - self.logger.info( - f"{distgit_key}: stage {stage_num}: {len(reinstall_pkgs)} Dockerfile " - "packages will be reinstalled into lockfile (bare update stage)" - ) - elif stage_num == final_stage_num and not is_update_only and not has_bare_update: + if stage_num == final_stage_num and not is_update_only and not has_bare_update: if image_pullspec: # --image mode: pass base image packages as reinstallPackages # so they appear in the lockfile at repo versions. base.reinstall() @@ -504,24 +468,17 @@ async def _resolve_all_stages( # so they appear in the lockfile even when already installed # on some architectures, and add base image packages to the # install list for conflict detection. - # - # When the builder image RHEL version differs from the repos - # (e.g. el8 builder with el9 repos), skip --image mode and - # base image packages — cross-RHEL depsolve is unsolvable. - if self._has_rhel_version_mismatch(stage_num, repo_list, distgit_key): - image_pullspec = None - else: - if image_pullspec: - reinstall_pkgs = list(packages) - base_pkgs = await self._get_base_image_packages(stage_num, image_pullspec, distgit_key) - if base_pkgs: - extra = [p for p in base_pkgs if p not in packages] - if extra: - packages = packages + extra - self.logger.info( - f"{distgit_key}: stage {stage_num}: {len(extra)} base image " - "packages added to install list for conflict detection" - ) + if image_pullspec: + reinstall_pkgs = list(packages) + base_pkgs = await self._get_base_image_packages(stage_num, image_pullspec, distgit_key) + if base_pkgs: + extra = [p for p in base_pkgs if p not in packages] + if extra: + packages = packages + extra + self.logger.info( + f"{distgit_key}: stage {stage_num}: {len(extra)} base image " + "packages added to install list for conflict detection" + ) enable_only = [s.split("/")[0] for s in stage_info.module_specs] if stage_info.module_specs else None @@ -643,80 +600,6 @@ async def _determine_stage_pullspec(self, stage_num: int, distgit_key: str) -> s image_pullspec = resolved return image_pullspec - @staticmethod - def _extract_rhel_version_from_pullspec(pullspec: str) -> int | None: - """ - Extract RHEL major version from an image pullspec tag. - - Handles two tag formats: - - rhel-8-golang-..., ubi-9-minimal, etc. - - NVR-style: openshift-golang-builder-container-v1.25.9-...el8 - - Arg(s): - pullspec (str): Image pullspec with tag or digest. - Return Value(s): - int | None: RHEL major version (e.g. 8, 9), or None if - not detectable. - """ - if ":" not in pullspec: - return None - tag = pullspec.split("@")[0].split(":")[-1] - m = re.search(r"(?:rhel|ubi|centos|scos)-?(\d+)", tag) - if m: - return int(m.group(1)) - m = re.search(r"\.el(\d+)", tag) - if m: - return int(m.group(1)) - return None - - @staticmethod - def _extract_rhel_version_from_repos(repo_list: list[RepoEntry]) -> int | None: - """ - Extract RHEL major version from repo content set IDs. - - Arg(s): - repo_list (list[RepoEntry]): Repository entries with repoid - fields like "rhel-9-for-x86_64-baseos-e4s-rpms__9_DOT_6". - Return Value(s): - int | None: RHEL major version, or None if not detectable. - """ - for repo in repo_list: - m = re.search(r"rhel-(\d+)", repo.repoid) - if m: - return int(m.group(1)) - return None - - def _has_rhel_version_mismatch(self, stage_num: int, repo_list: list[RepoEntry], distgit_key: str) -> bool: - """ - Check whether a builder stage's RHEL version differs from the - repos' RHEL version. Returns False when either version cannot - be determined (fail-open). - - Arg(s): - stage_num (int): Dockerfile stage number. - repo_list (list[RepoEntry]): Repository entries. - distgit_key (str): Image identifier for logging. - Return Value(s): - bool: True if a RHEL version mismatch is detected. - """ - if stage_num >= len(self.downstream_parents): - return False - pullspec = self.downstream_parents[stage_num] - if not pullspec or "/" not in pullspec: - return False - builder_rhel = self._extract_rhel_version_from_pullspec(pullspec) - repo_rhel = self._extract_rhel_version_from_repos(repo_list) - if builder_rhel is None or repo_rhel is None: - return False - if builder_rhel != repo_rhel: - self.logger.warning( - f"{distgit_key}: stage {stage_num}: RHEL version mismatch — " - f"builder image is el{builder_rhel} but repos are el{repo_rhel}; " - f"skipping base image packages and --image mode for this stage" - ) - return True - return False - async def _handle_update_only_stage( self, stage_num: int, image_pullspec: str | None, distgit_key: str ) -> tuple[list[str], list[str], str | None]: @@ -797,14 +680,6 @@ async def _resolve_builddep_packages( resolved: set[str] = set() for pattern in builddep_patterns: - if pattern.endswith(".spec"): - self.logger.warning( - f"{distgit_key}: builddep target '{pattern}' is a spec file — " - "only SRPMs are supported for BuildRequires extraction; " - "spec files cannot be parsed reliably due to macro resolution" - ) - continue - srpm_pattern = pattern if pattern.endswith(".src.rpm") else f"{pattern}.src.rpm" matching_srpms = [ f @@ -812,9 +687,18 @@ async def _resolve_builddep_packages( if f.is_file() and f.name.endswith(".src.rpm") and fnmatch.fnmatch(f.name, srpm_pattern) ] + if not matching_srpms: + matching_specs = [ + f + for f in dest_dir.iterdir() + if f.is_file() and f.name.endswith(".spec") and fnmatch.fnmatch(f.name, pattern) + ] + if matching_specs: + matching_srpms = matching_specs + if not matching_srpms: self.logger.warning( - f"{distgit_key}: no SRPM matching '{pattern}' found in {dest_dir}, " + f"{distgit_key}: no SRPM or spec file matching '{pattern}' found in {dest_dir}, " "builddep packages will not be included in lockfile" ) continue @@ -837,65 +721,6 @@ async def _resolve_builddep_packages( self.logger.info(f"{distgit_key}: resolved {len(resolved)} builddep packages: {sorted(resolved)}") return sorted(resolved) - def _build_resolve_config( - self, - repo_list: list[RepoEntry], - arches: list[str], - packages: list[str], - arch_pkgs: dict[str, list[str]], - update_targets: list[str], - reinstall: list[str], - promote_reinstall_to_upgrade: bool, - image_pullspec: str | None, - module_enable: list[str] | None, - ) -> RpmsInConfig: - """ - Build the rpms.in.yaml config for a resolve attempt. - """ - upgrade_extras = reinstall if promote_reinstall_to_upgrade else [] - effective_upgrade = list(set(update_targets + upgrade_extras)) if image_pullspec else None - return build_rpms_in_yaml( - repo_list, - arches, - packages, - arch_specific_packages=arch_pkgs, - reinstall_packages=reinstall if image_pullspec else None, - upgrade_packages=effective_upgrade, - module_enable=module_enable, - ) - - @staticmethod - def _strip_missing_packages( - missing: set[str], - remaining_packages: list[str], - remaining_update_targets: list[str], - remaining_reinstall: list[str], - arch_pkgs: dict[str, list[str]], - ) -> int: - """ - Remove missing packages from all lists. Returns count of packages removed. - """ - prev_count = ( - len(remaining_packages) - + sum(len(v) for v in arch_pkgs.values()) - + len(remaining_update_targets) - + len(remaining_reinstall) - ) - remaining_packages[:] = [p for p in remaining_packages if p not in missing] - remaining_update_targets[:] = [p for p in remaining_update_targets if p not in missing] - remaining_reinstall[:] = [p for p in remaining_reinstall if p not in missing] - for arch in list(arch_pkgs.keys()): - arch_pkgs[arch] = [p for p in arch_pkgs[arch] if p not in missing] - if not arch_pkgs[arch]: - del arch_pkgs[arch] - new_count = ( - len(remaining_packages) - + sum(len(v) for v in arch_pkgs.values()) - + len(remaining_update_targets) - + len(remaining_reinstall) - ) - return prev_count - new_count - async def _resolve_stage_with_retry( self, repo_list: list[RepoEntry], @@ -948,16 +773,16 @@ async def _resolve_stage_with_retry( retries_exhausted = False for _attempt in range(MAX_RESOLUTION_RETRIES): - in_yaml = self._build_resolve_config( + upgrade_extras = remaining_reinstall if promote_reinstall_to_upgrade else [] + effective_upgrade = list(set(remaining_update_targets + upgrade_extras)) if image_pullspec else None + in_yaml = build_rpms_in_yaml( repo_list, arches, remaining_packages, - arch_pkgs, - remaining_update_targets, - remaining_reinstall, - promote_reinstall_to_upgrade, - image_pullspec, - module_enable, + arch_specific_packages=arch_pkgs, + reinstall_packages=remaining_reinstall if image_pullspec else None, + upgrade_packages=effective_upgrade, + module_enable=module_enable, ) try: @@ -976,25 +801,26 @@ async def _resolve_stage_with_retry( f"{distgit_key}: stage {stage_num}: required Dockerfile packages not available " f"in configured repos, stripping: {sorted(required_missing)}" ) - reinstall_only = missing & set(remaining_reinstall) - fully_missing = missing - reinstall_only - removed = 0 - if reinstall_only: - remaining_reinstall[:] = [p for p in remaining_reinstall if p not in reinstall_only] - removed += len(reinstall_only) - self.logger.info( - f"{distgit_key}: stage {stage_num}: stripped from reinstall only " - f"(keeping in install/upgrade): {sorted(reinstall_only)}" - ) - if fully_missing: - removed += self._strip_missing_packages( - fully_missing, - remaining_packages, - remaining_update_targets, - remaining_reinstall, - arch_pkgs, - ) - if not removed: + prev_count = ( + len(remaining_packages) + + sum(len(v) for v in arch_pkgs.values()) + + len(remaining_update_targets) + + len(remaining_reinstall) + ) + remaining_packages = [p for p in remaining_packages if p not in missing] + remaining_update_targets = [p for p in remaining_update_targets if p not in missing] + remaining_reinstall = [p for p in remaining_reinstall if p not in missing] + for arch in list(arch_pkgs.keys()): + arch_pkgs[arch] = [p for p in arch_pkgs[arch] if p not in missing] + if not arch_pkgs[arch]: + del arch_pkgs[arch] + new_count = ( + len(remaining_packages) + + sum(len(v) for v in arch_pkgs.values()) + + len(remaining_update_targets) + + len(remaining_reinstall) + ) + if new_count == prev_count: raise if stripped_tracker is not None: stripped_tracker.update(missing) @@ -1032,22 +858,16 @@ async def _resolve_stage_with_retry( "continuing without reinstall packages" ) remaining_reinstall.clear() - # Clear all upgrade targets and disable reinstall→upgrade promotion - # for the fallback — upgrade targets that reference packages not in - # the base image's rpmdb cause PackagesNotInstalledError from DNF. - remaining_update_targets.clear() - promote_reinstall_to_upgrade = False - self.upgrades_dropped = True - in_yaml = self._build_resolve_config( + fallback_upgrade_extras = remaining_reinstall if promote_reinstall_to_upgrade else [] + effective_upgrade = list(set(remaining_update_targets + fallback_upgrade_extras)) if image_pullspec else None + in_yaml = build_rpms_in_yaml( repo_list, arches, remaining_packages, - arch_pkgs, - remaining_update_targets, - remaining_reinstall, - promote_reinstall_to_upgrade, - image_pullspec, - module_enable, + arch_specific_packages=arch_pkgs, + reinstall_packages=remaining_reinstall if image_pullspec else None, + upgrade_packages=effective_upgrade, + module_enable=module_enable, ) return await self._resolver.resolve(in_yaml, image_pullspec=resolver_pullspec) diff --git a/doozer/doozerlib/lockfile_prototype/rebaser_hooks.py b/doozer/doozerlib/lockfile_prototype/rebaser_hooks.py index 80e673cdde..baafe3b64e 100644 --- a/doozer/doozerlib/lockfile_prototype/rebaser_hooks.py +++ b/doozer/doozerlib/lockfile_prototype/rebaser_hooks.py @@ -93,8 +93,6 @@ async def generate_lockfile( pkg_names.add(name) metadata.lockfile_packages = sorted(pkg_names) - metadata.lockfile_upgrades_dropped = generator.upgrades_dropped - return shared_dnf_cache diff --git a/doozer/doozerlib/lockfile_prototype/resolver.py b/doozer/doozerlib/lockfile_prototype/resolver.py index f85df4c5d5..8de1c21414 100644 --- a/doozer/doozerlib/lockfile_prototype/resolver.py +++ b/doozer/doozerlib/lockfile_prototype/resolver.py @@ -86,14 +86,13 @@ async def resolve( if rc != 0: if image_pullspec and self._is_rpmdb_corrupt(stderr): - self._clear_rpmdb_cache(image_pullspec) - self.logger.info("Retrying rpm-lockfile-prototype after RPMDB cache error") - rc, _, stderr = await cmd_gather_async(cmd, check=False, env=env) - if rc == 0: - return LockfileData.model_validate(yaml.safe_load(out_file.read_text())) - error_summary = stderr.strip().rsplit("\n", 1)[-1] - self.logger.warning("Retry also failed (exit code %d): %s", rc, error_summary) - self.logger.debug("Full retry stderr:\n%s", stderr) + cleared = self._clear_rpmdb_cache(image_pullspec) + if cleared: + self.logger.info("Retrying rpm-lockfile-prototype after clearing corrupt RPMDB cache") + retry_rc, _, retry_stderr = await cmd_gather_async(cmd, check=False, env=env) + if retry_rc == 0: + return LockfileData.model_validate(yaml.safe_load(out_file.read_text())) + self.logger.warning("Retry also failed (exit code %d): %s", retry_rc, retry_stderr) raise RuntimeError(f"rpm-lockfile-prototype failed (exit code {rc}): {stderr}") @@ -151,9 +150,8 @@ def parse_missing_packages(error_text: str) -> set[str]: """ Parse missing package names from rpm-lockfile-prototype error output. - Handles the CLI format ("missing packages: X, Y"), DNF install/upgrade - errors ("No match for argument: X"), and DNF reinstall errors - ("no package matched: X"). + Handles both the CLI format ("missing packages: X, Y") and the + DNF format ("No match for argument: X"). Arg(s): error_text (str): Error message from rpm-lockfile-prototype. @@ -168,7 +166,4 @@ def parse_missing_packages(error_text: str) -> set[str]: m = re.search(r"No match for argument:\s*(\S+)", line.strip()) if m: missing.add(m.group(1).strip().rstrip(":")) - m = re.search(r"no package matched:\s*(\S+)", line.strip()) - if m: - missing.add(m.group(1).strip().rstrip(":")) return {p for p in missing if VALID_PKG_NAME.match(p)} diff --git a/doozer/doozerlib/lockfile_prototype/shell_parser.py b/doozer/doozerlib/lockfile_prototype/shell_parser.py index 60a0cad658..73764f1bab 100644 --- a/doozer/doozerlib/lockfile_prototype/shell_parser.py +++ b/doozer/doozerlib/lockfile_prototype/shell_parser.py @@ -11,7 +11,7 @@ import re import bashlex -from artcommonlib.arch_util import BREW_ARCHES, brew_arch_for_go_arch +from artcommonlib.arch_util import brew_arch_for_go_arch from pydantic import BaseModel, Field from doozerlib.lockfile_prototype.constants import ARCH_KEYWORDS @@ -338,16 +338,7 @@ def _process_assignments( if has_cmdsub: extracted = _extract_subshell_packages(var_value) if extracted: - target_arches = _extract_subshell_arch_condition(var_value) - if target_arches: - per_arch = ctx.arch_shell_vars.setdefault(var_name, {}) - for arch in target_arches: - if arch in per_arch: - per_arch[arch] = f"{per_arch[arch]} {extracted}" - else: - per_arch[arch] = extracted - else: - ctx.shell_vars[var_name] = extracted + ctx.shell_vars[var_name] = extracted elif arch_context: per_arch = ctx.arch_shell_vars.setdefault(var_name, {}) for arch in arch_context: @@ -382,7 +373,7 @@ def _detect_pkg_action(word_values: list[str], ctx: _WalkContext) -> tuple[str | if wl in ("update", "upgrade"): ctx.has_update = True return "update", idx - if wl in ("builddep", "build-dep"): + if wl == "builddep": return "builddep", idx if wl == "module": for sub_idx, sub_w in enumerate(word_values[idx + 1 :], idx + 1): @@ -511,10 +502,7 @@ def _process_command_node( return word_values = [w.word for w in words] - all_vars = {**ctx.variables, **ctx.shell_vars} - resolved_first = resolve_bash_expansion(word_values[0], all_vars) if word_values else "" - resolved_word_values = [resolved_first] + word_values[1:] if word_values else word_values - action, action_idx = _detect_pkg_action(resolved_word_values, ctx) + action, action_idx = _detect_pkg_action(word_values, ctx) if not action: return @@ -540,40 +528,6 @@ def _process_command_node( _classify_package_tokens(resolved_tokens, arch_resolved_tokens, action, ctx, arch_context) -def _extract_subshell_arch_condition(subshell_body: str) -> list[str] | None: - """ - Detect architecture conditions inside a subshell body and return - the list of arches where the subshell produces output. - - Arg(s): - subshell_body (str): Text inside a $(...) subshell, e.g. - 'if [ "$(uname -m)" != "s390x" ]; then echo -n mstflint; fi' - Return Value(s): - list[str] | None: Target arches (Brew names), or None if no - arch condition detected or pattern is ambiguous. - """ - if not _has_arch_test(subshell_body): - return None - - neq_arches = ARCH_NEQ_VALUE_RE.findall(subshell_body) - eq_arches = ARCH_VALUE_RE.findall(subshell_body) - - if neq_arches and eq_arches: - return None - - if neq_arches: - excluded = set(_normalize_arch_names(neq_arches)) - target = sorted(set(BREW_ARCHES) - excluded) - return target if target else None - - if eq_arches: - if len(eq_arches) > 1: - return None - return _normalize_arch_names(eq_arches) - - return None - - def _extract_subshell_packages(subshell_body: str) -> str: """ Extract package names from echo commands inside a $(...) subshell. diff --git a/doozer/tests/backend/test_konflux_client.py b/doozer/tests/backend/test_konflux_client.py index ab349d3f66..f63ef47e07 100644 --- a/doozer/tests/backend/test_konflux_client.py +++ b/doozer/tests/backend/test_konflux_client.py @@ -905,10 +905,7 @@ class TestSkipTasks(IsolatedAsyncioTestCase): "taskRunTemplate": {"serviceAccountName": "default"}, "pipelineSpec": { "tasks": [ - { - "name": "clone-repository", - "params": [{"name": "refspec", "value": ""}, {"name": "fetchTags", "value": "false"}], - }, + {"name": "clone-repository", "params": [{"name": "refspec", "value": ""}]}, {"name": "build-images", "params": [{"name": "SBOM_TYPE", "value": ""}]}, {"name": "clair-scan", "params": []}, {"name": "sast-snyk-check", "params": []}, diff --git a/doozer/tests/backend/test_konflux_fbc.py b/doozer/tests/backend/test_konflux_fbc.py index 931701d084..40d67973f2 100644 --- a/doozer/tests/backend/test_konflux_fbc.py +++ b/doozer/tests/backend/test_konflux_fbc.py @@ -678,27 +678,17 @@ async def test_fetch_olm_bundle_blob(self, mock_render): mock_render.assert_called_once_with("test-image-pullspec", migrate_level="none", auth=ANY) def test_categorize_catalog_blobs(self): - deprecation_blob = { - "schema": "olm.deprecations", - "package": "test-package", - "entries": [ - {"reference": {"schema": "olm.bundle", "name": "test-bundle.v1.0.0"}, "message": "deprecated"}, - ], - } catalog_blobs = [ {"schema": "olm.package", "name": "test-package"}, {"schema": "olm.channel", "name": "test-channel", "package": "test-package"}, {"schema": "olm.bundle", "name": "test-bundle", "package": "test-package"}, - deprecation_blob, {"schema": "olm.package", "name": "test-package2"}, {"schema": "olm.channel", "name": "test-channel2", "package": "test-package2"}, {"schema": "olm.bundle", "name": "test-bundle2", "package": "test-package2"}, ] actual = self.rebaser._catagorize_catalog_blobs(catalog_blobs) self.assertEqual(actual.keys(), {"test-package", "test-package2"}) - self.assertEqual( - actual["test-package"].keys(), {"olm.package", "olm.channel", "olm.bundle", "olm.deprecations"} - ) + self.assertEqual(actual["test-package"].keys(), {"olm.package", "olm.channel", "olm.bundle"}) self.assertEqual( actual["test-package"]["olm.package"]["test-package"], {"schema": "olm.package", "name": "test-package"} ) @@ -710,7 +700,6 @@ def test_categorize_catalog_blobs(self): actual["test-package"]["olm.bundle"]["test-bundle"], {"schema": "olm.bundle", "name": "test-bundle", "package": "test-package"}, ) - self.assertEqual(actual["test-package"]["olm.deprecations"]["test-package"], deprecation_blob) self.assertEqual(actual["test-package2"].keys(), {"olm.package", "olm.channel", "olm.bundle"}) self.assertEqual( actual["test-package2"]["olm.package"]["test-package2"], {"schema": "olm.package", "name": "test-package2"} diff --git a/doozer/tests/cli/test_images_streams.py b/doozer/tests/cli/test_images_streams.py index 378cc5c7f9..079e774318 100644 --- a/doozer/tests/cli/test_images_streams.py +++ b/doozer/tests/cli/test_images_streams.py @@ -278,70 +278,6 @@ def test_get_upstreaming_entries_with_final_user(mock_runtime, mock_image_meta): assert result['ose-test'].final_user == '1001' -# Tests for --image filtering in _get_upstreaming_entries - - -def test_get_upstreaming_entries_image_names_only(mock_runtime, mock_image_meta): - """Test filtering by image_names skips streams and only processes specified images.""" - mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' - mock_image_meta.pull_url.return_value = 'brew-registry.example.com/ci-openshift-base:latest' - mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} - - result = images_streams._get_upstreaming_entries(mock_runtime, image_names=['ci-openshift-base.rhel10']) - - assert len(result) == 1 - assert 'ci-openshift-base.rhel10' in result - mock_runtime.ordered_image_metas.assert_not_called() - mock_runtime.get_stream_names.assert_not_called() - - -def test_get_upstreaming_entries_image_not_found(mock_runtime): - """Test error when specified image_name is not in the group.""" - mock_runtime.image_map = {} - - with pytest.raises(IOError, match='Image nonexistent not found in group metadata'): - images_streams._get_upstreaming_entries(mock_runtime, image_names=['nonexistent']) - - -def test_get_upstreaming_entries_image_not_eligible(mocker, mock_runtime): - """Test error when image lacks ci_alignment.upstream_image.""" - ineligible_meta = mocker.MagicMock() - ineligible_meta.config.content.source.ci_alignment.upstream_image = Missing - mock_runtime.image_map = {'ose-cli': ineligible_meta} - - with pytest.raises(IOError, match='not eligible for CI sync'): - images_streams._get_upstreaming_entries(mock_runtime, image_names=['ose-cli']) - - -def test_get_upstreaming_entries_stream_and_image_union(mocker, mock_runtime, mock_image_meta): - """Test that providing both stream_names and image_names returns the union.""" - mock_runtime.streams = { - 'golang': Model({'upstream_image': 'registry.ci.openshift.org/ocp/4.17:golang'}), - } - mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' - mock_image_meta.pull_url.return_value = 'brew-registry.example.com/ci-openshift-base:latest' - mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} - - result = images_streams._get_upstreaming_entries( - mock_runtime, stream_names=['golang'], image_names=['ci-openshift-base.rhel10'] - ) - - assert len(result) == 2 - assert 'golang' in result - assert 'ci-openshift-base.rhel10' in result - - -def test_get_upstreaming_entries_image_build_not_found_skips(mocker, mock_runtime, mock_image_meta): - """Test that an image with no build is gracefully skipped.""" - mock_image_meta.distgit_key = 'ci-openshift-base.rhel10' - mock_image_meta.pull_url.side_effect = IOError('No build found') - mock_runtime.image_map = {'ci-openshift-base.rhel10': mock_image_meta} - - result = images_streams._get_upstreaming_entries(mock_runtime, image_names=['ci-openshift-base.rhel10']) - - assert result == {} - - # Tests for images:streams gen-buildconfigs command @@ -359,7 +295,7 @@ def test_gen_buildconfigs_uses_get_upstreaming_entries(): source = inspect.getsource(callback) # Verify it calls _get_upstreaming_entries - assert '_get_upstreaming_entries(runtime, streams' in source + assert '_get_upstreaming_entries(runtime, streams)' in source # Verify it uses the configuration fields from entries assert 'config.transform' in source diff --git a/doozer/tests/lockfile_prototype/test_generator.py b/doozer/tests/lockfile_prototype/test_generator.py index 36ad5b73f9..1526fe5530 100644 --- a/doozer/tests/lockfile_prototype/test_generator.py +++ b/doozer/tests/lockfile_prototype/test_generator.py @@ -13,7 +13,6 @@ from doozerlib.lockfile_prototype.container_utils import ContainerImageHelper from doozerlib.lockfile_prototype.generator import ( RpmLockfilePrototypeGenerator, - _is_local_rpm, build_rpms_in_yaml, ) from doozerlib.lockfile_prototype.models import ( @@ -110,52 +109,6 @@ def test_repo_options_flattened_in_serialization(self): self.assertNotIn("options", repo_dict) -class TestIsLocalRpm(unittest.TestCase): - def test_explicit_rpm_file(self): - self.assertTrue(_is_local_rpm("foo.rpm")) - self.assertTrue(_is_local_rpm("/tmp/bar-1.0.x86_64.rpm")) - - def test_path_glob(self): - self.assertTrue(_is_local_rpm("/path/to/*.rpm")) - self.assertTrue(_is_local_rpm("/opt/rpms/*")) - - def test_normal_packages(self): - self.assertFalse(_is_local_rpm("nfs-utils")) - self.assertFalse(_is_local_rpm("golang-*1.23*")) - self.assertFalse(_is_local_rpm("python3-six")) - - def test_build_rpms_in_yaml_filters_local_rpms(self): - """ - Local RPM file tokens extracted by the parser must be filtered - out before reaching rpm-lockfile-prototype. - """ - repos = [RepoEntry(repoid="baseos", baseurl="https://example.com/$basearch/")] - result = build_rpms_in_yaml( - repos=repos, - arches=["x86_64"], - packages=["nfs-utils", "/tmp/extras/*.rpm", "jq", "local.rpm"], - arch_specific_packages={"x86_64": ["librtas", "/opt/rpms/*"]}, - ) - pkg_names = [p if isinstance(p, str) else p.name for p in result.packages] - self.assertEqual(pkg_names, ["nfs-utils", "jq", "librtas"]) - - def test_build_rpms_in_yaml_filters_local_rpms_from_reinstall_and_upgrade(self): - """ - Local RPM tokens must also be filtered from reinstallPackages and - upgradePackages, not just from packages and arch_specific_packages. - """ - repos = [RepoEntry(repoid="baseos", baseurl="https://example.com/$basearch/")] - result = build_rpms_in_yaml( - repos=repos, - arches=["x86_64"], - packages=["curl"], - reinstall_packages=["curl", "/tmp/extras/foo.rpm", "glibc"], - upgrade_packages=["bash", "/opt/rpms/*", "openssl"], - ) - self.assertEqual(result.reinstallPackages, ["curl", "glibc"]) - self.assertEqual(result.upgradePackages, ["bash", "openssl"]) - - FAKE_LOCKFILE_DATA = LockfileData( lockfileVersion=1, lockfileVendor="redhat", @@ -588,13 +541,12 @@ async def capture_resolve(config, image_pullspec=None): self.assertIn("libreswan", pkg_names) self.assertIn("openssl", pkg_names) - def test_bare_update_keeps_image_mode_reinstalls_dockerfile_packages(self): + def test_bare_update_keeps_image_mode_without_upgrade_targets(self): """ Bare yum/dnf update with --image mode should NOT expand base image packages as upgrade targets (many are virtual provides or - renamed and cause resolution failures). Dockerfile install - packages must be reinstalled so they appear in the lockfile, and - also promoted to upgradePackages as a fallback if reinstall fails. + renamed and cause resolution failures). Instead, dnf update is + kept in the Dockerfile and runs at build time with cachi2 RPMs. """ meta = self._make_mock_image_meta() meta.config.konflux.cachi2.lockfile.get.return_value = None @@ -620,20 +572,17 @@ async def capture_resolve(config, image_pullspec=None): self.assertEqual(len(captured_configs), 1) # --image mode preserved self.assertIsNotNone(captured_pullspecs[0]) - # Dockerfile packages reinstalled so they appear in lockfile - self.assertEqual(captured_configs[0].reinstallPackages, ["libreswan"]) - # Dockerfile packages promoted to upgrade as reinstall fallback - self.assertEqual(captured_configs[0].upgradePackages, ["libreswan"]) + # No upgrade targets — dnf update handled at build time + self.assertEqual(captured_configs[0].upgradePackages, []) - def test_mixed_install_and_bare_update_reinstalls_dockerfile_packages(self): + def test_mixed_install_and_bare_update_skips_reinstall(self): """ When a stage has both explicit installs and a bare update (e.g. microdnf update -y && microdnf install -y openssl), - Dockerfile install packages must appear in reinstallPackages - so they end up in the lockfile even when already installed in - the base image. Base image packages must NOT be reinstalled — - they use upgrade semantics via upgradePackages to pick up - latest versions without pinning. + reinstallPackages must be empty. Otherwise reinstall pins base + image versions and overrides upgrade semantics, preventing the + update from picking up latest RPMs — causing a perpetual + rebuild loop when scan-sources detects outdated packages. """ meta = self._make_mock_image_meta() meta.config.konflux.cachi2.lockfile.get.return_value = None @@ -657,15 +606,15 @@ async def capture_resolve(config, image_pullspec=None): asyncio.run(generator.generate_lockfile(meta, dest_dir)) self.assertEqual(len(captured_configs), 1) - # Dockerfile install packages must be reinstalled so they appear - # in the lockfile even when already installed in the base image - self.assertEqual(captured_configs[0].reinstallPackages, ["openssl"]) - # Explicit install package must be present in packages list + # reinstallPackages must be empty: bare update means we want + # upgrade semantics, and reinstall would pin old versions + self.assertEqual(captured_configs[0].reinstallPackages, []) + # Explicit install package must be present pkg_names = [p if isinstance(p, str) else p.name for p in captured_configs[0].packages] self.assertIn("openssl", pkg_names) # Base image packages must appear as upgrade targets so # dnf update picks up latest versions from repos - self.assertEqual(sorted(captured_configs[0].upgradePackages), ["audit", "bash", "glibc", "openssl"]) + self.assertEqual(sorted(captured_configs[0].upgradePackages), ["audit", "bash", "glibc"]) def test_reinstall_packages_also_passed_as_upgrade_targets(self): """ @@ -739,40 +688,6 @@ async def mock_resolve(config, image_pullspec=None): pkg_names = [p if isinstance(p, str) else p.name for p in fallback_config.packages] self.assertIn("curl", pkg_names) - def test_fallback_sets_upgrades_dropped_flag(self): - """ - When the retry loop exhausts retries and the fallback clears - upgrade targets, generator.upgrades_dropped must be True so - the rebaser strips dnf update from the Dockerfile. - """ - meta = self._make_mock_image_meta() - meta.config.konflux.cachi2.lockfile.get.return_value = None - generator = self._make_generator() - generator.downstream_parents = ["quay.io/test/base@sha256:abc123"] - generator._container.get_installed_packages = AsyncMock(return_value=["bad-pkg", "glibc"]) - - call_count = 0 - - async def mock_resolve(config, image_pullspec=None): - nonlocal call_count - call_count += 1 - if call_count == 1: - raise RuntimeError("No match for argument: bad-pkg") - return FAKE_LOCKFILE_DATA.model_copy(deep=True) - - generator._resolver.resolve = AsyncMock(side_effect=mock_resolve) - - self.assertFalse(generator.upgrades_dropped) - - with TemporaryDirectory() as tmpdir: - dest_dir = Path(tmpdir) - (dest_dir / "Dockerfile").write_text("FROM base\nRUN dnf install -y curl\n") - asyncio.run(generator.generate_lockfile(meta, dest_dir)) - - self.assertTrue(generator.upgrades_dropped) - fallback_config = generator._resolver.resolve.call_args_list[-1][0][0] - self.assertEqual(fallback_config.upgradePackages, []) - def test_fallback_packages_used_when_image_unreachable(self): """ When base image is unreachable but fallback_installed has data @@ -1148,66 +1063,6 @@ async def mock_resolve(config, image_pullspec=None): for pkg in failing_pkgs: self.assertNotIn(pkg, final_config.reinstallPackages) - def test_fallback_disables_reinstall_to_upgrade_promotion(self): - """ - When the retry loop exhausts and the fallback fires, reinstall - packages must NOT be promoted to upgradePackages. Otherwise - base.upgrade() raises PackagesNotInstalledError for packages - that are in reinstall but not installed on all arches. - """ - generator = self._make_generator() - generator.downstream_parents = ["quay.io/test/base@sha256:abc123"] - - failing_pkgs = [f"base-pkg-{i}" for i in range(5)] - call_count = 0 - - async def mock_resolve(config, image_pullspec=None): - nonlocal call_count - call_count += 1 - if call_count <= 5: - raise RuntimeError(f"No match for argument: {failing_pkgs[call_count - 1]}") - return FAKE_LOCKFILE_DATA.model_copy(deep=True) - - generator._resolver.resolve = AsyncMock(side_effect=mock_resolve) - - repos = [ - RepoEntry( - repoid="rhel-9-baseos-rpms", - baseurl="https://example.com/baseos/$basearch/os/", - ) - ] - - # util-linux is a Dockerfile package AND a base image package. - # It must stay in reinstallPackages (graceful skip if missing) - # but NOT appear in upgradePackages (throws if not installed). - all_packages = ["util-linux", "curl"] + failing_pkgs - reinstall = ["util-linux", "curl"] + failing_pkgs - strippable = set(failing_pkgs) - - result = asyncio.run( - generator._resolve_stage_with_retry( - repo_list=repos, - arches=["x86_64", "aarch64"], - packages=all_packages, - arch_pkgs={}, - update_targets=[], - image_pullspec="quay.io/test/base@sha256:abc123", - distgit_key="ose-vmware-vsphere-csi-driver", - stage_num=0, - reinstall_packages=reinstall, - strippable_packages=strippable, - ) - ) - - self.assertIsNotNone(result) - self.assertEqual(call_count, 6) - fallback_config = generator._resolver.resolve.call_args_list[5][0][0] - # Required packages must stay in reinstallPackages - self.assertIn("util-linux", fallback_config.reinstallPackages) - self.assertIn("curl", fallback_config.reinstallPackages) - # upgradePackages must be empty — promotion disabled in fallback - self.assertEqual(fallback_config.upgradePackages, []) - def test_upgrade_targets_not_strippable_in_final_stage(self): """ Dockerfile update targets (e.g. 'yum update -y python3-six') that @@ -1755,212 +1610,22 @@ async def mock_gather(cmd, check=True, env=None): self.assertEqual(result, ["gcc", "make", "openssl-devel"]) - def test_spec_file_skipped_with_warning(self): + def test_matching_spec_file(self): gen = self._make_gen() with TemporaryDirectory() as tmpdir: - spec_path = Path(tmpdir) / "tuned.spec" + spec_path = Path(tmpdir) / "pkcs11-helper.spec" spec_path.touch() - result = asyncio.run(gen._resolve_builddep_packages(["tuned.spec"], Path(tmpdir), "test-img")) - self.assertEqual(result, []) - - -class TestExtractRhelVersionFromPullspec(unittest.TestCase): - def test_rhel_8_golang_tag(self): - ps = "registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.25-openshift-4.21" - self.assertEqual(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec(ps), 8) - - def test_rhel_9_golang_tag(self): - ps = "registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.25" - self.assertEqual(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec(ps), 9) - - def test_ubi_9_in_path_not_tag_returns_none(self): - ps = "registry.access.redhat.com/ubi9/ubi-minimal:latest" - self.assertIsNone(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec(ps)) - - def test_ubi_9_in_tag(self): - ps = "registry.access.redhat.com/ubi9/ubi-minimal:ubi-9-minimal" - self.assertEqual(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec(ps), 9) - def test_nvr_el8_tag(self): - ps = "registry.redhat.io/openshift/art-images-base:openshift-golang-builder-container-v1.25.9-202605121249.p2.g2aa6a05.el8" - self.assertEqual(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec(ps), 8) - - def test_nvr_el9_tag(self): - ps = "registry.redhat.io/openshift/art-images-base:openshift-golang-builder-container-v1.25.9-202605121249.p2.g2aa6a05.el9" - self.assertEqual(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec(ps), 9) - - def test_digest_only_returns_none(self): - ps = "quay.io/test/builder@sha256:abc123def456" - self.assertIsNone(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec(ps)) - - def test_no_colon_returns_none(self): - self.assertIsNone(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec("builder_stage")) - - def test_unrecognized_tag_returns_none(self): - ps = "quay.io/test/builder:latest" - self.assertIsNone(RpmLockfilePrototypeGenerator._extract_rhel_version_from_pullspec(ps)) - - -class TestExtractRhelVersionFromRepos(unittest.TestCase): - def test_rhel9_baseos_content_set(self): - repos = [ - RepoEntry(repoid="rhel-9-for-x86_64-baseos-e4s-rpms__9_DOT_6", baseurl="https://example.com/baseos/"), - ] - self.assertEqual(RpmLockfilePrototypeGenerator._extract_rhel_version_from_repos(repos), 9) - - def test_rhel8_content_set(self): - repos = [ - RepoEntry(repoid="rhel-8-for-x86_64-baseos-rpms", baseurl="https://example.com/baseos/"), - ] - self.assertEqual(RpmLockfilePrototypeGenerator._extract_rhel_version_from_repos(repos), 8) - - def test_no_rhel_in_repoid_returns_none(self): - repos = [ - RepoEntry(repoid="custom-repo-rpms", baseurl="https://example.com/custom/"), - ] - self.assertIsNone(RpmLockfilePrototypeGenerator._extract_rhel_version_from_repos(repos)) - - def test_empty_repos_returns_none(self): - self.assertIsNone(RpmLockfilePrototypeGenerator._extract_rhel_version_from_repos([])) - - def test_first_rhel_repo_wins(self): - repos = [ - RepoEntry(repoid="custom-repo", baseurl="https://example.com/custom/"), - RepoEntry(repoid="rhel-9-for-x86_64-appstream-rpms", baseurl="https://example.com/appstream/"), - ] - self.assertEqual(RpmLockfilePrototypeGenerator._extract_rhel_version_from_repos(repos), 9) - - -class TestHasRhelVersionMismatch(unittest.TestCase): - def _make_generator(self) -> RpmLockfilePrototypeGenerator: - repos = MagicMock() - return RpmLockfilePrototypeGenerator( - repos=repos, - working_dir=Path(tempfile.mkdtemp()), - container_helper=MagicMock(spec=ContainerImageHelper), - resolver=MagicMock(spec=RpmResolver), - ) - - def test_el8_builder_el9_repos_is_mismatch(self): - gen = self._make_generator() - gen.downstream_parents = [ - "registry.redhat.io/openshift/art-images-base:openshift-golang-builder-container-v1.25.9.el8", - "quay.io/test/base:rhel-9", - ] - repos = [RepoEntry(repoid="rhel-9-for-x86_64-baseos-rpms", baseurl="https://example.com/")] - self.assertTrue(gen._has_rhel_version_mismatch(0, repos, "test-img")) - - def test_el9_builder_el9_repos_no_mismatch(self): - gen = self._make_generator() - gen.downstream_parents = [ - "registry.ci.openshift.org/ocp/builder:rhel-9-golang-1.25", - ] - repos = [RepoEntry(repoid="rhel-9-for-x86_64-baseos-rpms", baseurl="https://example.com/")] - self.assertFalse(gen._has_rhel_version_mismatch(0, repos, "test-img")) - - def test_undetectable_builder_returns_false(self): - gen = self._make_generator() - gen.downstream_parents = [ - "quay.io/test/builder@sha256:abc123", - ] - repos = [RepoEntry(repoid="rhel-9-for-x86_64-baseos-rpms", baseurl="https://example.com/")] - self.assertFalse(gen._has_rhel_version_mismatch(0, repos, "test-img")) - - def test_undetectable_repos_returns_false(self): - gen = self._make_generator() - gen.downstream_parents = [ - "registry.ci.openshift.org/ocp/builder:rhel-8-golang-1.25", - ] - repos = [RepoEntry(repoid="custom-repo", baseurl="https://example.com/")] - self.assertFalse(gen._has_rhel_version_mismatch(0, repos, "test-img")) - - def test_stage_alias_returns_false(self): - gen = self._make_generator() - gen.downstream_parents = ["builder_stage"] - repos = [RepoEntry(repoid="rhel-9-for-x86_64-baseos-rpms", baseurl="https://example.com/")] - self.assertFalse(gen._has_rhel_version_mismatch(0, repos, "test-img")) - - def test_out_of_range_stage_returns_false(self): - gen = self._make_generator() - gen.downstream_parents = [] - repos = [RepoEntry(repoid="rhel-9-for-x86_64-baseos-rpms", baseurl="https://example.com/")] - self.assertFalse(gen._has_rhel_version_mismatch(5, repos, "test-img")) - - -class TestRhelMismatchEndToEnd(unittest.TestCase): - """ - End-to-end: builder stage with el8 pullspec + el9 repos should skip - base image packages and resolve only Dockerfile packages in bare mode. - """ - - def _make_mock_repos(self) -> MagicMock: - repos = MagicMock() - baseos = MagicMock() - baseos.name = "rhel-9-baseos-rpms" - baseos.baseurl.return_value = "https://example.com/baseos/x86_64/os/" - baseos.content_set.return_value = "rhel-9-for-x86_64-baseos-rpms" - baseos.cs_optional = False - baseos._data.conf.get.return_value = {} - repo_map = {"rhel-9-baseos-rpms": baseos} - repos.__getitem__ = lambda self_repos, key: repo_map[key] - return repos - - def _make_mock_image_meta(self) -> MagicMock: - meta = MagicMock() - meta.distgit_key = "hive" - meta.get_arches.return_value = ["x86_64"] - meta.get_enabled_repos.return_value = {"rhel-9-baseos-rpms"} - meta.is_lockfile_generation_enabled.return_value = True - lockfile_config = MagicMock() - lockfile_config.get.return_value = None - meta.config.konflux.cachi2.lockfile = lockfile_config - return meta - - def test_el8_builder_skips_base_image_packages(self): - container = MagicMock(spec=ContainerImageHelper) - container.resolve_to_digest = AsyncMock(side_effect=lambda p: p.split(":")[0] + "@sha256:abc123") - container.get_installed_packages = AsyncMock(return_value=["gcc", "glibc", "readline"]) - container.read_file_from_image = AsyncMock(return_value="") - - resolver = MagicMock(spec=RpmResolver) - resolver.resolve = AsyncMock(return_value=FAKE_LOCKFILE_DATA.model_copy(deep=True)) + async def mock_gather(cmd, check=True, env=None): + return 0, "gcc\nmake\n", "" - generator = RpmLockfilePrototypeGenerator( - repos=self._make_mock_repos(), - working_dir=Path(tempfile.mkdtemp()), - container_helper=container, - resolver=resolver, - ) - generator.downstream_parents = [ - "registry.redhat.io/openshift/art-images-base:golang-builder-v1.25.el8", - "quay.io/test/base:rhel-9-base", - ] + import doozerlib.lockfile_prototype.generator as gen_mod - with tempfile.TemporaryDirectory() as tmpdir: - dest_dir = Path(tmpdir) - (dest_dir / "Dockerfile").write_text( - "FROM golang-builder AS builder_el8\n" - "RUN dnf install -y subscription-manager\n" - "\n" - "FROM base-rhel9\n" - "COPY --from=builder_el8 /bin/app /usr/bin/app\n" - ) - asyncio.run(generator.generate_lockfile(self._make_mock_image_meta(), dest_dir)) - - # Resolver should have been called for stage 0 in bare mode - # (image_pullspec=None) because of RHEL mismatch - calls = resolver.resolve.call_args_list - stage0_call = calls[0] - self.assertIsNone( - stage0_call.kwargs.get("image_pullspec"), - "Stage 0 should use bare mode (image_pullspec=None) due to RHEL mismatch", - ) + original = gen_mod.cmd_gather_async + gen_mod.cmd_gather_async = mock_gather + try: + result = asyncio.run(gen._resolve_builddep_packages(["pkcs11-helper*"], Path(tmpdir), "test-img")) + finally: + gen_mod.cmd_gather_async = original - # Base image packages should NOT have been added to the install - # list — only the Dockerfile package should be present - config = stage0_call.args[0] - pkg_names = [p if isinstance(p, str) else p.name for p in config.packages] - self.assertIn("subscription-manager", pkg_names) - self.assertNotIn("gcc", pkg_names, "Base image packages should not be in install list") - self.assertNotIn("glibc", pkg_names, "Base image packages should not be in install list") - self.assertNotIn("readline", pkg_names, "Base image packages should not be in install list") + self.assertEqual(result, ["gcc", "make"]) diff --git a/doozer/tests/lockfile_prototype/test_resolver.py b/doozer/tests/lockfile_prototype/test_resolver.py index 1f83843bb8..fa61429507 100644 --- a/doozer/tests/lockfile_prototype/test_resolver.py +++ b/doozer/tests/lockfile_prototype/test_resolver.py @@ -200,26 +200,6 @@ def test_packages_not_installed_error_format(self): missing = RpmResolver.parse_missing_packages(error) self.assertEqual(missing, {"policycoreutils-python-utils"}) - def test_reinstall_not_available_format(self): - """ - DNF PackagesNotAvailableError from base.reinstall() outputs - "no package matched: " when the installed version is not - in the configured repos. - """ - error = "dnf.exceptions.PackagesNotAvailableError: no package matched: git" - missing = RpmResolver.parse_missing_packages(error) - self.assertEqual(missing, {"git"}) - - def test_glob_pattern_in_missing_packages(self): - """ - DNF glob patterns like *-server-ose* can appear in missing packages - errors. VALID_PKG_NAME must accept wildcards so the retry loop can - strip them. - """ - error = "missing packages: *-server-ose*" - missing = RpmResolver.parse_missing_packages(error) - self.assertEqual(missing, {"*-server-ose*"}) - def test_no_match(self): error = "Some other error message\n" missing = RpmResolver.parse_missing_packages(error) @@ -239,15 +219,6 @@ def test_detects_failed_loading_rpmdb(self): stderr = "OSError: failed loading RPMDB\n" self.assertTrue(RpmResolver._is_rpmdb_corrupt(stderr)) - def test_detects_no_such_file(self): - stderr = ( - "shutil.Error: [('/home/jenkins/.cache/rpm-lockfile-prototype/rpmdbs/ppc64le/" - "sha256:abc123/var/lib/rpm/rpmdb.sqlite', '/tmp/xyz/var/lib/rpm/rpmdb.sqlite', " - "\"[Errno 2] No such file or directory: '/home/jenkins/.cache/rpm-lockfile-prototype/" - "rpmdbs/ppc64le/sha256:abc123/var/lib/rpm/rpmdb.sqlite'\")]" - ) - self.assertTrue(RpmResolver._is_rpmdb_corrupt(stderr)) - def test_no_false_positive(self): stderr = "No match for argument: foo\n" self.assertFalse(RpmResolver._is_rpmdb_corrupt(stderr)) @@ -365,44 +336,6 @@ async def mock_cmd(cmd, **kwargs): self.assertEqual(call_count, 2) mock_clear.assert_called_once() - @patch("doozerlib.lockfile_prototype.resolver.RpmResolver._clear_rpmdb_cache") - @patch("doozerlib.lockfile_prototype.resolver.cmd_gather_async") - def test_retries_when_cache_already_gone(self, mock_gather, mock_clear): - """ - Cache deleted by another process (_clear returns False) — should still retry. - """ - cache_race_stderr = ( - "shutil.Error: [('/home/jenkins/.cache/rpm-lockfile-prototype/rpmdbs/ppc64le/" - "sha256:abc123/var/lib/rpm/rpmdb.sqlite', '/tmp/xyz/var/lib/rpm/rpmdb.sqlite', " - "\"[Errno 2] No such file or directory: '/home/jenkins/.cache/rpm-lockfile-prototype/" - "rpmdbs/ppc64le/sha256:abc123/var/lib/rpm/rpmdb.sqlite'\")]" - ) - call_count = 0 - - async def mock_cmd(cmd, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - return (1, "", cache_race_stderr) - outfile_idx = cmd.index("--outfile") + 1 - with open(cmd[outfile_idx], "w") as f: - yaml.safe_dump(self.FAKE_LOCKFILE_DATA, f) - return (0, "", "") - - mock_gather.side_effect = mock_cmd - mock_clear.return_value = False - - resolver = RpmResolver(working_dir=Path(tempfile.mkdtemp())) - config = RpmsInConfig( - arches=["x86_64"], - contentOrigin={"repos": []}, - packages=[], - ) - result = asyncio.run(resolver.resolve(config, image_pullspec="registry.example.com/repo@sha256:abc123")) - self.assertIsInstance(result, LockfileData) - self.assertEqual(call_count, 2) - mock_clear.assert_called_once() - @patch("doozerlib.lockfile_prototype.resolver.RpmResolver._clear_rpmdb_cache") @patch("doozerlib.lockfile_prototype.resolver.cmd_gather_async") def test_raises_after_retry_fails(self, mock_gather, mock_clear): @@ -469,34 +402,3 @@ async def mock_fail(cmd, **kwargs): with self.assertRaises(RuntimeError): asyncio.run(resolver.resolve(config, image_pullspec="registry.example.com/repo@sha256:abc123")) mock_clear.assert_not_called() - - @patch("doozerlib.lockfile_prototype.resolver.RpmResolver._clear_rpmdb_cache") - @patch("doozerlib.lockfile_prototype.resolver.cmd_gather_async") - def test_retry_raises_retry_error_not_original(self, mock_gather, mock_clear): - """ - When cache corruption triggers a retry and the retry fails with a - different error, the raised RuntimeError must contain the retry - stderr so the outer retry loop can parse it. - """ - retry_stderr = "dnf.exceptions.PackagesNotInstalledError: No match for argument: nfs-utils: nfs-utils" - call_count = 0 - - async def mock_cmd(cmd, **kwargs): - nonlocal call_count - call_count += 1 - if call_count == 1: - return (1, "", self.CORRUPTION_STDERR) - return (1, "", retry_stderr) - - mock_gather.side_effect = mock_cmd - mock_clear.return_value = True - - resolver = RpmResolver(working_dir=Path(tempfile.mkdtemp())) - config = RpmsInConfig( - arches=["x86_64"], - contentOrigin={"repos": []}, - packages=[], - ) - with self.assertRaises(RuntimeError) as ctx: - asyncio.run(resolver.resolve(config, image_pullspec="registry.example.com/repo@sha256:abc123")) - self.assertIn("nfs-utils", str(ctx.exception)) diff --git a/doozer/tests/lockfile_prototype/test_shell_parser.py b/doozer/tests/lockfile_prototype/test_shell_parser.py index c1c18e3df9..4c3d251c26 100644 --- a/doozer/tests/lockfile_prototype/test_shell_parser.py +++ b/doozer/tests/lockfile_prototype/test_shell_parser.py @@ -133,41 +133,12 @@ def test_subshell_arch_conditional_var(self): "yum -y install pciutils hwdata kmod $ARCH_DEP_PKGS" ] common, arch = extract_packages_from_run_commands(run_values) - self.assertNotIn("mstflint", common) + self.assertIn("mstflint", common) self.assertIn("pciutils", common) self.assertIn("hwdata", common) self.assertIn("kmod", common) - for a in ("x86_64", "aarch64", "ppc64le"): - self.assertIn("mstflint", arch.get(a, [])) - self.assertNotIn("s390x", arch) - - def test_subshell_arch_conditional_var_eq(self): - run_values = [ - 'SPECIAL=$(if [ "$(uname -m)" == "x86_64" ]; then echo -n intel-pkg ; fi) && ' - "yum -y install base-pkg $SPECIAL" - ] - common, arch = extract_packages_from_run_commands(run_values) - self.assertNotIn("intel-pkg", common) - self.assertIn("base-pkg", common) - self.assertEqual(arch, {"x86_64": ["intel-pkg"]}) - - def test_subshell_no_arch_condition(self): - run_values = ['EXTRA=$(echo extra-pkg) && yum -y install base-pkg $EXTRA'] - common, arch = extract_packages_from_run_commands(run_values) - self.assertIn("extra-pkg", common) - self.assertIn("base-pkg", common) self.assertEqual(arch, {}) - def test_subshell_arch_conditional_var_go_arch(self): - run_values = [ - 'SPECIAL=$(if [ "$(go env GOARCH)" == "amd64" ]; then echo -n x86-only ; fi) && ' - "yum -y install common-pkg $SPECIAL" - ] - common, arch = extract_packages_from_run_commands(run_values) - self.assertNotIn("x86-only", common) - self.assertIn("common-pkg", common) - self.assertEqual(arch, {"x86_64": ["x86-only"]}) - def test_if_else_var_with_nested_resolution(self): run_values = [ 'if yum install -y iotop >/dev/null 2>&1; then IOTOP_PKG="iotop"; ' @@ -584,17 +555,6 @@ def test_builddep_spec_file(self): _, _, _, _, builddep, _ = analyze_run_commands(run_values) self.assertEqual(builddep, ["mypackage.spec"]) - def test_build_dep_hyphenated(self): - run_values = ["dnf build-dep tuned.spec -y"] - _, _, _, _, builddep, _ = analyze_run_commands(run_values) - self.assertEqual(builddep, ["tuned.spec"]) - - def test_build_dep_hyphenated_with_install(self): - run_values = ["dnf install -y gcc rpm-build && cd assets/tuned/daemon && dnf build-dep tuned.spec -y"] - common, _, _, _, builddep, _ = analyze_run_commands(run_values) - self.assertIn("gcc", common) - self.assertEqual(builddep, ["tuned.spec"]) - class TestModuleParsing(unittest.TestCase): def test_module_install(self): @@ -622,25 +582,3 @@ def test_module_without_stream_ignored(self): run_values = ["dnf module enable -y nodejs"] _, _, _, _, _, modules = analyze_run_commands(run_values) self.assertEqual(modules, []) - - def test_variable_package_manager(self): - run_values = ["${DNF} install -y openssh-clients"] - pkgs, _, _, _, _, _ = analyze_run_commands(run_values, env_vars={"DNF": "microdnf"}) - self.assertEqual(pkgs, ["openssh-clients"]) - - def test_variable_package_manager_in_conditional(self): - run_values = [ - "if ! rpm -q openssh-clients; then ${DNF} install -y openssh-clients " - "&& ${DNF} clean all && rm -rf /var/cache/dnf/*; fi" - ] - pkgs, _, _, _, _, _ = analyze_run_commands(run_values, env_vars={"DNF": "microdnf"}) - self.assertEqual(pkgs, ["openssh-clients"]) - - def test_variable_package_manager_multiple_commands(self): - run_values = [ - "if ! rpm -q openssh-clients; then ${DNF} install -y openssh-clients && ${DNF} clean all; fi", - "if ! rpm -q libvirt-libs; then ${DNF} install -y libvirt-libs && ${DNF} clean all; fi", - "if ! command -v tar; then ${DNF} install -y tar && ${DNF} clean all; fi", - ] - pkgs, _, _, _, _, _ = analyze_run_commands(run_values, env_vars={"DNF": "microdnf"}) - self.assertEqual(pkgs, ["libvirt-libs", "openssh-clients", "tar"]) diff --git a/doozer/tests/test_assembly_inspector.py b/doozer/tests/test_assembly_inspector.py index ba5adc15b2..2684a2be08 100644 --- a/doozer/tests/test_assembly_inspector.py +++ b/doozer/tests/test_assembly_inspector.py @@ -79,27 +79,4 @@ def test_check_installed_rpms_in_image(self): issues = ai.check_installed_rpms_in_image("foo", build_inspector, None) self.assertEqual(issues, []) - def test_is_installed_rpm_in_tag(self): - rt = MagicMock(mode="both", group_config=Model({})) - brew_session = MagicMock() - ai = AssemblyInspector(rt, brew_session) - - # None installed_rpm returns False - self.assertFalse(ai._is_installed_rpm_in_tag(None, "my-candidate-tag")) - - # Installed RPM is tagged in the candidate tag - brew_session.listTags.return_value = [ - {"name": "my-candidate-tag"}, - {"name": "other-tag"}, - ] - installed_rpm = {"id": 100, "nvr": "cri-o-1.28.0-1.el9"} - self.assertTrue(ai._is_installed_rpm_in_tag(installed_rpm, "my-candidate-tag")) - - # Installed RPM is NOT tagged in the candidate tag - brew_session.listTags.return_value = [ - {"name": "rhel-9.2.0-z"}, - {"name": "RHSA-2026:13971-released"}, - ] - self.assertFalse(ai._is_installed_rpm_in_tag(installed_rpm, "my-candidate-tag")) - pass diff --git a/elliott/elliottlib/cli/process_release_from_fbc_bugs_cli.py b/elliott/elliottlib/cli/process_release_from_fbc_bugs_cli.py index 6e129c74e2..2968a30450 100644 --- a/elliott/elliottlib/cli/process_release_from_fbc_bugs_cli.py +++ b/elliott/elliottlib/cli/process_release_from_fbc_bugs_cli.py @@ -6,10 +6,9 @@ import click from artcommonlib.jira_config import JIRA_SERVER_URL from artcommonlib.util import new_roundtrip_yaml_handler -from doozerlib.util import konflux_application_name, konflux_image_component_name from elliottlib.bzutil import JIRABugTracker -from elliottlib.cli.attach_cve_flaws_cli import _image_uses_builder, get_konflux_component_by_component +from elliottlib.cli.attach_cve_flaws_cli import get_konflux_component_by_component from elliottlib.cli.common import cli, click_coroutine from elliottlib.runtime import Runtime from elliottlib.shipment_model import CveAssociation, ReleaseNotes @@ -19,10 +18,6 @@ YAML = new_roundtrip_yaml_handler() logger = logging.getLogger(__name__) -GOLANG_BUILDER_DELIVERY_REPO = "openshift4/openshift-golang-builder" -GOLANG_BUILDER_COMPONENT = "openshift-golang-builder-container" -GOLANG_BUILDER_PSCOMPONENTS = {GOLANG_BUILDER_DELIVERY_REPO, GOLANG_BUILDER_COMPONENT} - def _create_jira_tracker() -> JIRABugTracker: """Create a JIRABugTracker with minimal config, bypassing bug.yml from ocp-build-data. @@ -60,16 +55,6 @@ def _extract_pscomponent_from_labels(labels: List[str]) -> Optional[str]: return None -def _get_golang_builder_components(runtime) -> list[str]: - """Find all Konflux component names in the group that use the golang builder.""" - application = konflux_application_name(runtime.group) - components = [] - for image_meta in runtime.image_metas(): - if _image_uses_builder(image_meta, logger): - components.append(konflux_image_component_name(application, image_meta.distgit_key)) - return sorted(components) - - async def process_bugs(runtime: Runtime, jira_ids: List[str]) -> ReleaseNotes: """Process a list of JIRA IDs and produce a ReleaseNotes object. @@ -89,7 +74,6 @@ async def process_bugs(runtime: Runtime, jira_ids: List[str]) -> ReleaseNotes: cve_associations: list[CveAssociation] = [] flaw_bug_ids: list[int] = [] all_jira_ids: list[str] = [] - cve_mapping_errors: list[str] = [] for jira_id in jira_ids: jira_id = jira_id.strip() @@ -110,58 +94,32 @@ async def process_bugs(runtime: Runtime, jira_ids: List[str]) -> ReleaseNotes: cve_id = _extract_cve_id_from_labels(labels) if not cve_id: - msg = f"JIRA {jira_id} is Vulnerability but has no CVE label" - logger.error(msg) - cve_mapping_errors.append(msg) + logger.warning("JIRA %s is Vulnerability but has no CVE label, skipping CVE association", jira_id) continue pscomponent = _extract_pscomponent_from_labels(labels) if not pscomponent: - msg = f"JIRA {jira_id} (CVE {cve_id}) has no pscomponent label" - logger.error(msg) - cve_mapping_errors.append(msg) + logger.warning("JIRA %s (CVE %s) has no pscomponent label, skipping CVE association", jira_id, cve_id) continue distgit_component = get_component_by_delivery_repo(runtime, pscomponent) if not distgit_component: - if pscomponent in GOLANG_BUILDER_PSCOMPONENTS: - builder_components = _get_golang_builder_components(runtime) - if not builder_components: - msg = f"JIRA {jira_id} (CVE {cve_id}): golang builder CVE but no components use the builder" - logger.error(msg) - cve_mapping_errors.append(msg) - continue - logger.info( - "JIRA %s: CVE %s is a golang builder CVE, fanning out to %d components: %s", - jira_id, - cve_id, - len(builder_components), - builder_components, - ) - for comp in builder_components: - cve_associations.append(CveAssociation(key=cve_id, component=comp)) - else: - msg = f"JIRA {jira_id} (CVE {cve_id}): pscomponent '{pscomponent}' not found in delivery_repo_names" - logger.error(msg) - cve_mapping_errors.append(msg) - continue - - bug_flaw_ids = _extract_flaw_bug_ids_from_labels(labels) - if bug_flaw_ids: - logger.info("JIRA %s: found flaw bug IDs %s", jira_id, bug_flaw_ids) - flaw_bug_ids.extend(bug_flaw_ids) - else: - logger.warning("JIRA %s (CVE %s) has no flaw:bz# labels", jira_id, cve_id) + logger.warning( + "JIRA %s (CVE %s): pscomponent '%s' not found in delivery_repo_names, skipping", + jira_id, + cve_id, + pscomponent, + ) continue konflux_component = get_konflux_component_by_component(runtime, distgit_component) if not konflux_component: - msg = ( - f"JIRA {jira_id} (CVE {cve_id}): distgit component '{distgit_component}' " - f"could not be mapped to a Konflux component" + logger.warning( + "JIRA %s (CVE %s): distgit component '%s' could not be mapped to a Konflux component, skipping", + jira_id, + cve_id, + distgit_component, ) - logger.error(msg) - cve_mapping_errors.append(msg) continue logger.info( @@ -181,12 +139,6 @@ async def process_bugs(runtime: Runtime, jira_ids: List[str]) -> ReleaseNotes: else: logger.warning("JIRA %s (CVE %s) has no flaw:bz# labels", jira_id, cve_id) - if cve_mapping_errors: - raise RuntimeError( - f"Failed to map {len(cve_mapping_errors)} CVE(s) to components:\n" - + "\n".join(f" - {e}" for e in cve_mapping_errors) - ) - advisory_type = "RHSA" if cve_associations else "RHBA" logger.info("Advisory type determined: %s (CVEs found: %d)", advisory_type, len(cve_associations)) diff --git a/elliott/tests/test_process_release_from_fbc_bugs_cli.py b/elliott/tests/test_process_release_from_fbc_bugs_cli.py index 1642209fd0..81b04f5f25 100644 --- a/elliott/tests/test_process_release_from_fbc_bugs_cli.py +++ b/elliott/tests/test_process_release_from_fbc_bugs_cli.py @@ -3,7 +3,6 @@ from artcommonlib.jira_config import JIRA_DOMAIN_NAME from elliottlib.cli.process_release_from_fbc_bugs_cli import ( - GOLANG_BUILDER_DELIVERY_REPO, _extract_cve_id_from_labels, _extract_flaw_bug_ids_from_labels, _extract_pscomponent_from_labels, @@ -13,7 +12,6 @@ PATCH_CREATE_TRACKER = "elliottlib.cli.process_release_from_fbc_bugs_cli._create_jira_tracker" PATCH_GET_DELIVERY_REPO = "elliottlib.cli.process_release_from_fbc_bugs_cli.get_component_by_delivery_repo" PATCH_GET_KONFLUX_COMPONENT = "elliottlib.cli.process_release_from_fbc_bugs_cli.get_konflux_component_by_component" -PATCH_GET_BUILDER_COMPONENTS = "elliottlib.cli.process_release_from_fbc_bugs_cli._get_golang_builder_components" class TestLabelExtraction(unittest.TestCase): @@ -148,8 +146,8 @@ async def test_mixed_cve_and_regular_bugs(self, mock_create_tracker, mock_get_de @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) @patch(PATCH_CREATE_TRACKER) - async def test_cve_without_cve_label_raises(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """A Vulnerability JIRA without a CVE label should raise RuntimeError.""" + async def test_cve_without_cve_label_skipped(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): + """A Vulnerability JIRA without a CVE label should be skipped for CVE association but still listed.""" bug = self._make_jira_bug( "OADP-9999", is_vulnerability=True, @@ -160,17 +158,22 @@ async def test_cve_without_cve_label_raises(self, mock_create_tracker, mock_get_ mock_tracker.get_bug.return_value = bug mock_create_tracker.return_value = mock_tracker - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-9999"]) + result = await process_bugs(self.mock_runtime, ["OADP-9999"]) - self.assertIn("OADP-9999", str(ctx.exception)) - self.assertIn("no CVE label", str(ctx.exception)) + self.assertEqual(result.type, "RHBA") + self.assertIsNone(result.cves) + fixed_ids = [i.id for i in result.issues.fixed] + self.assertIn("OADP-9999", fixed_ids) + mock_get_delivery.assert_not_called() + mock_get_konflux.assert_not_called() @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) @patch(PATCH_CREATE_TRACKER) - async def test_cve_without_pscomponent_label_raises(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """A Vulnerability with CVE label but no pscomponent label should raise RuntimeError.""" + async def test_cve_without_pscomponent_label_skipped( + self, mock_create_tracker, mock_get_delivery, mock_get_konflux + ): + """A Vulnerability with CVE label but no pscomponent label should skip CVE association.""" bug = self._make_jira_bug( "OADP-8888", is_vulnerability=True, @@ -181,17 +184,18 @@ async def test_cve_without_pscomponent_label_raises(self, mock_create_tracker, m mock_tracker.get_bug.return_value = bug mock_create_tracker.return_value = mock_tracker - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-8888"]) + result = await process_bugs(self.mock_runtime, ["OADP-8888"]) - self.assertIn("OADP-8888", str(ctx.exception)) - self.assertIn("no pscomponent label", str(ctx.exception)) + self.assertEqual(result.type, "RHBA") + self.assertIsNone(result.cves) + mock_get_delivery.assert_not_called() + mock_get_konflux.assert_not_called() @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) @patch(PATCH_CREATE_TRACKER) - async def test_unmapped_delivery_repo_raises(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """When pscomponent can't be found in delivery_repo_names, should raise RuntimeError.""" + async def test_unmapped_delivery_repo_skipped(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): + """When pscomponent can't be found in delivery_repo_names, CVE association is skipped.""" mock_get_delivery.return_value = None bug = self._make_jira_bug( @@ -204,17 +208,20 @@ async def test_unmapped_delivery_repo_raises(self, mock_create_tracker, mock_get mock_tracker.get_bug.return_value = bug mock_create_tracker.return_value = mock_tracker - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-7777"]) + result = await process_bugs(self.mock_runtime, ["OADP-7777"]) - self.assertIn("OADP-7777", str(ctx.exception)) - self.assertIn("unknown/unknown-container", str(ctx.exception)) + self.assertEqual(result.type, "RHBA") + self.assertIsNone(result.cves) + fixed_ids = [i.id for i in result.issues.fixed] + self.assertIn("OADP-7777", fixed_ids) + mock_get_delivery.assert_called_once_with(self.mock_runtime, "unknown/unknown-container") + mock_get_konflux.assert_not_called() @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) @patch(PATCH_CREATE_TRACKER) - async def test_unmapped_konflux_component_raises(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """When distgit component can't be mapped to a Konflux component, should raise RuntimeError.""" + async def test_unmapped_konflux_component_skipped(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): + """When distgit component can't be mapped to a Konflux component, CVE association is skipped.""" mock_get_delivery.return_value = "some-container" mock_get_konflux.return_value = None @@ -228,148 +235,14 @@ async def test_unmapped_konflux_component_raises(self, mock_create_tracker, mock mock_tracker.get_bug.return_value = bug mock_create_tracker.return_value = mock_tracker - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-7777"]) - - self.assertIn("OADP-7777", str(ctx.exception)) - self.assertIn("some-container", str(ctx.exception)) - - @patch(PATCH_GET_KONFLUX_COMPONENT) - @patch(PATCH_GET_DELIVERY_REPO) - @patch(PATCH_CREATE_TRACKER) - async def test_multiple_mapping_errors_all_reported(self, mock_create_tracker, mock_get_delivery, mock_get_konflux): - """All CVE mapping errors should be collected and reported in a single RuntimeError.""" - mock_get_delivery.return_value = None + result = await process_bugs(self.mock_runtime, ["OADP-7777"]) - bug1 = self._make_jira_bug( - "OADP-1111", - is_vulnerability=True, - labels=["CVE-2025-00001", "pscomponent:unknown/repo-a"], - ) - bug2 = self._make_jira_bug( - "OADP-2222", - is_vulnerability=True, - labels=["CVE-2025-00002", "pscomponent:unknown/repo-b"], - ) - - mock_tracker = Mock() - mock_tracker.get_bug.side_effect = lambda k: {"OADP-1111": bug1, "OADP-2222": bug2}[k] - mock_create_tracker.return_value = mock_tracker - - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-1111", "OADP-2222"]) - - error_msg = str(ctx.exception) - self.assertIn("2 CVE(s)", error_msg) - self.assertIn("OADP-1111", error_msg) - self.assertIn("OADP-2222", error_msg) - - @patch(PATCH_GET_BUILDER_COMPONENTS) - @patch(PATCH_GET_KONFLUX_COMPONENT) - @patch(PATCH_GET_DELIVERY_REPO) - @patch(PATCH_CREATE_TRACKER) - async def test_golang_builder_cve_fans_out( - self, mock_create_tracker, mock_get_delivery, mock_get_konflux, mock_get_builder - ): - """A golang builder CVE should fan out to all components using the builder.""" - mock_get_delivery.return_value = None - mock_get_builder.return_value = ["oadp-1-4-oadp-operator", "oadp-1-4-oadp-velero"] - - bug = self._make_jira_bug( - "OADP-7902", - is_vulnerability=True, - labels=[ - "CVE-2026-32281", - f"pscomponent:{GOLANG_BUILDER_DELIVERY_REPO}", - "flaw:bz#2456333", - ], - ) - - mock_tracker = Mock() - mock_tracker.get_bug.return_value = bug - mock_create_tracker.return_value = mock_tracker - - result = await process_bugs(self.mock_runtime, ["OADP-7902"]) - - self.assertEqual(result.type, "RHSA") - self.assertEqual(len(result.cves), 2) - cve_components = {c.component for c in result.cves} - self.assertEqual(cve_components, {"oadp-1-4-oadp-operator", "oadp-1-4-oadp-velero"}) - for cve in result.cves: - self.assertEqual(cve.key, "CVE-2026-32281") - - flaw_ids = {i.id for i in result.issues.fixed if i.source == "bugzilla.redhat.com"} - self.assertIn("2456333", flaw_ids) - - @patch(PATCH_GET_BUILDER_COMPONENTS) - @patch(PATCH_GET_KONFLUX_COMPONENT) - @patch(PATCH_GET_DELIVERY_REPO) - @patch(PATCH_CREATE_TRACKER) - async def test_golang_builder_cve_no_consumers_raises( - self, mock_create_tracker, mock_get_delivery, mock_get_konflux, mock_get_builder - ): - """A golang builder CVE with no builder-consuming components should raise RuntimeError.""" - mock_get_delivery.return_value = None - mock_get_builder.return_value = [] - - bug = self._make_jira_bug( - "OADP-7902", - is_vulnerability=True, - labels=["CVE-2026-32281", f"pscomponent:{GOLANG_BUILDER_DELIVERY_REPO}"], - ) - - mock_tracker = Mock() - mock_tracker.get_bug.return_value = bug - mock_create_tracker.return_value = mock_tracker - - with self.assertRaises(RuntimeError) as ctx: - await process_bugs(self.mock_runtime, ["OADP-7902"]) - - self.assertIn("OADP-7902", str(ctx.exception)) - self.assertIn("no components use the builder", str(ctx.exception)) - - @patch(PATCH_GET_BUILDER_COMPONENTS) - @patch(PATCH_GET_KONFLUX_COMPONENT) - @patch(PATCH_GET_DELIVERY_REPO) - @patch(PATCH_CREATE_TRACKER) - async def test_golang_builder_cve_mixed_with_regular( - self, mock_create_tracker, mock_get_delivery, mock_get_konflux, mock_get_builder - ): - """A golang builder CVE and a regular CVE should both produce associations.""" - mock_get_delivery.side_effect = lambda _rt, repo: { - "oadp/oadp-velero-rhel9": "oadp-velero-container", - }.get(repo) - mock_get_konflux.side_effect = lambda _rt, comp: { - "oadp-velero-container": "oadp-1-4-oadp-velero", - }.get(comp) - mock_get_builder.return_value = ["oadp-1-4-oadp-operator", "oadp-1-4-oadp-velero"] - - builder_bug = self._make_jira_bug( - "OADP-7902", - is_vulnerability=True, - labels=["CVE-2026-32281", f"pscomponent:{GOLANG_BUILDER_DELIVERY_REPO}", "flaw:bz#2456333"], - ) - regular_cve = self._make_jira_bug( - "OADP-7792", - is_vulnerability=True, - labels=["CVE-2026-32280", "pscomponent:oadp/oadp-velero-rhel9", "flaw:bz#2456339"], - ) - - mock_tracker = Mock() - mock_tracker.get_bug.side_effect = lambda k: {"OADP-7902": builder_bug, "OADP-7792": regular_cve}[k] - mock_create_tracker.return_value = mock_tracker - - result = await process_bugs(self.mock_runtime, ["OADP-7902", "OADP-7792"]) - - self.assertEqual(result.type, "RHSA") - self.assertEqual(len(result.cves), 3) - - builder_cves = [c for c in result.cves if c.key == "CVE-2026-32281"] - self.assertEqual(len(builder_cves), 2) - - regular_cves = [c for c in result.cves if c.key == "CVE-2026-32280"] - self.assertEqual(len(regular_cves), 1) - self.assertEqual(regular_cves[0].component, "oadp-1-4-oadp-velero") + self.assertEqual(result.type, "RHBA") + self.assertIsNone(result.cves) + fixed_ids = [i.id for i in result.issues.fixed] + self.assertIn("OADP-7777", fixed_ids) + mock_get_delivery.assert_called_once_with(self.mock_runtime, "oadp/some-rhel9") + mock_get_konflux.assert_called_once_with(self.mock_runtime, "some-container") @patch(PATCH_GET_KONFLUX_COMPONENT) @patch(PATCH_GET_DELIVERY_REPO) diff --git a/ocp-build-data-validator/tests/test_schema/test_group_schema.py b/ocp-build-data-validator/tests/test_schema/test_group_schema.py index 86ba687b11..6552cf190c 100644 --- a/ocp-build-data-validator/tests/test_schema/test_group_schema.py +++ b/ocp-build-data-validator/tests/test_schema/test_group_schema.py @@ -26,29 +26,6 @@ def test_validate_with_invalid_bridge_release_config(self): } self.assertIn("'yes' is not of type 'boolean'", group_schema.validate("group.yml", invalid_data)) - def test_validate_reposync_requires_enabled(self): - data_missing_enabled = { - "repos": { - "my-repo": { - "conf": {"baseurl": {"x86_64": "https://example.com/repo/"}}, - "reposync": {"latest_only": False}, - } - } - } - result = group_schema.validate("group.yml", data_missing_enabled) - self.assertIn("'enabled' is a required property", result) - - def test_validate_reposync_with_enabled(self): - data_with_enabled = { - "repos": { - "my-repo": { - "conf": {"baseurl": {"x86_64": "https://example.com/repo/"}}, - "reposync": {"enabled": False}, - } - } - } - self.assertEqual("", group_schema.validate("group.yml", data_with_enabled)) - def test_validate_with_mismatched_bridge_release_basis_group(self): invalid_data = { "name": "openshift-4.23", diff --git a/ocp-build-data-validator/validator/json_schemas/repos.schema.json b/ocp-build-data-validator/validator/json_schemas/repos.schema.json index 07fdbbc4b3..ad7857b528 100644 --- a/ocp-build-data-validator/validator/json_schemas/repos.schema.json +++ b/ocp-build-data-validator/validator/json_schemas/repos.schema.json @@ -47,7 +47,6 @@ "type": "boolean" } }, - "required": ["enabled"], "additionalProperties": false }, "reposync!": { diff --git a/pyartcd/pyartcd/__main__.py b/pyartcd/pyartcd/__main__.py index ebef686b87..1c73545165 100644 --- a/pyartcd/pyartcd/__main__.py +++ b/pyartcd/pyartcd/__main__.py @@ -19,6 +19,7 @@ cleanup_locks, fbc_import_from_index, gen_assembly, + golang_builder_shipment, images_health, layered_products_scan_konflux, monitor_nightly_delays, @@ -78,6 +79,7 @@ "cleanup_locks", "fbc_import_from_index", "gen_assembly", + "golang_builder_shipment", "images_health", "layered_products_scan_konflux", "monitor_nightly_delays", diff --git a/pyartcd/pyartcd/pipelines/__init__.py b/pyartcd/pyartcd/pipelines/__init__.py index d23d73ef63..1704d98014 100644 --- a/pyartcd/pyartcd/pipelines/__init__.py +++ b/pyartcd/pyartcd/pipelines/__init__.py @@ -19,6 +19,7 @@ fbc_import_from_index, gen_assembly, gen_assembly_targeted, + golang_builder_shipment, images_health, layered_products_scan_konflux, ocp, @@ -67,6 +68,7 @@ 'fbc_import_from_index', 'gen_assembly', 'gen_assembly_targeted', + 'golang_builder_shipment', 'images_health', 'layered_products_scan_konflux', 'ocp', diff --git a/pyartcd/pyartcd/pipelines/build_microshift_bootc.py b/pyartcd/pyartcd/pipelines/build_microshift_bootc.py index a6c9f22948..d6126ea800 100644 --- a/pyartcd/pyartcd/pipelines/build_microshift_bootc.py +++ b/pyartcd/pyartcd/pipelines/build_microshift_bootc.py @@ -509,7 +509,7 @@ async def _rebuild_needed(): self._logger.info("Skipping plashet sync for %s", microshift_plashet_name) return - result = jenkins.start_build_plashets( + jenkins.start_build_plashets( group=self.group, release=default_release_suffix(), assembly=self.assembly, @@ -519,11 +519,6 @@ async def _rebuild_needed(): dry_run=self.runtime.dry_run, block_until_complete=True, ) - if result != "SUCCESS": - raise RuntimeError( - f"build-plashets job for {microshift_plashet_name} failed with result: {result}. " - f"Plashet must be built successfully before proceeding with the bootc image build." - ) async def _get_microshift_rpm_commit(self) -> str: """ diff --git a/pyartcd/pyartcd/pipelines/golang_builder_shipment.py b/pyartcd/pyartcd/pipelines/golang_builder_shipment.py new file mode 100644 index 0000000000..2e248e7ead --- /dev/null +++ b/pyartcd/pyartcd/pipelines/golang_builder_shipment.py @@ -0,0 +1,519 @@ +""" +Pipeline to create shipment MRs in ocp-shipment-data for golang builder releases. + +Invocation:: + + artcd golang-builder-shipment \\ + --ocp-version 4.22 \\ + --golang-nvrs golang-1.25.9-1.el9 + + # or with explicit Konflux image NVRs: + artcd golang-builder-shipment \\ + --ocp-version 4.22 \\ + --golang-group rhel-9-golang-1.25 \\ + openshift-golang-builder-container-v1.25.9-202605121249.p2.gdf787b0.el9 + +This pipeline: +1. Derives golang_group from the NVRs (or accepts it explicitly) +2. Resolves Konflux image NVRs from golang RPM NVRs if needed +3. Reads software_lifecycle.phase from ocp-build-data to pick the correct ReleasePlan +4. Builds a ShipmentConfig YAML with snapshot from the resolved NVRs +5. Creates a draft MR in ocp-shipment-data +""" + +import logging +import os +import re +import shutil +import tempfile +from datetime import datetime, timezone +from io import StringIO +from pathlib import Path +from typing import List, Optional, Tuple +from urllib.parse import urlparse + +import aiohttp +import click +import gitlab as python_gitlab +from artcommonlib import exectools +from artcommonlib.constants import GOLANG_BUILDER_IMAGE_NAME, SHIPMENT_DATA_URL_TEMPLATE +from artcommonlib.konflux.konflux_build_record import ArtifactType, Engine, KonfluxBuildOutcome +from artcommonlib.konflux.konflux_db import KonfluxDb +from artcommonlib.release_util import SoftwareLifecyclePhase, isolate_el_version_in_release +from artcommonlib.rpm_utils import parse_nvr +from artcommonlib.util import new_roundtrip_yaml_handler +from doozerlib.constants import ART_IMAGES_BASE_APPLICATION +from elliottlib.constants import GOLANG_BUILDER_CVE_COMPONENT +from elliottlib.shipment_model import ( + Data, + Environments, + Metadata, + ReleaseNotes, + Shipment, + ShipmentConfig, + ShipmentEnv, + Snapshot, + SnapshotSpec, +) + +from pyartcd import constants +from pyartcd.cli import cli, click_coroutine, pass_runtime +from pyartcd.git import GitRepository +from pyartcd.runtime import Runtime + +_LOGGER = logging.getLogger(__name__) +yaml = new_roundtrip_yaml_handler() + +GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP = { + "prod": "ocp-art-golang-builder-prod-rhel9", + "ec": "ocp-art-golang-builder-ec-rhel9", +} + + +def derive_golang_group(nvrs: List[str]) -> str: + """Derive the ocp-build-data golang group from NVR patterns. + + Supports both golang RPM NVRs (golang-1.25.9-1.el9) and + Konflux image NVRs (openshift-golang-builder-container-v1.25.9-...). + """ + for nvr in nvrs: + # Konflux image NVR: openshift-golang-builder-container-v1.25.9-...el9 + m = re.search(r"v(\d+)\.(\d+)\.\d+.*\.el(\d+)", nvr) + if m: + return f"rhel-{m.group(3)}-golang-{m.group(1)}.{m.group(2)}" + + # RPM NVR: golang-1.25.9-1.el9 + parsed = parse_nvr(nvr) + if parsed["name"] == "golang": + major_minor = ".".join(parsed["version"].split(".")[:2]) + el_v = isolate_el_version_in_release(parsed["release"]) + if el_v is not None: + return f"rhel-{el_v}-golang-{major_minor}" + + raise ValueError(f"Cannot derive golang group from NVRs: {nvrs}") + + +async def resolve_konflux_image_nvrs(golang_nvrs: List[str]) -> List[str]: + """Resolve golang RPM NVRs to Konflux golang-builder image NVRs via Konflux DB.""" + image_nvrs = [] + db = KonfluxDb() + + for nvr in golang_nvrs: + parsed = parse_nvr(nvr) + go_version = parsed["version"] + el_v = isolate_el_version_in_release(parsed["release"]) + if el_v is None: + raise ValueError(f"Cannot detect RHEL version from NVR: {nvr}") + + extra_patterns = {"nvr": f"{GOLANG_BUILDER_CVE_COMPONENT}-v{go_version}"} + record = await anext( + db.search_builds_by_fields( + where={ + "name": GOLANG_BUILDER_IMAGE_NAME, + "el_target": f"el{el_v}", + "artifact_type": str(ArtifactType.IMAGE), + "outcome": str(KonfluxBuildOutcome.SUCCESS), + "engine": str(Engine.KONFLUX), + }, + extra_patterns=extra_patterns, + limit=1, + ), + None, + ) + if not record: + raise RuntimeError( + f"No Konflux golang-builder image found for go {go_version} el{el_v}. " + f"Has update-golang built one for {nvr}?" + ) + _LOGGER.info("Resolved %s → %s", nvr, record.nvr) + image_nvrs.append(record.nvr) + + return image_nvrs + + +async def resolve_lifecycle_env(ocp_version: str, data_path: Optional[str] = None) -> str: + """Determine 'prod' or 'ec' by reading software_lifecycle.phase from ocp-build-data. + + Reads group.yml from the openshift-{ocp_version} branch. If the phase is + ``pre-release``, returns ``'ec'``; otherwise returns ``'prod'``. + """ + base_url = (data_path or constants.OCP_BUILD_DATA_URL).rstrip("/") + branch = f"openshift-{ocp_version}" + + # GitHub raw URL — works for both github.com and GHE + if "github.com" in base_url: + raw_url = base_url.replace("github.com", "raw.githubusercontent.com") + f"/{branch}/group.yml" + else: + raw_url = f"{base_url}/raw/{branch}/group.yml" + + _LOGGER.info("Fetching lifecycle phase from %s", raw_url) + async with aiohttp.ClientSession() as session: + async with session.get(raw_url, timeout=aiohttp.ClientTimeout(total=30)) as resp: + if resp.status != 200: + raise RuntimeError( + f"Failed to fetch group.yml from {raw_url} (HTTP {resp.status}). " + f"Does the branch '{branch}' exist in ocp-build-data?" + ) + content = await resp.text() + + group_config = yaml.load(content) + phase_str = None + if group_config and isinstance(group_config, dict): + lifecycle = group_config.get("software_lifecycle") + if lifecycle and isinstance(lifecycle, dict): + phase_str = lifecycle.get("phase") + + if not phase_str: + _LOGGER.warning("No software_lifecycle.phase in group.yml for %s; defaulting to prod", branch) + return "prod" + + try: + phase = SoftwareLifecyclePhase.from_name(phase_str) + except ValueError: + _LOGGER.warning("Unknown lifecycle phase '%s' for %s; defaulting to prod", phase_str, branch) + return "prod" + + if phase == SoftwareLifecyclePhase.PRE_RELEASE: + _LOGGER.info("OCP %s is pre-release → using ec ReleasePlan", ocp_version) + return "ec" + + _LOGGER.info("OCP %s is %s → using prod ReleasePlan", ocp_version, phase_str) + return "prod" + + +class GolangBuilderShipmentPipeline: + """Creates a shipment MR in ocp-shipment-data for golang builder images.""" + + def __init__( + self, + runtime: Runtime, + ocp_version: str, + nvrs: List[str], + golang_group: Optional[str] = None, + art_jira: str = "", + shipment_data_repo_url: Optional[str] = None, + data_path: Optional[str] = None, + ): + self.runtime = runtime + self.ocp_version = ocp_version + self.golang_group = golang_group or derive_golang_group(nvrs) + self.nvrs = sorted(nvrs) + self.art_jira = art_jira + self.dry_run = runtime.dry_run + self.data_path = data_path + + self.product = "ocp" + self.gitlab_url = runtime.config.get("gitlab_url", "https://gitlab.cee.redhat.com") + self.working_dir = runtime.working_dir.absolute() + self._shipment_data_repo_dir = self.working_dir / "shipment-data-push" + + self.shipment_data_repo_pull_url = ( + shipment_data_repo_url + or runtime.config.get("shipment_config", {}).get("shipment_data_url") + or SHIPMENT_DATA_URL_TEMPLATE + ) + self.shipment_data_repo_push_url = ( + runtime.config.get("shipment_config", {}).get("shipment_data_push_url") or SHIPMENT_DATA_URL_TEMPLATE + ) + self.shipment_data_repo = GitRepository(self._shipment_data_repo_dir, self.dry_run) + + # Resolved lazily in run() via resolve_lifecycle_env + self.env: Optional[str] = None + self.release_plan: Optional[str] = None + + @staticmethod + def resolve_release_plan(env: str) -> str: + """Map lifecycle env to the correct ReleasePlan name.""" + plan = GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP.get(env) + if not plan: + raise ValueError( + f"Unknown env '{env}'. Must be one of: {list(GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP.keys())}" + ) + return plan + + @staticmethod + def basic_auth_url(url: str, token: str) -> str: + """Inject token into a GitLab URL for push authentication.""" + parsed = urlparse(url) + return f"{parsed.scheme}://oauth2:{token}@{parsed.hostname}{parsed.path}" + + async def run(self) -> str: + """Execute the pipeline end to end. + + Returns: + The URL of the created shipment MR. + """ + self.env = await resolve_lifecycle_env(self.ocp_version, self.data_path) + self.release_plan = self.resolve_release_plan(self.env) + + _LOGGER.info( + "Starting golang-builder-shipment pipeline: ocp_version=%s golang_group=%s env=%s release_plan=%s nvrs=%s", + self.ocp_version, + self.golang_group, + self.env, + self.release_plan, + self.nvrs, + ) + + self.setup_working_dir() + await self.setup_repos() + + shipment_config = await self.build_shipment_config() + mr_url = await self.create_shipment_mr(shipment_config) + + _LOGGER.info("Shipment MR created: %s", mr_url) + return mr_url + + def setup_working_dir(self) -> None: + self.working_dir.mkdir(parents=True, exist_ok=True) + if self._shipment_data_repo_dir.exists(): + shutil.rmtree(self._shipment_data_repo_dir, ignore_errors=True) + + async def setup_repos(self): + self._gitlab_token = os.getenv("GITLAB_TOKEN") + if not self._gitlab_token: + raise ValueError("GITLAB_TOKEN environment variable is required") + + await self.shipment_data_repo.setup( + remote_url=self.basic_auth_url(self.shipment_data_repo_push_url, self._gitlab_token), + upstream_remote_url=self.shipment_data_repo_pull_url, + ) + await self.shipment_data_repo.fetch_switch_branch("main") + + async def build_shipment_config(self) -> ShipmentConfig: + """Construct a ShipmentConfig from NVRs using elliott snapshot new.""" + + snapshot = await self._create_snapshot() + + # Override application to match RP/RPA configuration. elliott infers the + # Konflux application from the build pipeline URL (e.g. "rhel-9-golang-1-25"), + # but our ReleasePlan/RPA pair is registered under "art-images-base". + snapshot.spec.application = ART_IMAGES_BASE_APPLICATION + + metadata = Metadata( + product=self.product, + application=snapshot.spec.application, + group=self.golang_group, + assembly="stream", + ) + + # Stage and prod use the same ReleasePlan for golang builders — the + # RPA is per-lifecycle (ec vs prod), not per-environment. + environments = Environments( + stage=ShipmentEnv(releasePlan=self.release_plan), + prod=ShipmentEnv(releasePlan=self.release_plan), + ) + + release_notes = ReleaseNotes( + type="RHBA", + synopsis=f"Golang builder image update for OpenShift {self.ocp_version}", + topic=( + f"An update for the golang builder images is now available for " + f"Red Hat OpenShift Container Platform {self.ocp_version}." + ), + description=( + f"This update provides rebuilt golang builder images for " + f"Red Hat OpenShift Container Platform {self.ocp_version}.\n\n" + f"Golang group: {self.golang_group}" + ), + solution="The golang builder images are available from registry.redhat.io/openshift/golang-builder.", + ) + if self.art_jira: + release_notes.references = [f"https://redhat.atlassian.net/browse/{self.art_jira}"] + + shipment = Shipment( + metadata=metadata, + environments=environments, + snapshot=snapshot, + data=Data(releaseNotes=release_notes), + ) + + config = ShipmentConfig(shipment=shipment) + _LOGGER.info("Built ShipmentConfig with %d NVRs", len(self.nvrs)) + return config + + async def _create_snapshot(self) -> Snapshot: + """Create a Snapshot from NVRs using ``elliott snapshot new --builds-file``.""" + + with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f: + for nvr in self.nvrs: + f.write(nvr + "\n") + builds_file = f.name + + try: + cmd = [ + "elliott", + "--group", + self.golang_group, + "--assembly", + "stream", + "snapshot", + "new", + f"--builds-file={builds_file}", + ] + quay_auth_file = os.getenv("QUAY_AUTH_FILE") + if quay_auth_file: + cmd.append(f"--pull-secret={quay_auth_file}") + + rc, stdout, stderr = await exectools.cmd_gather_async(cmd, stderr=None, check=False) + if rc != 0: + raise RuntimeError(f"elliott snapshot new failed (rc={rc}): {stderr or stdout}") + if stdout: + _LOGGER.info("elliott snapshot new output:\n%s", stdout) + finally: + os.unlink(builds_file) + + snapshot_obj = yaml.load(stdout) + if not snapshot_obj or not isinstance(snapshot_obj, dict): + raise ValueError(f"elliott snapshot new returned invalid output: {stdout!r}") + + spec = snapshot_obj.get("spec") + if not spec: + raise ValueError(f"elliott snapshot new output missing 'spec': {snapshot_obj}") + + return Snapshot( + spec=SnapshotSpec(**spec), + nvrs=self.nvrs, + ) + + async def create_shipment_mr(self, shipment_config: ShipmentConfig) -> str: + """Write the shipment YAML and open a draft MR in ocp-shipment-data.""" + + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S") + source_branch = f"golang-builder-shipment-{self.golang_group}-{timestamp}" + + await self.shipment_data_repo.create_branch(source_branch) + + # Persist the YAML + application = shipment_config.shipment.metadata.application + relative_target_dir = Path("shipment") / self.product / self.golang_group / application / self.env + target_dir = self.shipment_data_repo._directory / relative_target_dir + target_dir.mkdir(parents=True, exist_ok=True) + + filename = f"stream.image.{timestamp}.yaml" + filepath = relative_target_dir / filename + shipment_dump = shipment_config.model_dump(exclude_unset=True, exclude_none=True) + out = StringIO() + yaml.dump(shipment_dump, out) + await self.shipment_data_repo.write_file(filepath, out.getvalue()) + await self.shipment_data_repo.add_all() + await self.shipment_data_repo.log_diff() + + commit_message = f"Add golang builder shipment for {self.golang_group}" + if self.art_jira: + commit_message += f"\n\nRef: {self.art_jira}" + job_url = os.getenv("BUILD_URL", "") + if job_url: + commit_message += f"\n{job_url}" + + pushed = await self.shipment_data_repo.commit_push(commit_message, safe=True) + if not pushed: + raise RuntimeError("Failed to push shipment data to remote") + + mr_title = f"Draft: Golang builder shipment for {self.golang_group}" + mr_description = f"Golang builder shipment for OCP {self.ocp_version}\n\n" + mr_description += f"Group: {self.golang_group}\n" + mr_description += f"Environment: {self.env}\n" + mr_description += f"ReleasePlan: {self.release_plan}\n" + mr_description += f"NVRs: {len(self.nvrs)}\n" + if self.art_jira: + mr_description += f"\nRef: https://redhat.atlassian.net/browse/{self.art_jira}" + if job_url: + mr_description += f"\nCreated by: {job_url}" + + if self.dry_run: + _LOGGER.info("[DRY-RUN] Would create MR: %s", mr_title) + return f"{self.gitlab_url}/placeholder/-/merge_requests/placeholder" + + gl = python_gitlab.Gitlab(self.gitlab_url, private_token=self._gitlab_token) + + def _get_project(url): + parsed = urlparse(url) + project_path = parsed.path.strip("/").removesuffix(".git") + return gl.projects.get(project_path) + + source_project = _get_project(self.shipment_data_repo_push_url) + target_project = _get_project(self.shipment_data_repo_pull_url) + + mr = source_project.mergerequests.create( + { + "source_branch": source_branch, + "target_project_id": target_project.id, + "target_branch": "main", + "title": mr_title, + "description": mr_description, + "remove_source_branch": True, + } + ) + _LOGGER.info("Created Draft MR: %s", mr.web_url) + return mr.web_url + + +@cli.command("golang-builder-shipment") +@click.option("--ocp-version", required=True, help="OCP version (e.g. 4.22)") +@click.option( + "--golang-group", + required=False, + default=None, + help="Golang builder group (e.g. rhel-9-golang-1.25). Derived from NVRs if omitted.", +) +@click.option( + "--golang-nvrs", + required=False, + default=None, + help="Golang RPM NVRs (comma-separated). Resolves to Konflux image NVRs automatically.", +) +@click.option("--art-jira", default="", help="Related ART Jira ticket (e.g. ART-20930)") +@click.option("--shipment-data-repo-url", default=None, help="Override ocp-shipment-data repo URL") +@click.option( + "--data-path", + required=False, + default=constants.OCP_BUILD_DATA_URL, + help="ocp-build-data URL (used to read software_lifecycle.phase)", +) +@click.argument("nvrs", nargs=-1, required=False) +@pass_runtime +@click_coroutine +async def golang_builder_shipment( + runtime: Runtime, + ocp_version: str, + golang_group: Optional[str], + golang_nvrs: Optional[str], + art_jira: str, + shipment_data_repo_url: Optional[str], + data_path: str, + nvrs: Tuple[str, ...], +): + """Create a shipment MR in ocp-shipment-data for golang builder images. + + Builds a ShipmentConfig YAML from the provided NVRs, auto-detects whether + to use the prod or ec ReleasePlan from ocp-build-data software_lifecycle.phase, + and opens a draft MR in ocp-shipment-data for ERT approval. + + Accepts either explicit Konflux image NVRs as positional arguments, or + --golang-nvrs with golang RPM NVRs (auto-resolved to Konflux image NVRs). + The --golang-group is derived from the NVRs when not specified. + """ + resolved_nvrs: List[str] = list(nvrs) + + if not resolved_nvrs and golang_nvrs: + rpm_nvrs = [n.strip() for n in golang_nvrs.replace(",", " ").split() if n.strip()] + _LOGGER.info("Resolving golang RPM NVRs to Konflux image NVRs: %s", rpm_nvrs) + resolved_nvrs = await resolve_konflux_image_nvrs(rpm_nvrs) + if not golang_group: + golang_group = derive_golang_group(rpm_nvrs) + + if not resolved_nvrs: + raise click.UsageError("Provide Konflux image NVRs as arguments or --golang-nvrs with golang RPM NVRs") + + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version=ocp_version, + nvrs=resolved_nvrs, + golang_group=golang_group, + art_jira=art_jira, + shipment_data_repo_url=shipment_data_repo_url, + data_path=data_path, + ) + mr_url = await pipeline.run() + click.echo(f"Shipment MR: {mr_url}") diff --git a/pyartcd/pyartcd/pipelines/images_health.py b/pyartcd/pyartcd/pipelines/images_health.py index 6ecbfd4361..ac54cca8c3 100644 --- a/pyartcd/pyartcd/pipelines/images_health.py +++ b/pyartcd/pyartcd/pipelines/images_health.py @@ -624,7 +624,8 @@ async def notify_release_channel(self, version): async def notify_public_channel(self): """ Send a summary message to the public channel with a link to the dashboard. - Each group's detailed report is sent as a thread response to keep the main channel clean. + Instead of posting detailed failure lists (which can overflow Jira API limits), + we now only post a summary grouped by version and direct users to the art-build-failures dashboard. """ self.slack_client.bind_channel(self.public_channel) @@ -658,17 +659,9 @@ async def notify_public_channel(self): all_groups.update(f'openshift-{v}' for v in self.release_failures.keys()) all_groups.update(f'openshift-{v}' for v in self.rebase_failures.keys()) - # Send the main alert message with just the dashboard link - dashboard_url = ART_BUILD_FAILURES_URL - summary_message = ( - f':alert: There are some issues to look into for OpenShift builds.\n\n' - f'For detailed information, please check the {self.url_text(dashboard_url, "ART Build Failures Dashboard")}' - ) - response = await self.slack_client.say( - summary_message, link_build_url=False, unfurl_links=False, unfurl_media=False - ) + # Build message grouped by version + message_parts = [':alert: There are some issues to look into for OpenShift builds:\n'] - # Send each group's summary as a thread response for group in sorted(all_groups): version = group.replace('openshift-', '') group_summary = [] @@ -697,19 +690,19 @@ async def notify_public_channel(self): n = len(rebase_fails) group_summary.append(f'{n} image{"s" if n > 1 else ""} with rebase failures') - # Send the group summary as a thread response if group_summary: - group_message = f'*{group}*:\n' + message_parts.append(f'\n*{group}*:') for item in group_summary: - group_message += f'- {item}\n' - - await self.slack_client.say( - group_message.rstrip(), - thread_ts=response['ts'], - link_build_url=False, - unfurl_links=False, - unfurl_media=False, - ) + message_parts.append(f'- {item}') + + # Link to art-build-failures dashboard + dashboard_url = ART_BUILD_FAILURES_URL + message_parts.append( + f'\nFor detailed information, please check the {self.url_text(dashboard_url, "ART Build Failures Dashboard")}' + ) + + message = '\n'.join(message_parts) + await self.slack_client.say(message, link_build_url=False, unfurl_links=False, unfurl_media=False) @staticmethod def _group_failures_by_image(failures_by_version: dict) -> dict[str, list[dict]]: diff --git a/pyartcd/pyartcd/pipelines/okd.py b/pyartcd/pyartcd/pipelines/okd.py index 813c12f795..03f2090e1f 100644 --- a/pyartcd/pyartcd/pipelines/okd.py +++ b/pyartcd/pyartcd/pipelines/okd.py @@ -28,7 +28,7 @@ reset_fail_counter, ) -OKD_ARCHES = ['x86_64', 'aarch64'] +OKD_ARCHES = ['x86_64'] class BuildPlan: diff --git a/pyartcd/pyartcd/pipelines/quay_doomsday_backup.py b/pyartcd/pyartcd/pipelines/quay_doomsday_backup.py index f8b602b2cf..f256c8d285 100644 --- a/pyartcd/pyartcd/pipelines/quay_doomsday_backup.py +++ b/pyartcd/pyartcd/pipelines/quay_doomsday_backup.py @@ -99,14 +99,13 @@ async def run(self) -> None: self.runtime.logger.info("[DRY RUN] Would have messaged Slack") # Synchronize individual arches sequentially to help with quay returning 502 - failed_arches = [] + results = [] for arch in self.arches: - if not await self.sync_arch(arch): - failed_arches.append(arch) + results.append(await self.sync_arch(arch)) # Report the results to Slack if not self.runtime.dry_run: - if not failed_arches: + if all(results): await self.slack_client._client.reactions_add( channel=slack_channel_id, timestamp=main_message_ts, name="done_it_is" ) @@ -115,9 +114,6 @@ async def run(self) -> None: else: self.runtime.logger.info("[DRY RUN] Would have messaged Slack") - if failed_arches: - raise RuntimeError(f"Failed to sync arches for {self.version}: {', '.join(failed_arches)}") - @cli.command( "quay-doomsday-backup", diff --git a/pyartcd/pyartcd/pipelines/sync_ci_images.py b/pyartcd/pyartcd/pipelines/sync_ci_images.py index 294ba5c744..622aab5820 100644 --- a/pyartcd/pyartcd/pipelines/sync_ci_images.py +++ b/pyartcd/pyartcd/pipelines/sync_ci_images.py @@ -53,7 +53,6 @@ def __init__( data_path: str = "", data_gitref: str = "", only_stream: str = "", - images: str = "", add_labels: str = "", assembly: str = "stream", skip_prs: bool = False, @@ -70,7 +69,6 @@ def __init__( data_path: ocp-build-data fork URL (default: official repo) data_gitref: ocp-build-data git branch/tag/sha (default: use version branch) only_stream: Specific stream from streams.yml. - images: Comma-separated distgit keys of images with ci_alignment.upstream_image. add_labels: Space-delimited labels to add to PRs assembly: Assembly name (default: "stream") skip_prs: Skip opening reconciliation PRs @@ -84,7 +82,6 @@ def __init__( self.data_path = data_path or OCP_BUILD_DATA_URL self.data_gitref = data_gitref self.only_stream = only_stream - self.images = [i.strip() for i in images.split(',') if i.strip()] if images else [] self.add_labels = add_labels self.assembly = assembly self.skip_prs = skip_prs @@ -109,9 +106,9 @@ def _validate_parameters(self) -> None: if not re.match(r'^\d+\.\d+$', self.version): raise ValueError(f"Invalid FOR_RELEASE format: {self.version}. Expected format: X.Y (e.g., 4.17)") - # Auto-set SKIP_PRS if ONLY_STREAM or IMAGES is set. - if (self.only_stream or self.images) and not self.skip_prs: - self._logger.info("Setting SKIP_PRS to true because ONLY_STREAM or IMAGES is set") + # Auto-set SKIP_PRS if ONLY_STREAM is set. + if self.only_stream and not self.skip_prs: + self._logger.info("Setting SKIP_PRS to true because ONLY_STREAM is set") self.skip_prs = True # Validate assembly format (alphanumeric, dash, dot, underscore) @@ -514,37 +511,24 @@ def _build_doozer_options(self, group_dir: Path, auth_file: str) -> str: f"--build-system {self.BUILD_SYSTEM} " f"--registry-config {auth_file}" ) + if self.only_stream: + doozer_opts += f" --only-streams {self.only_stream}" return doozer_opts - @property - def _stream_arg(self) -> str: - """Return the --stream subcommand arg if only_stream is set, empty string otherwise.""" - return f"--stream {self.only_stream}" if self.only_stream else "" - - @property - def _image_args(self) -> str: - """Return --image args for each image distgit key, empty string if none.""" - return " ".join(f"--image {img}" for img in self.images) if self.images else "" - - @property - def _filter_args(self) -> str: - """Return combined --stream and --image subcommand args.""" - return f"{self._stream_arg} {self._image_args}".strip() - async def _generate_and_apply_buildconfigs(self, doozer_opts: str) -> None: """Generate BuildConfigs and apply them to CI cluster.""" self._logger.info(f"{self.version}: Generating BuildConfigs") apply_flag = "" if self.runtime.dry_run else "--apply" await self._run_doozer_command( - doozer_opts, - "images:streams gen-buildconfigs", - f"{self._filter_args} -o {self._working_dir}/buildconfigs.yaml {apply_flag}", + doozer_opts, "images:streams gen-buildconfigs", f"-o {self._working_dir}/buildconfigs.yaml {apply_flag}" ) async def _mirror_images_to_ci(self, doozer_opts: str, auth_file: str) -> None: """Mirror builder and base images to CI registries.""" self._logger.info(f"{self.version}: Mirroring images") - mirror_args = f"{self._filter_args} --registry-auth {auth_file} " + # Pass --registry-auth at command level to work around doozer bug where + # global --registry-config doesn't get passed to oc image mirror commands + mirror_args = f"--registry-auth {auth_file} " if self.update_images_only_when_missing: mirror_args += "--only-if-missing " if self.runtime.dry_run: @@ -554,7 +538,8 @@ async def _mirror_images_to_ci(self, doozer_opts: str, auth_file: str) -> None: async def _trigger_ci_builds(self, doozer_opts: str, auth_file: str) -> None: """Start CI builds for updated images.""" self._logger.info(f"{self.version}: Starting builds") - start_builds_args = f"{self._filter_args} --registry-auth {auth_file} " + # Pass --registry-auth for image info and mirror operations in pre-build steps + start_builds_args = f"--registry-auth {auth_file} " if self.runtime.dry_run: start_builds_args += "--dry-run" await self._run_doozer_command(doozer_opts, "images:streams start-builds", start_builds_args.strip()) @@ -568,9 +553,7 @@ async def _wait_for_builds(self) -> None: async def _verify_upstream_consistency(self, doozer_opts: str, auth_file: str) -> None: """Verify CI imagestreams match expected state.""" self._logger.info(f"{self.version}: Checking upstream consistency") - await self._run_doozer_command( - doozer_opts, "images:streams check-upstream", f"{self._filter_args} --registry-auth {auth_file}" - ) + await self._run_doozer_command(doozer_opts, "images:streams check-upstream", f"--registry-auth {auth_file}") async def _open_reconciliation_prs(self, doozer_opts: str) -> int: """Open PRs to reconcile BuildConfig drift.""" @@ -708,12 +691,6 @@ async def run(self) -> int: default='', help='Process only specific stream from streams.yml. Automatically sets SKIP_PRS=true.', ) -@click.option( - '--images', - default='', - help='Comma-separated distgit keys to sync (e.g. ci-openshift-base.rhel10). ' - 'Each must have ci_alignment.upstream_image set. Automatically sets SKIP_PRS=true.', -) @click.option( '--add-labels', default='', help='Space-delimited labels to add to reconciliation PRs (e.g., "backport candidate")' ) @@ -741,7 +718,6 @@ async def sync_ci_images_cli( data_path: str, data_gitref: str, only_stream: str, - images: str, add_labels: str, assembly: str, skip_prs: bool, @@ -771,7 +747,6 @@ async def sync_ci_images_cli( data_path=data_path, data_gitref=data_gitref, only_stream=only_stream, - images=images, add_labels=add_labels, assembly=assembly, skip_prs=skip_prs, diff --git a/pyartcd/tests/pipelines/test_build_microshift_bootc.py b/pyartcd/tests/pipelines/test_build_microshift_bootc.py index 3a44f2a2b0..4966b87ebc 100644 --- a/pyartcd/tests/pipelines/test_build_microshift_bootc.py +++ b/pyartcd/tests/pipelines/test_build_microshift_bootc.py @@ -632,56 +632,3 @@ def test_validate_shipment_mr_raises_on_closed_mr(self): pipeline._validate_shipment_mr(shipment_url) self.assertIn("closed", str(ctx.exception)) self.assertIn("not opened", str(ctx.exception)) - - @patch("pyartcd.pipelines.build_microshift_bootc.jenkins.start_build_plashets") - @patch("pyartcd.pipelines.build_microshift_bootc.get_microshift_builds", new_callable=AsyncMock) - @patch("pyartcd.pipelines.build_microshift_bootc.default_release_suffix", return_value="202601290005.p0") - async def test_build_plashet_raises_on_failure(self, mock_release_suffix, mock_get_builds, mock_start_plashets): - """ - Test that _build_plashet_for_bootc raises RuntimeError when the - build-plashets Jenkins job returns a non-SUCCESS result. - """ - # given - mock_start_plashets.return_value = "FAILURE" - pipeline = self._make_pipeline() - pipeline.group_config = {"all_repos": []} - pipeline.force_plashet_sync = True # skip the _rebuild_needed() check - - # Mock plashet config discovery - mock_plashet = Mock() - mock_plashet.disabled = False - mock_plashet.type = "plashet" - mock_plashet.name = "rhel-9-server-microshift-rpms" - with patch("pyartcd.pipelines.build_microshift_bootc.RepoList") as mock_repo_list: - mock_repo_list.model_validate.return_value.root = [mock_plashet] - - # when / then - with self.assertRaises(RuntimeError) as ctx: - await pipeline._build_plashet_for_bootc() - self.assertIn("FAILURE", str(ctx.exception)) - self.assertIn("rhel-9-server-microshift-rpms", str(ctx.exception)) - - @patch("pyartcd.pipelines.build_microshift_bootc.jenkins.start_build_plashets") - @patch("pyartcd.pipelines.build_microshift_bootc.get_microshift_builds", new_callable=AsyncMock) - @patch("pyartcd.pipelines.build_microshift_bootc.default_release_suffix", return_value="202601290005.p0") - async def test_build_plashet_succeeds_on_success(self, mock_release_suffix, mock_get_builds, mock_start_plashets): - """ - Test that _build_plashet_for_bootc completes without error when the - build-plashets Jenkins job returns SUCCESS. - """ - # given - mock_start_plashets.return_value = "SUCCESS" - pipeline = self._make_pipeline() - pipeline.group_config = {"all_repos": []} - pipeline.force_plashet_sync = True # skip the _rebuild_needed() check - - # Mock plashet config discovery - mock_plashet = Mock() - mock_plashet.disabled = False - mock_plashet.type = "plashet" - mock_plashet.name = "rhel-9-server-microshift-rpms" - with patch("pyartcd.pipelines.build_microshift_bootc.RepoList") as mock_repo_list: - mock_repo_list.model_validate.return_value.root = [mock_plashet] - - # when / then - should not raise - await pipeline._build_plashet_for_bootc() diff --git a/pyartcd/tests/pipelines/test_golang_builder_shipment.py b/pyartcd/tests/pipelines/test_golang_builder_shipment.py new file mode 100644 index 0000000000..0b5155ecc7 --- /dev/null +++ b/pyartcd/tests/pipelines/test_golang_builder_shipment.py @@ -0,0 +1,627 @@ +import unittest +from pathlib import Path +from unittest import IsolatedAsyncioTestCase +from unittest.mock import AsyncMock, MagicMock, Mock, patch + +from doozerlib.constants import ART_IMAGES_BASE_APPLICATION +from pyartcd.pipelines.golang_builder_shipment import ( + GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP, + GolangBuilderShipmentPipeline, + derive_golang_group, + resolve_lifecycle_env, +) + + +class TestResolveReleasePlan(unittest.TestCase): + """Test lifecycle -> releasePlan mapping.""" + + def test_prod_returns_prod_plan(self): + plan = GolangBuilderShipmentPipeline.resolve_release_plan("prod") + self.assertEqual(plan, "ocp-art-golang-builder-prod-rhel9") + + def test_ec_returns_ec_plan(self): + plan = GolangBuilderShipmentPipeline.resolve_release_plan("ec") + self.assertEqual(plan, "ocp-art-golang-builder-ec-rhel9") + + def test_unknown_env_raises(self): + with self.assertRaises(ValueError): + GolangBuilderShipmentPipeline.resolve_release_plan("staging") + + def test_map_keys_are_complete(self): + self.assertIn("prod", GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP) + self.assertIn("ec", GOLANG_BUILDER_SHIPMENT_RELEASE_PLAN_MAP) + + +class _FakeResponse: + def __init__(self, status, body=""): + self.status = status + self._body = body + + async def text(self): + return self._body + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + +class _FakeSession: + def __init__(self, response): + self._response = response + + def get(self, *args, **kwargs): + return self._response + + async def __aenter__(self): + return self + + async def __aexit__(self, *args): + pass + + +class TestResolveLifecycleEnv(IsolatedAsyncioTestCase): + """Test auto-resolution of prod/ec from ocp-build-data group.yml.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_release_phase_returns_prod(self, mock_session_cls): + resp = _FakeResponse(200, "software_lifecycle:\n phase: release\n") + mock_session_cls.return_value = _FakeSession(resp) + result = await resolve_lifecycle_env("4.18") + self.assertEqual(result, "prod") + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_pre_release_phase_returns_ec(self, mock_session_cls): + resp = _FakeResponse(200, "software_lifecycle:\n phase: pre-release\n") + mock_session_cls.return_value = _FakeSession(resp) + result = await resolve_lifecycle_env("4.23") + self.assertEqual(result, "ec") + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_missing_lifecycle_defaults_to_prod(self, mock_session_cls): + resp = _FakeResponse(200, "vars:\n GOLANG_VERSION: '1.22'\n") + mock_session_cls.return_value = _FakeSession(resp) + result = await resolve_lifecycle_env("4.16") + self.assertEqual(result, "prod") + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_http_error_raises(self, mock_session_cls): + resp = _FakeResponse(404) + mock_session_cls.return_value = _FakeSession(resp) + with self.assertRaises(RuntimeError): + await resolve_lifecycle_env("99.99") + + +class TestBasicAuthUrl(unittest.TestCase): + def test_injects_token(self): + url = "https://gitlab.cee.redhat.com/hybrid-platforms/art/ocp-shipment-data.git" + result = GolangBuilderShipmentPipeline.basic_auth_url(url, "mytoken") + self.assertIn("oauth2:mytoken@", result) + self.assertIn("gitlab.cee.redhat.com", result) + self.assertTrue(result.startswith("https://")) + + +class TestPipelineInit(unittest.TestCase): + def _make_runtime(self, dry_run=False): + runtime = Mock() + runtime.dry_run = dry_run + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + return runtime + + def test_init_sorts_nvrs(self): + runtime = self._make_runtime() + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["golang-builder-container-v1.25-1.el9", "golang-builder-container-v1.25-1.el8"], + ) + self.assertEqual(pipeline.product, "ocp") + self.assertEqual( + pipeline.nvrs, + [ + "golang-builder-container-v1.25-1.el8", + "golang-builder-container-v1.25-1.el9", + ], + ) + # env and release_plan are None until run() resolves them + self.assertIsNone(pipeline.env) + self.assertIsNone(pipeline.release_plan) + + +class TestBuildShipmentConfig(IsolatedAsyncioTestCase): + def _make_pipeline(self, env="prod"): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["golang-builder-container-v1.25-202606220000.el9"], + art_jira="ART-20930", + ) + pipeline.env = env + pipeline.release_plan = pipeline.resolve_release_plan(env) + return pipeline + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_build_shipment_config_prod(self, mock_unlink, mock_cmd): + mock_cmd.return_value = ( + 0, + """ +apiVersion: appstudio.redhat.com/v1alpha1 +kind: Snapshot +spec: + application: art-images-base + components: + - name: golang-builder-v1.25-rhel9 + containerImage: quay.io/redhat-user-workloads/ocp-art-tenant/art-images@sha256:abc123 + source: + git: + url: https://github.com/openshift-priv/builder + revision: abc123 +""", + "", + ) + + pipeline = self._make_pipeline(env="prod") + config = await pipeline.build_shipment_config() + + self.assertEqual(config.shipment.metadata.product, "ocp") + self.assertEqual(config.shipment.metadata.application, "art-images-base") + self.assertEqual(config.shipment.metadata.group, "rhel-9-golang-1.25") + self.assertEqual(config.shipment.metadata.assembly, "stream") + self.assertEqual( + config.shipment.environments.prod.releasePlan, + "ocp-art-golang-builder-prod-rhel9", + ) + self.assertEqual(config.shipment.data.releaseNotes.type, "RHBA") + self.assertIn( + "https://redhat.atlassian.net/browse/ART-20930", + config.shipment.data.releaseNotes.references, + ) + + cmd_args = mock_cmd.call_args[0][0] + self.assertTrue(any("--builds-file=" in str(a) for a in cmd_args)) + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_build_shipment_config_ec(self, mock_unlink, mock_cmd): + mock_cmd.return_value = ( + 0, + """ +apiVersion: appstudio.redhat.com/v1alpha1 +kind: Snapshot +spec: + application: art-images-base + components: + - name: golang-builder-v1.26-rhel9 + containerImage: quay.io/redhat-user-workloads/ocp-art-tenant/art-images@sha256:def456 + source: + git: + url: https://github.com/openshift-priv/builder + revision: def456 +""", + "", + ) + + pipeline = self._make_pipeline(env="ec") + config = await pipeline.build_shipment_config() + + self.assertEqual( + config.shipment.environments.prod.releasePlan, + "ocp-art-golang-builder-ec-rhel9", + ) + + +class TestCreateShipmentMR(IsolatedAsyncioTestCase): + def _make_pipeline(self, dry_run=False): + runtime = Mock() + runtime.dry_run = dry_run + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + art_jira="ART-20930", + ) + pipeline.env = "prod" + pipeline.release_plan = "ocp-art-golang-builder-prod-rhel9" + pipeline._gitlab_token = "test-token" + pipeline.shipment_data_repo = AsyncMock() + pipeline.shipment_data_repo._directory = Path("/tmp/test-working/shipment-data-push") + pipeline.shipment_data_repo.commit_push = AsyncMock(return_value=True) + return pipeline + + def _make_config_mock(self): + config = Mock() + config.shipment.metadata.application = "art-images-base" + config.model_dump.return_value = {"shipment": {}} + return config + + @patch("pyartcd.pipelines.golang_builder_shipment.python_gitlab") + async def test_dry_run_returns_placeholder(self, mock_gitlab): + pipeline = self._make_pipeline(dry_run=True) + pipeline.env = "prod" + config = self._make_config_mock() + + mock_project = MagicMock() + mock_gitlab.Gitlab.return_value.projects.get.return_value = mock_project + + with patch("pathlib.Path.mkdir"): + result = await pipeline.create_shipment_mr(config) + + self.assertIn("placeholder", result) + mock_project.mergerequests.create.assert_not_called() + + @patch("pyartcd.pipelines.golang_builder_shipment.python_gitlab") + async def test_creates_mr_with_correct_title(self, mock_gitlab): + pipeline = self._make_pipeline(dry_run=False) + config = self._make_config_mock() + + mock_project = MagicMock() + mock_mr = MagicMock() + mock_mr.web_url = "https://gitlab.cee.redhat.com/test/-/merge_requests/1" + mock_project.mergerequests.create.return_value = mock_mr + mock_gitlab.Gitlab.return_value.projects.get.return_value = mock_project + + with patch("pathlib.Path.mkdir"): + result = await pipeline.create_shipment_mr(config) + + self.assertEqual(result, mock_mr.web_url) + create_args = mock_project.mergerequests.create.call_args[0][0] + self.assertIn("Golang builder shipment", create_args["title"]) + self.assertEqual(create_args["target_branch"], "main") + + +class TestSetupReposNoToken(IsolatedAsyncioTestCase): + @patch.dict("os.environ", {}, clear=True) + async def test_missing_gitlab_token_raises(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + with self.assertRaises(ValueError): + await pipeline.setup_repos() + + +class TestSnapshotWithQuayAuth(IsolatedAsyncioTestCase): + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {"QUAY_AUTH_FILE": "/tmp/quay-auth.json"}) + async def test_pull_secret_flag_added(self, mock_unlink, mock_cmd): + mock_cmd.return_value = ( + 0, + """ +spec: + application: art-images-base + components: + - name: golang-builder-v1.25-rhel9 + containerImage: quay.io/test@sha256:abc + source: + git: + url: https://github.com/openshift-priv/builder + revision: abc123 +""", + "", + ) + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + await pipeline._create_snapshot() + cmd_args = mock_cmd.call_args[0][0] + self.assertTrue(any("--pull-secret=" in str(a) for a in cmd_args)) + + +class TestApplicationOverride(IsolatedAsyncioTestCase): + """Verify snapshot application is overridden to art-images-base.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_elliott_app_overridden_to_art_images_base(self, mock_unlink, mock_cmd): + # elliott returns "rhel-9-golang-1-25" as the application + mock_cmd.return_value = ( + 0, + """ +spec: + application: rhel-9-golang-1-25 + components: + - name: golang-builder-v1.25-rhel9 + containerImage: quay.io/test@sha256:abc + source: + git: + url: https://github.com/openshift-priv/builder + revision: abc123 +""", + "", + ) + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + art_jira="ART-20930", + ) + pipeline.env = "prod" + pipeline.release_plan = pipeline.resolve_release_plan("prod") + + config = await pipeline.build_shipment_config() + + self.assertEqual(config.shipment.metadata.application, ART_IMAGES_BASE_APPLICATION) + self.assertEqual(config.shipment.snapshot.spec.application, ART_IMAGES_BASE_APPLICATION) + + +class TestCreateSnapshotErrors(IsolatedAsyncioTestCase): + """Test error paths in _create_snapshot.""" + + def _make_pipeline(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + return GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_elliott_nonzero_rc_raises(self, mock_unlink, mock_cmd): + mock_cmd.return_value = (1, "", "elliott error: NVR not found") + pipeline = self._make_pipeline() + with self.assertRaises(RuntimeError) as ctx: + await pipeline._create_snapshot() + self.assertIn("elliott snapshot new failed", str(ctx.exception)) + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_empty_stdout_raises(self, mock_unlink, mock_cmd): + mock_cmd.return_value = (0, "", "") + pipeline = self._make_pipeline() + with self.assertRaises(ValueError) as ctx: + await pipeline._create_snapshot() + self.assertIn("invalid output", str(ctx.exception)) + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_missing_spec_raises(self, mock_unlink, mock_cmd): + mock_cmd.return_value = (0, "apiVersion: v1\nkind: Snapshot\n", "") + pipeline = self._make_pipeline() + with self.assertRaises(ValueError) as ctx: + await pipeline._create_snapshot() + self.assertIn("missing 'spec'", str(ctx.exception)) + + +class TestCommitPushFailure(IsolatedAsyncioTestCase): + """Test create_shipment_mr when commit_push returns False.""" + + async def test_push_failure_raises(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + art_jira="ART-20930", + ) + pipeline.env = "prod" + pipeline.release_plan = "ocp-art-golang-builder-prod-rhel9" + pipeline._gitlab_token = "test-token" + pipeline.shipment_data_repo = AsyncMock() + pipeline.shipment_data_repo._directory = Path("/tmp/test-working/shipment-data-push") + pipeline.shipment_data_repo.commit_push = AsyncMock(return_value=False) + + config = Mock() + config.shipment.metadata.application = "art-images-base" + config.model_dump.return_value = {"shipment": {}} + + with patch("pathlib.Path.mkdir"): + with self.assertRaises(RuntimeError) as ctx: + await pipeline.create_shipment_mr(config) + self.assertIn("Failed to push", str(ctx.exception)) + + +class TestRunOrchestration(IsolatedAsyncioTestCase): + """Test run() wiring.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.resolve_lifecycle_env") + async def test_run_calls_steps_in_order(self, mock_resolve_env): + mock_resolve_env.return_value = "prod" + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + pipeline.setup_working_dir = Mock() + pipeline.setup_repos = AsyncMock() + pipeline.build_shipment_config = AsyncMock(return_value=Mock()) + pipeline.create_shipment_mr = AsyncMock(return_value="https://gitlab.example.com/-/merge_requests/1") + + result = await pipeline.run() + + self.assertEqual(result, "https://gitlab.example.com/-/merge_requests/1") + pipeline.setup_working_dir.assert_called_once() + pipeline.setup_repos.assert_called_once() + pipeline.build_shipment_config.assert_called_once() + pipeline.create_shipment_mr.assert_called_once() + self.assertEqual(pipeline.env, "prod") + self.assertEqual(pipeline.release_plan, "ocp-art-golang-builder-prod-rhel9") + + +class TestResolveLifecycleEnvUnknownPhase(IsolatedAsyncioTestCase): + """Test unknown phase falls back to prod.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.aiohttp.ClientSession") + async def test_unknown_phase_defaults_to_prod(self, mock_session_cls): + resp = _FakeResponse(200, "software_lifecycle:\n phase: maintenance\n") + mock_session_cls.return_value = _FakeSession(resp) + result = await resolve_lifecycle_env("4.14") + self.assertEqual(result, "prod") + + +class TestBuildShipmentConfigNoJira(IsolatedAsyncioTestCase): + """Test build_shipment_config without art_jira.""" + + @patch("pyartcd.pipelines.golang_builder_shipment.exectools.cmd_gather_async") + @patch("pyartcd.pipelines.golang_builder_shipment.os.unlink") + @patch.dict("os.environ", {}, clear=False) + async def test_no_jira_omits_references(self, mock_unlink, mock_cmd): + mock_cmd.return_value = ( + 0, + """ +spec: + application: art-images-base + components: + - name: golang-builder-v1.25-rhel9 + containerImage: quay.io/test@sha256:abc + source: + git: + url: https://github.com/openshift-priv/builder + revision: abc123 +""", + "", + ) + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + art_jira="", + ) + pipeline.env = "prod" + pipeline.release_plan = pipeline.resolve_release_plan("prod") + + config = await pipeline.build_shipment_config() + + self.assertFalse( + hasattr(config.shipment.data.releaseNotes, 'references') and config.shipment.data.releaseNotes.references + ) + + +class TestSetupReposSuccess(IsolatedAsyncioTestCase): + """Test setup_repos happy path.""" + + @patch.dict("os.environ", {"GITLAB_TOKEN": "test-token"}, clear=False) + async def test_setup_repos_configures_repo(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + golang_group="rhel-9-golang-1.25", + nvrs=["some-nvr"], + ) + pipeline.shipment_data_repo = AsyncMock() + + await pipeline.setup_repos() + + self.assertEqual(pipeline._gitlab_token, "test-token") + pipeline.shipment_data_repo.setup.assert_called_once() + setup_kwargs = pipeline.shipment_data_repo.setup.call_args[1] + self.assertIn("oauth2:test-token@", setup_kwargs["remote_url"]) + pipeline.shipment_data_repo.fetch_switch_branch.assert_called_once_with("main") + + +class TestDeriveGolangGroup(unittest.TestCase): + """Test golang group derivation from NVR patterns.""" + + def test_from_rpm_nvr(self): + self.assertEqual(derive_golang_group(["golang-1.25.9-1.el9"]), "rhel-9-golang-1.25") + + def test_from_rpm_nvr_el8(self): + self.assertEqual(derive_golang_group(["golang-1.22.3-2.el8"]), "rhel-8-golang-1.22") + + def test_from_konflux_image_nvr(self): + nvr = "openshift-golang-builder-container-v1.25.9-202605121249.p2.gdf787b0.el9" + self.assertEqual(derive_golang_group([nvr]), "rhel-9-golang-1.25") + + def test_unknown_nvr_raises(self): + with self.assertRaises(ValueError): + derive_golang_group(["not-a-golang-nvr-1.0-1.noarch"]) + + def test_init_derives_group_from_image_nvr(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + nvrs=["openshift-golang-builder-container-v1.25.9-202605121249.p2.gdf787b0.el9"], + ) + self.assertEqual(pipeline.golang_group, "rhel-9-golang-1.25") + + def test_init_derives_group_from_rpm_nvr(self): + runtime = Mock() + runtime.dry_run = False + runtime.config = {} + runtime.working_dir = Path("/tmp/test-working") + pipeline = GolangBuilderShipmentPipeline( + runtime=runtime, + ocp_version="4.22", + nvrs=["golang-1.25.9-1.el9"], + ) + self.assertEqual(pipeline.golang_group, "rhel-9-golang-1.25") + + +class TestShipmentFilePath(unittest.TestCase): + """Verify the shipment YAML file path matches the convention.""" + + def test_path_format(self): + application = "art-images-base" + product = "ocp" + golang_group = "rhel-9-golang-1.25" + env = "prod" + expected_prefix = Path("shipment") / "ocp" / "rhel-9-golang-1.25" / "art-images-base" / "prod" + actual = Path("shipment") / product / golang_group / application / env + self.assertEqual(actual, expected_prefix) + + +if __name__ == "__main__": + unittest.main() diff --git a/pyartcd/tests/pipelines/test_images_health.py b/pyartcd/tests/pipelines/test_images_health.py index 09e0761850..7d040f7601 100644 --- a/pyartcd/tests/pipelines/test_images_health.py +++ b/pyartcd/tests/pipelines/test_images_health.py @@ -79,33 +79,17 @@ async def test_with_build_concerns(self): "failing-image", "openshift-4.20", ConcernCode.LATEST_ATTEMPT_FAILED.value, latest_success_idx=3 ), ] - mock_slack_client.say.return_value = {"ts": "thread-123"} + mock_slack_client.say.return_value = {"ts": ""} await pipeline.notify_public_channel() - # Should post a summary message + one thread response per group (2 groups) + # Now only posts a single summary message with dashboard link calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 3) # 1 summary + 2 groups - - # Check main summary message only has alert and dashboard link - summary_args = calls[0][0][0] - self.assertIn(":alert:", summary_args) - self.assertIn("ART Build Failures Dashboard", summary_args) - # Group names should NOT be in the main message - self.assertNotIn("openshift-4.20:", summary_args) - self.assertNotIn("openshift-4.21:", summary_args) - - # Check thread responses contain group summaries - thread_1 = calls[1][0][0] - thread_2 = calls[2][0][0] - - # One should be 4.20, the other 4.21 - self.assertTrue("openshift-4.20" in thread_1 or "openshift-4.20" in thread_2) - self.assertTrue("openshift-4.21" in thread_1 or "openshift-4.21" in thread_2) - self.assertTrue("build failures" in thread_1 or "build failures" in thread_2) - - self.assertEqual(calls[1][1]['thread_ts'], "thread-123") - self.assertEqual(calls[2][1]['thread_ts'], "thread-123") + self.assertEqual(len(calls), 1) + first_call_args = calls[0][0][0] + self.assertIn(":alert:", first_call_args) + self.assertIn("build failures", first_call_args) + self.assertIn("ART Build Failures Dashboard", first_call_args) async def test_with_mixed_failure_types(self): mock_slack_client = self.mock_runtime.new_slack_client.return_value @@ -122,29 +106,19 @@ async def test_with_mixed_failure_types(self): pipeline.rebase_failures = { '4.22': {'rebase-fail-image': {'failure_count': 1, 'jenkins_url': 'http://j/3', 'build_system': 'konflux'}}, } - mock_slack_client.say.return_value = {"ts": "thread-456"} + mock_slack_client.say.return_value = {"ts": ""} await pipeline.notify_public_channel() - # Should post a summary message + one thread response for openshift-4.22 + # Now only posts a single summary message with dashboard link calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 2) # 1 summary + 1 group - - # Check main summary message only has alert and dashboard link + self.assertEqual(len(calls), 1) summary = calls[0][0][0] - self.assertIn(":alert:", summary) + self.assertIn("build failures", summary) + self.assertIn("EC verification failures", summary) + self.assertIn("release to authz failures", summary) + self.assertIn("rebase failures", summary) self.assertIn("ART Build Failures Dashboard", summary) - # Group details should NOT be in the main message - self.assertNotIn("openshift-4.22:", summary) - - # Check thread response contains group summary - thread_message = calls[1][0][0] - self.assertIn("openshift-4.22", thread_message) - self.assertIn("build failures", thread_message) - self.assertIn("EC verification failures", thread_message) - self.assertIn("release to authz failures", thread_message) - self.assertIn("rebase failures", thread_message) - self.assertEqual(calls[1][1]['thread_ts'], "thread-456") async def test_binds_to_configured_channel(self): mock_slack_client = self.mock_runtime.new_slack_client.return_value @@ -173,27 +147,17 @@ async def test_only_ec_failures(self): pipeline.ec_failures = { '4.22': {'image-a': {'failure_count': 5, 'jenkins_url': '', 'pipeline_url': 'http://ec/1'}}, } - mock_slack_client.say.return_value = {"ts": "thread-789"} + mock_slack_client.say.return_value = {"ts": ""} await pipeline.notify_public_channel() - # Should post a summary message + one thread response for openshift-4.22 + # Now only posts a single summary message with dashboard link calls = mock_slack_client.say.call_args_list - self.assertEqual(len(calls), 2) # 1 summary + 1 group - - # Check main summary message only has alert and dashboard link + self.assertEqual(len(calls), 1) summary = calls[0][0][0] - self.assertIn(":alert:", summary) + self.assertIn("EC verification failures", summary) + self.assertNotIn("build failures", summary) self.assertIn("ART Build Failures Dashboard", summary) - # Group details should NOT be in the main message - self.assertNotIn("openshift-4.22:", summary) - - # Check thread response - thread_message = calls[1][0][0] - self.assertIn("openshift-4.22", thread_message) - self.assertIn("EC verification failures", thread_message) - self.assertNotIn("build failures", thread_message) - self.assertEqual(calls[1][1]['thread_ts'], "thread-789") class TestNotifyReleaseChannel(IsolatedAsyncioTestCase): diff --git a/pyartcd/tests/pipelines/test_quay_doomsday_backup.py b/pyartcd/tests/pipelines/test_quay_doomsday_backup.py deleted file mode 100644 index 576527732a..0000000000 --- a/pyartcd/tests/pipelines/test_quay_doomsday_backup.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Unit tests for quay_doomsday_backup pipeline.""" - -from unittest import IsolatedAsyncioTestCase -from unittest.mock import AsyncMock, Mock, patch - -from pyartcd.pipelines.quay_doomsday_backup import QuayDoomsdaySync -from pyartcd.runtime import Runtime -from pyartcd.slack import SlackClient - - -class TestQuayDoomsdaySync(IsolatedAsyncioTestCase): - def setUp(self): - self.runtime = Mock(spec=Runtime) - self.runtime.dry_run = False - self.runtime.logger = Mock() - self.mock_slack_client = Mock(spec=SlackClient) - self.mock_slack_client.say_in_thread = AsyncMock( - return_value={"channel": "C123", "message": {"ts": "1234.5678"}} - ) - self.mock_slack_client._client = Mock() - self.mock_slack_client._client.reactions_add = AsyncMock() - self.runtime.new_slack_client = Mock(return_value=self.mock_slack_client) - self.version = "4.15.5" - self.arches = "x86_64,s390x" - - def _make_pipeline(self) -> QuayDoomsdaySync: - return QuayDoomsdaySync(runtime=self.runtime, version=self.version, arches=self.arches) - - @patch("pyartcd.pipelines.quay_doomsday_backup.mkdirs") - async def test_run_succeeds_when_all_arches_sync(self, _mkdirs_mock): - pipeline = self._make_pipeline() - pipeline.sync_arch = AsyncMock(return_value=True) - - await pipeline.run() - - self.assertEqual(pipeline.sync_arch.await_count, 2) - self.mock_slack_client._client.reactions_add.assert_awaited_once() - - @patch("pyartcd.pipelines.quay_doomsday_backup.mkdirs") - async def test_run_raises_when_some_arches_fail(self, _mkdirs_mock): - pipeline = self._make_pipeline() - pipeline.sync_arch = AsyncMock(side_effect=[True, False]) - - with self.assertRaisesRegex(RuntimeError, "Failed to sync arches for 4.15.5: s390x"): - await pipeline.run() - - self.mock_slack_client.say_in_thread.assert_awaited() - self.mock_slack_client._client.reactions_add.assert_not_awaited() - - @patch("pyartcd.pipelines.quay_doomsday_backup.mkdirs") - async def test_run_raises_when_all_arches_fail(self, _mkdirs_mock): - pipeline = self._make_pipeline() - pipeline.sync_arch = AsyncMock(return_value=False) - - with self.assertRaisesRegex(RuntimeError, "Failed to sync arches for 4.15.5: x86_64, s390x"): - await pipeline.run() - - self.mock_slack_client.say_in_thread.assert_awaited() - self.mock_slack_client._client.reactions_add.assert_not_awaited() diff --git a/pyartcd/tests/pipelines/test_sync_ci_images.py b/pyartcd/tests/pipelines/test_sync_ci_images.py index 3fd5be13b5..d72288565d 100644 --- a/pyartcd/tests/pipelines/test_sync_ci_images.py +++ b/pyartcd/tests/pipelines/test_sync_ci_images.py @@ -124,66 +124,6 @@ def test_validate_skip_prs_already_true_not_changed(self, mock_runtime): assert pipeline.skip_prs is True - def test_validate_auto_sets_skip_prs_for_images(self, mock_runtime): - """Test SKIP_PRS automatically set to true when IMAGES specified.""" - pipeline = SyncCIImagesPipeline( - mock_runtime, - for_release="4.17", - images="ci-openshift-base.rhel10", - skip_prs=False, - ) - assert pipeline.skip_prs is True - - def test_images_parsed_as_list(self, mock_runtime): - """Test comma-separated IMAGES string is parsed into a list.""" - pipeline = SyncCIImagesPipeline( - mock_runtime, - for_release="4.17", - images="ci-openshift-base.rhel10,ci-openshift-base.rhel9", - ) - assert pipeline.images == ["ci-openshift-base.rhel10", "ci-openshift-base.rhel9"] - - def test_images_empty_string_yields_empty_list(self, mock_runtime): - """Test empty IMAGES string yields empty list.""" - pipeline = SyncCIImagesPipeline(mock_runtime, for_release="4.17", images="") - assert pipeline.images == [] - - def test_images_whitespace_handling(self, mock_runtime): - """Test IMAGES strips whitespace around entries.""" - pipeline = SyncCIImagesPipeline( - mock_runtime, - for_release="4.17", - images=" ci-openshift-base.rhel10 , ci-openshift-base.rhel9 ", - ) - assert pipeline.images == ["ci-openshift-base.rhel10", "ci-openshift-base.rhel9"] - - def test_filter_args_with_stream_only(self, mock_runtime): - """Test _filter_args returns only --stream when no images.""" - pipeline = SyncCIImagesPipeline(mock_runtime, for_release="4.17", only_stream="rhel10") - assert pipeline._filter_args == "--stream rhel10" - - def test_filter_args_with_images_only(self, mock_runtime): - """Test _filter_args returns only --image args when no stream.""" - pipeline = SyncCIImagesPipeline(mock_runtime, for_release="4.17", images="ci-openshift-base.rhel10") - assert pipeline._filter_args == "--image ci-openshift-base.rhel10" - - def test_filter_args_with_both(self, mock_runtime): - """Test _filter_args returns both --stream and --image args.""" - pipeline = SyncCIImagesPipeline( - mock_runtime, - for_release="4.17", - only_stream="rhel10", - images="ci-openshift-base.rhel10,ci-openshift-base.rhel9", - ) - assert ( - pipeline._filter_args == "--stream rhel10 --image ci-openshift-base.rhel10 --image ci-openshift-base.rhel9" - ) - - def test_filter_args_empty_when_neither(self, mock_runtime): - """Test _filter_args returns empty string when neither stream nor images.""" - pipeline = SyncCIImagesPipeline(mock_runtime, for_release="4.17") - assert pipeline._filter_args == "" - def test_init_with_custom_data_path(self, mock_runtime): """Test initialization with custom data path and gitref.""" pipeline = SyncCIImagesPipeline(