diff --git a/ceph-dev-pipeline/build/Jenkinsfile b/ceph-dev-pipeline/build/Jenkinsfile index 94788b341..683e2cade 100644 --- a/ceph-dev-pipeline/build/Jenkinsfile +++ b/ceph-dev-pipeline/build/Jenkinsfile @@ -37,7 +37,7 @@ def get_os_info(dist) { os.pkg_type = "deb" } else if ( ubuntu_releases.keySet().contains(dist) ) { os.name = "ubuntu" - os.version = ubuntu_releases[env.DIST] + os.version = ubuntu_releases[dist] os.pkg_type = "deb" } // We need to set matcher to null right after using it to avoid a java.io.NotSerializableException @@ -141,118 +141,124 @@ def get_ceph_release_repo_text(base_url) { return text.toString() } -// Stage 1 (source distribution): trigger the ceph-source-dist job and record its number/URL OR reuse SETUP_BUILD_ID. -def doSourceDistribution() { - if ( ! env.SETUP_BUILD_ID ) { +// Stage 1 (source distribution): trigger the source-dist setup job (or reuse setupBuildId) and return its number/URL. +def doSourceDistribution(Map args) { + def buildId = args.setupBuildId + if ( !buildId ) { def setup_build = build( - job: env.SETUP_JOB, + job: args.setupJob, parameters: [ - string(name: "BRANCH", value: env.BRANCH), + string(name: "BRANCH", value: args.branch), // Below are just for ceph-source-dist - string(name: "SHA1", value: env.SHA1), - string(name: "CEPH_REPO", value: env.CEPH_REPO), - string(name: "CEPH_BUILD_BRANCH", value: env.CEPH_BUILD_BRANCH), + string(name: "SHA1", value: args.sha1), + string(name: "CEPH_REPO", value: args.cephRepo), + string(name: "CEPH_BUILD_BRANCH", value: args.cephBuildBranch), // Below are only for actual releases - string(name: 'RELEASE_TYPE', value: env.RELEASE_TYPE ?: ''), - string(name: 'RELEASE_BUILD', value: env.RELEASE_BUILD ?: ''), - string(name: 'VERSION', value: env.VERSION ?: '') + string(name: 'RELEASE_TYPE', value: args.releaseType ?: ''), + string(name: 'RELEASE_BUILD', value: args.releaseBuild ?: ''), + string(name: 'VERSION', value: args.version ?: '') ] ) - env.SETUP_BUILD_ID = setup_build.getNumber() + buildId = setup_build.getNumber() } - println "SETUP_BUILD_ID=${env.SETUP_BUILD_ID}" - env.SETUP_BUILD_URL = new URI([env.JENKINS_URL, "job", env.SETUP_JOB, env.SETUP_BUILD_ID].join("/")).normalize() - println "${env.SETUP_BUILD_URL}" + println "SETUP_BUILD_ID=${buildId}" + def buildUrl = new URI([args.jenkinsUrl, "job", args.setupJob, buildId].join("/")).normalize() + println "${buildUrl}" + return [setupBuildId: buildId, setupBuildUrl: buildUrl] } // Stage 2 (node): seed build_matrix for this cell, log the assigned node, and install the container runtime. Kept inline (no helper) so it can read the matrix-axis bindings DIST/ARCH/FLAVOR directly. // Stage 3 (checkout ceph-build): clone the ceph-build repo into the workspace. Kept inline because it's a single declarative `checkout` step with no script-block logic. -// Stage 4 (copy artifacts): copy artifacts from the setup job, verify SHA1, set VERSION/build description, and extract the source tarball. -def doCopyArtifactsStage() { +// Stage 4 (copy artifacts): copy artifacts from the setup job, verify sha1, compute version/build description, and extract the source tarball. Returns the resolved sha1/version and any extra env props from dist/other_envvars for the caller to thread into env. +def doCopyArtifactsStage(Map args) { def artifact_filter = "dist/sha1,dist/version,dist/other_envvars,dist/ceph-*.tar.bz2" - def os = get_os_info(env.DIST) - if ( env.CI_COMPILE && os.pkg_type == "deb" ) { + def os = get_os_info(args.dist) + if ( args.ciCompile && os.pkg_type == "deb" ) { artifact_filter += ",dist/ceph_*.diff.gz,dist/ceph_*.dsc" } println artifact_filter copyArtifacts( - projectName: env.SETUP_JOB, - selector: specific(buildNumber: env.SETUP_BUILD_ID), + projectName: args.setupJob, + selector: specific(buildNumber: args.setupBuildId), filter: artifact_filter, ) - sh 'sudo journalctl --show-cursor -n 0 --no-pager | tail -n1 | cut -d\" \" -f3 > $WORKSPACE/cursor' - def sha1_trimmed = env.SHA1.trim().toLowerCase() - def sha1_props = readProperties file: "${env.WORKSPACE}/dist/sha1" + sh "sudo journalctl --show-cursor -n 0 --no-pager | tail -n1 | cut -d' ' -f3 > ${args.workspace}/cursor" + def sha1_input = args.sha1 + def sha1_trimmed = sha1_input?.trim()?.toLowerCase() + def sha1_props = readProperties file: "${args.workspace}/dist/sha1" def sha1_from_artifact = sha1_props.SHA1.trim().toLowerCase() - if ( env.SHA1 && sha1_from_artifact != sha1_trimmed ) { + def sha1 = sha1_input + if ( sha1_input && sha1_from_artifact != sha1_trimmed ) { error message: "SHA1 from artifact (${sha1_from_artifact}) does not match parameter value (${sha1_trimmed})" - } else if ( ! env.SHA1 ) { - env.SHA1 = sha1_from_artifact + } else if ( ! sha1_input ) { + sha1 = sha1_from_artifact } println "SHA1=${sha1_trimmed}" - env.VERSION = readFile(file: "${env.WORKSPACE}/dist/version").trim() - // In a release build, dist/other_envvars contains CEPH_REPO, and chacra_url - // as written during ceph-source-dist but we don't to be able to define + def version = readFile(file: "${args.workspace}/dist/version").trim() + // In a release build, dist/other_envvars contains CEPH_REPO and chacra_url + // as written during ceph-source-dist but we don't want to define // ceph-releases.git or chacra.ceph.com as parameters for ceph-dev-pipeline. - def props = readProperties file: "${env.WORKSPACE}/dist/other_envvars" - for (def key in props.keySet()) { - env."${key}" = props[key] - } - def branch_ui_value = env.BRANCH - def sha1_ui_value = env.SHA1 - if (env.CEPH_REPO?.find(/https?:\/\/github.com\//)) { + def extra_env = readProperties file: "${args.workspace}/dist/other_envvars" + // Apply prop overrides on top of inputs for the UI fields rendered below. + def branch = extra_env.BRANCH ?: args.branch + def ceph_repo = extra_env.CEPH_REPO ?: args.cephRepo + def effective_sha1 = extra_env.SHA1 ?: sha1 + def branch_ui_value = branch + def sha1_ui_value = effective_sha1 + if (ceph_repo?.find(/https?:\/\/github.com\//)) { // If this is a release build, link to ceph-release.git's $BRANCH-release branch - def suffix = (env.RELEASE_BUILD?.trim() == "true") ? "-release" : "" + def suffix = (args.releaseBuild?.trim() == "true") ? "-release" : "" - def branch_url = "${env.CEPH_REPO}/tree/${env.BRANCH}${suffix}" - branch_ui_value = "${env.BRANCH}${suffix}" - def commit_url = "${env.CEPH_REPO}/commit/${env.SHA1}" - sha1_ui_value = "${env.SHA1}" + def branch_url = "${ceph_repo}/tree/${branch}${suffix}" + branch_ui_value = "${branch}${suffix}" + def commit_url = "${ceph_repo}/commit/${effective_sha1}" + sha1_ui_value = "${effective_sha1}" } - def shaman_url = "https://shaman.ceph.com/builds/ceph/${env.BRANCH}/${env.SHA1}" + def shaman_url = "https://shaman.ceph.com/builds/ceph/${branch}/${effective_sha1}" def build_description = """\ BRANCH=${branch_ui_value}
SHA1=${sha1_ui_value}
- VERSION=${env.VERSION}
- DISTROS=${env.DISTROS}
- ARCHS=${env.ARCHS}
- FLAVORS=${env.FLAVORS}
- SETUP_BUILD_ID=${env.SETUP_BUILD_ID}
+ VERSION=${version}
+ DISTROS=${args.distros}
+ ARCHS=${args.archs}
+ FLAVORS=${args.flavors}
+ SETUP_BUILD_ID=${args.setupBuildId}
shaman builds for this branch+commit """.stripIndent() buildDescription build_description sh "sha256sum dist/*" sh "cat dist/sha1 dist/version" - sh '''#!/bin/bash + sh """#!/bin/bash set -ex cd dist mkdir ceph - tar --strip-components=1 -C ceph -xjf ceph-${VERSION}.tar.bz2 ceph-${VERSION}/{container,ceph.spec,ceph.spec.in,debian,Dockerfile.build,do_cmake.sh,install-deps.sh,run-make-check.sh,make-debs.sh,make-dist,make-srpm.sh,src/script} - ''' + tar --strip-components=1 -C ceph -xjf ceph-${version}.tar.bz2 ceph-${version}/{container,ceph.spec,ceph.spec.in,debian,Dockerfile.build,do_cmake.sh,install-deps.sh,run-make-check.sh,make-debs.sh,make-dist,make-srpm.sh,src/script} + """ + return [sha1: effective_sha1, version: version, extraEnv: extra_env] } -// Stage 5 (check for built packages): check chacra for an existing build (and skip compilation if found, unless FORCE) and set up the Pulp client if needed. -def doCheckForBuiltPackagesStage() { +// Stage 5 (check for built packages): check chacra for an existing build (and disable build_matrix entry if found, unless force) and set up the Pulp client if needed. +def doCheckForBuiltPackagesStage(Map args) { sh './scripts/setup_chacractl.sh' def chacra_url = sh( script: '''grep url ~/.chacractl | cut -d'"' -f2''', returnStdout: true, ).trim() - def os = get_os_info(env.DIST) - def chacra_endpoint = "ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/${env.ARCH}/flavors/${env.FLAVOR}/" + def os = get_os_info(args.dist) + def chacra_endpoint = "ceph/${args.branch}/${args.sha1}/${os.name}/${os.version_name}/${args.arch}/flavors/${args.flavor}/" def chacractl_rc = sh( - script: "${env.HOME}/.local/bin/chacractl exists binaries/${chacra_endpoint}", + script: "${args.home}/.local/bin/chacractl exists binaries/${chacra_endpoint}", returnStatus: true, ) - if ( chacractl_rc == 0 && env.FORCE != "true" ) { + if ( chacractl_rc == 0 && args.force != "true" ) { println("Skipping compilation since chacra already has artifacts. To override, use THROWAWAY=true (to skip this check) or FORCE=true (to re-upload artifacts).") - build_matrix["${env.DIST}_${env.ARCH}"] = false + build_matrix["${args.dist}_${args.arch}"] = false } // Set up Pulp client - if ( env.PULP_UPLOAD == "true" ) { + if ( args.pulpUpload == "true" ) { withCredentials([ usernamePassword(credentialsId: 'pulp-auth', usernameVariable: 'PULP_USERNAME', passwordVariable: 'PULP_PASSWORD') ]) { @@ -261,62 +267,70 @@ def doCheckForBuiltPackagesStage() { } } -// Stage 6 (builder container): log into container registries and pull/build/push the ceph-build container image for this matrix cell. -def doBuilderContainerStage() { - env.CEPH_BUILDER_IMAGE = "${env.CONTAINER_REPO_HOSTNAME}/${env.CONTAINER_REPO_ORGANIZATION}/ceph-build" - sh '''#!/bin/bash - set -ex - podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION} - podman login -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io - ''' - def ceph_builder_tag_short = "${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}" - def ceph_builder_tag = "${env.SHA1[0..6]}.${ceph_builder_tag_short}" +// Stage 6 (builder container): log into container registries and pull/build/push the ceph-build container image for this matrix cell. Returns the image name. +def doBuilderContainerStage(Map args) { + def cephBuilderImage = "${args.containerRepoHostname}/${args.containerRepoOrganization}/ceph-build" + withEnv([ + "CONTAINER_REPO_CREDS_USR=${args.containerRepoCredsUsr}", + "CONTAINER_REPO_CREDS_PSW=${args.containerRepoCredsPsw}", + "CONTAINER_REPO_HOSTNAME=${args.containerRepoHostname}", + "CONTAINER_REPO_ORGANIZATION=${args.containerRepoOrganization}", + "DOCKER_HUB_CREDS_USR=${args.dockerHubCredsUsr}", + "DOCKER_HUB_CREDS_PSW=${args.dockerHubCredsPsw}", + ]) { + sh '''#!/bin/bash + set -ex + podman login -u ${CONTAINER_REPO_CREDS_USR} -p ${CONTAINER_REPO_CREDS_PSW} ${CONTAINER_REPO_HOSTNAME}/${CONTAINER_REPO_ORGANIZATION} + podman login -u ${DOCKER_HUB_CREDS_USR} -p ${DOCKER_HUB_CREDS_PSW} docker.io + ''' + } + def ceph_builder_tag_short = "${args.branch}.${args.dist}.${args.arch}.${args.flavor}" + def ceph_builder_tag = "${args.sha1[0..6]}.${ceph_builder_tag_short}" sh """#!/bin/bash -ex - podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} || \ - podman pull ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} || \ + podman pull ${cephBuilderImage}:${ceph_builder_tag} || \ + podman pull ${cephBuilderImage}:${ceph_builder_tag_short} || \ true """ - def withCrimson = !(env.BRANCH in ['tentacle', 'squid', 'reef']) + def withCrimson = !(args.branch in ['tentacle', 'squid', 'reef']) sh """#!/bin/bash set -ex echo > .env echo "WITH_CRIMSON=${withCrimson}" >> .env cd dist/ceph - python3 src/script/build-with-container.py --env-file=${env.WORKSPACE}/.env --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} --image-variant=packages -d ${env.DIST} -e build-container - podman tag ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} + python3 src/script/build-with-container.py --env-file=${args.workspace}/.env --image-repo=${cephBuilderImage} --tag=${ceph_builder_tag} --image-variant=packages -d ${args.dist} -e build-container + podman tag ${cephBuilderImage}:${ceph_builder_tag} ${cephBuilderImage}:${ceph_builder_tag_short} """ sh """#!/bin/bash -ex - podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag_short} - podman push ${env.CEPH_BUILDER_IMAGE}:${ceph_builder_tag} + podman push ${cephBuilderImage}:${ceph_builder_tag_short} + podman push ${cephBuilderImage}:${ceph_builder_tag} """ + return cephBuilderImage } // Stage 7 (build): run the actual package build via build-with-container.py: assemble cmake/sccache/dwz flags per distro+flavor, then extract cephadm. -def doBuildStage() { - def os = get_os_info(env.DIST) - sh "./scripts/update_shaman.sh started ceph ${os.name} ${os.version_name} ${env.ARCH}" - env.AWS_ACCESS_KEY_ID = env.SCCACHE_BUCKET_CREDS_USR - env.AWS_SECRET_ACCESS_KEY = env.SCCACHE_BUCKET_CREDS_PSW - def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}" - def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${env.DIST} --image-variant=packages --ceph-version ${env.VERSION}" +def doBuildStage(Map args) { + def os = get_os_info(args.dist) + sh "./scripts/update_shaman.sh started ceph ${os.name} ${os.version_name} ${args.arch}" + def ceph_builder_tag = "${args.sha1[0..6]}.${args.branch}.${args.dist}.${args.arch}.${args.flavor}" + def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${args.cephBuilderImage} --tag=${ceph_builder_tag} -d ${args.dist} --image-variant=packages --ceph-version ${args.version}" def bwc_command = bwc_command_base def bwc_cmd_sccache_flags = "" - if ( env.DWZ == "false" ) { - sh '''#!/bin/bash - echo "DWZ=$DWZ" >> .env - ''' - bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env"; + if ( args.dwz == "false" ) { + sh """#!/bin/bash + echo "DWZ=${args.dwz}" >> .env + """ + bwc_cmd_sccache_flags = "--env-file=${args.workspace}/.env"; } - if ( env.SCCACHE == "true" ) { - sh '''#!/bin/bash - echo "SCCACHE=$SCCACHE" >> .env + if ( args.sccache == "true" ) { + sh """#!/bin/bash + echo "SCCACHE=${args.sccache}" >> .env echo "SCCACHE_CONF=/ceph/sccache.conf" >> .env echo "SCCACHE_ERROR_LOG=/ceph/sccache_log.txt" >> .env echo "SCCACHE_LOG=debug" >> .env - echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> .env - echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> .env + echo "AWS_ACCESS_KEY_ID=${args.sccacheBucketCredsUsr}" >> .env + echo "AWS_SECRET_ACCESS_KEY=${args.sccacheBucketCredsPsw}" >> .env echo "CEPH_BUILD_NORMALIZE_PATHS=true" >> .env - ''' + """ // TODO: un-hardcode this writeFile( file: "dist/ceph/sccache.conf", @@ -331,11 +345,11 @@ def doBuildStage() { region = "auto" """ ) - bwc_cmd_sccache_flags = "--env-file=${env.WORKSPACE}/.env"; + bwc_cmd_sccache_flags = "--env-file=${args.workspace}/.env"; } def ceph_extra_cmake_args = ""; def deb_build_profiles = ""; - switch (env.FLAVOR) { + switch (args.flavor) { case "default": ceph_extra_cmake_args += " -DALLOCATOR=tcmalloc" ceph_extra_cmake_args += " -DWITH_SYSTEM_BOOST=OFF -DWITH_BOOST_VALGRIND=ON" @@ -347,13 +361,13 @@ def doBuildStage() { ceph_extra_cmake_args += " -DCMAKE_BUILD_TYPE=Debug" break default: - println "FLAVOR=${env.FLAVOR} is invalid" + println "FLAVOR=${args.flavor} is invalid" assert false } bwc_command = "${bwc_command} ${bwc_cmd_sccache_flags}" if ( os.pkg_type == "deb" ) { def sccache_flag = "-DWITH_SCCACHE=ON" - if ( env.SCCACHE == "true" && ! ceph_extra_cmake_args.contains(sccache_flag) ) { + if ( args.sccache == "true" && ! ceph_extra_cmake_args.contains(sccache_flag) ) { ceph_extra_cmake_args += " ${sccache_flag}" } if ( deb_build_profiles ) { @@ -362,34 +376,34 @@ def doBuildStage() { """ } bwc_command = "${bwc_command} -e debs" - } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) { + } else if ( args.dist =~ /^(centos|rhel|rocky|fedora).*/ ) { def rpmbuild_args = "" - if ( env.SCCACHE == "true" ) rpmbuild_args += " -R--with=sccache" - if ( env.DWZ == "false" ) rpmbuild_args += " -R--without=dwz" - if ( env.FLAVOR == "default" ) rpmbuild_args += " -R--with=tcmalloc" + if ( args.sccache == "true" ) rpmbuild_args += " -R--with=sccache" + if ( args.dwz == "false" ) rpmbuild_args += " -R--without=dwz" + if ( args.flavor == "default" ) rpmbuild_args += " -R--with=tcmalloc" bwc_command = "${bwc_command}${rpmbuild_args} -e rpm" - } else if ( env.DIST =~ /suse|sles/ ) { - throw new Exception("bwc not implemented for ${env.DIST}") - } else if ( env.DIST =~ /windows/ ) { - throw new Exception("bwc not implemented for ${env.DIST}") + } else if ( args.dist =~ /suse|sles/ ) { + throw new Exception("bwc not implemented for ${args.dist}") + } else if ( args.dist =~ /windows/ ) { + throw new Exception("bwc not implemented for ${args.dist}") } else { - throw new Exception("DIST '${env.DIST}' is invalid!") + throw new Exception("DIST '${args.dist}' is invalid!") } sh """#!/bin/bash echo "CEPH_EXTRA_CMAKE_ARGS=${ceph_extra_cmake_args}" >> .env """ sh """#!/bin/bash -ex cd dist/ceph - ln ../ceph-${env.VERSION}.tar.bz2 . + ln ../ceph-${args.version}.tar.bz2 . ${bwc_command} """ if ( os.pkg_type == "deb" ) { sh """#!/bin/bash -ex cd dist/ceph - ${bwc_command_base} -e custom -- "dpkg-deb --fsys-tarfile /ceph/debs/*/pool/main/c/ceph/cephadm_${env.VERSION}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm" + ${bwc_command_base} -e custom -- "dpkg-deb --fsys-tarfile /ceph/debs/*/pool/main/c/ceph/cephadm_${args.version}*.deb | tar -x -f - --strip-components=3 ./usr/sbin/cephadm" ln ./cephadm ../../ """ - } else if ( env.DIST =~ /^(centos|rhel|rocky|fedora).*/ ) { + } else if ( args.dist =~ /^(centos|rhel|rocky|fedora).*/ ) { sh """#!/bin/bash -ex cd dist/ceph ${bwc_command_base} -e custom -- "rpm2cpio /ceph/rpmbuild/RPMS/noarch/cephadm-*.rpm | cpio -i --to-stdout *sbin/cephadm > cephadm" @@ -399,18 +413,18 @@ def doBuildStage() { } // Stage 7 (build) post-always: archive the sccache log and remove .qa symlinks Jenkins gets stuck recursing into. -def doBuildPostAlways() { +def doBuildPostAlways(Map args) { if (fileExists('dist/ceph/sccache_log.txt')) { sh """ - if [ -f "${env.WORKSPACE}/dist/ceph/sccache_log.txt" ]; then - ln dist/ceph/sccache_log.txt sccache_log_${env.DIST}_${env.ARCH}_${env.FLAVOR}.txt + if [ -f "${args.workspace}/dist/ceph/sccache_log.txt" ]; then + ln dist/ceph/sccache_log.txt sccache_log_${args.dist}_${args.arch}_${args.flavor}.txt fi """ // The below is to work around an issue where Jenkins would recurse into // dist/ceph/qa and get stuck trying to follow the .qa symlinks. This was // discovered by seeing many messages about "too many levels of symbolic links" // in the system journal. - sh "find ${env.WORKSPACE}/dist/ceph/ -name .qa -exec rm {} \\;" + sh "find ${args.workspace}/dist/ceph/ -name .qa -exec rm {} \\;" archiveArtifacts( artifacts: 'sccache_log*.txt', allowEmptyArchive: true, @@ -420,12 +434,15 @@ def doBuildPostAlways() { } // Stage 8 (upload packages): build the ceph-release rpm (for rpm distros) and upload the packages to chacra and optionally Pulp. -def doUploadPackagesStage() { +def doUploadPackagesStage(Map args) { def chacra_url = sh( script: '''grep url ~/.chacractl | cut -d'"' -f2''', returnStdout: true, ).trim() - def os = get_os_info(env.DIST) + def os = get_os_info(args.dist) + // Push packages to chacra.ceph.com under the 'test' ref if ceph-release-pipeline's TEST=true. + // Local override only; we don't propagate this back to env since no later stage reads it. + def sha1 = args.test?.toBoolean() ? 'test' : args.sha1 if ( os.pkg_type == "rpm" ) { sh """#!/bin/bash set -ex @@ -433,24 +450,22 @@ def doUploadPackagesStage() { mkdir -p ./rpmbuild/SRPMS/ ln ceph-*.src.rpm ./rpmbuild/SRPMS/ """ - // Push packages to chacra.ceph.com under the 'test' ref if ceph-release-pipeline's TEST=true - env.SHA1 = env.TEST?.toBoolean() ? 'test' : env.SHA1 - def spec_text = get_ceph_release_spec_text("${chacra_url}r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}/") + def spec_text = get_ceph_release_spec_text("${chacra_url}r/ceph/${args.branch}/${sha1}/${os.name}/${os.version_name}/flavors/${args.flavor}/") writeFile( file: "dist/ceph/rpmbuild/SPECS/ceph-release.spec", text: spec_text, ) - def repo_text = get_ceph_release_repo_text("${chacra_url}/r/ceph/${env.BRANCH}/${env.SHA1}/${os.name}/${os.version_name}/flavors/${env.FLAVOR}") + def repo_text = get_ceph_release_repo_text("${chacra_url}/r/ceph/${args.branch}/${sha1}/${os.name}/${os.version_name}/flavors/${args.flavor}") writeFile( file: "dist/ceph/rpmbuild/SOURCES/ceph.repo", text: repo_text, ) - def ceph_builder_tag = "${env.SHA1[0..6]}.${env.BRANCH}.${env.DIST}.${env.ARCH}.${env.FLAVOR}" - def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${env.CEPH_BUILDER_IMAGE} --tag=${ceph_builder_tag} -d ${env.DIST} --image-variant=packages --ceph-version ${env.VERSION}" + def ceph_builder_tag = "${sha1[0..6]}.${args.branch}.${args.dist}.${args.arch}.${args.flavor}" + def bwc_command_base = "python3 src/script/build-with-container.py --image-repo=${args.cephBuilderImage} --tag=${ceph_builder_tag} -d ${args.dist} --image-variant=packages --ceph-version ${args.version}" def bwc_command = "${bwc_command_base} -e custom -- rpmbuild -bb --define \\'_topdir /ceph/rpmbuild\\' /ceph/rpmbuild/SPECS/ceph-release.spec" sh """#!/bin/bash set -ex - cd ${env.WORKSPACE}/dist/ceph + cd ${args.workspace}/dist/ceph ${bwc_command} """ } @@ -460,41 +475,44 @@ def doUploadPackagesStage() { export OS_VERSION="${os.version}" export OS_VERSION_NAME="${os.version_name}" export OS_PKG_TYPE="${os.pkg_type}" - if [ "${env.THROWAWAY}" != "true" ]; then ./scripts/chacra_upload.sh; fi + if [ "${args.throwaway}" != "true" ]; then ./scripts/chacra_upload.sh; fi """ sh """#!/bin/bash - if [ "${env.THROWAWAY}" != "true" ] && [ "${env.PULP_UPLOAD}" == "true" ]; then - export CEPH_VERSION="${env.VERSION}" + if [ "${args.throwaway}" != "true" ] && [ "${args.pulpUpload}" == "true" ]; then + export CEPH_VERSION="${args.version}" export OS_NAME="${os.name}" export OS_VERSION="${os.version}" export OS_VERSION_NAME="${os.version_name}" export OS_PKG_TYPE="${os.pkg_type}" - export SHA1="${env.SHA1}" - export FLAVOR="${env.FLAVOR}" + export SHA1="${sha1}" + export FLAVOR="${args.flavor}" ./scripts/pulp_upload.sh else - echo "Skipping pulp upload because PULP_UPLOAD=${env.PULP_UPLOAD}" + echo "Skipping pulp upload because PULP_UPLOAD=${args.pulpUpload}" fi """ } // Stage 9 (container): build the Ceph container image via scripts/build_container. -def doContainerStage() { - env.CONTAINER_REPO_USERNAME = env.CONTAINER_REPO_CREDS_USR - env.CONTAINER_REPO_PASSWORD = env.CONTAINER_REPO_CREDS_PSW - def os = get_os_info(env.DIST) - def cephver = env.VERSION.trim() - sh """#!/bin/bash - export DISTRO=${os.name} - export RELEASE=${os.version} - export cephver=${cephver} - if [ "\$PULP_REGISTRY_UPLOAD" == "true" ]; then - export REMOVE_LOCAL_IMAGES=false - fi - ./scripts/build_container - """ +def doContainerStage(Map args) { + def os = get_os_info(args.dist) + def cephver = args.version.trim() + withEnv([ + "CONTAINER_REPO_USERNAME=${args.containerRepoCredsUsr}", + "CONTAINER_REPO_PASSWORD=${args.containerRepoCredsPsw}", + ]) { + sh """#!/bin/bash + export DISTRO=${os.name} + export RELEASE=${os.version} + export cephver=${cephver} + if [ "${args.pulpRegistryUpload}" == "true" ]; then + export REMOVE_LOCAL_IMAGES=false + fi + ./scripts/build_container + """ + } } // Stage 10 (registry upload): mirror the just-built dev container image from Quay to the Pulp container registry. @@ -518,7 +536,22 @@ pipeline { stages { stage("source distribution") { steps { - script { doSourceDistribution() } + script { + def out = doSourceDistribution( + setupBuildId: env.SETUP_BUILD_ID, + setupJob: env.SETUP_JOB, + branch: env.BRANCH, + sha1: env.SHA1, + cephRepo: env.CEPH_REPO, + cephBuildBranch: env.CEPH_BUILD_BRANCH, + releaseType: env.RELEASE_TYPE, + releaseBuild: env.RELEASE_BUILD, + version: env.VERSION, + jenkinsUrl: env.JENKINS_URL, + ) + env.SETUP_BUILD_ID = out.setupBuildId + env.SETUP_BUILD_URL = out.setupBuildUrl + } } } stage("parallel build") { @@ -605,7 +638,26 @@ pipeline { } stage("copy artifacts") { steps { - script { doCopyArtifactsStage() } + script { + def out = doCopyArtifactsStage( + setupJob: env.SETUP_JOB, + setupBuildId: env.SETUP_BUILD_ID, + setupBuildUrl: env.SETUP_BUILD_URL, + ciCompile: env.CI_COMPILE, + dist: env.DIST, + workspace: env.WORKSPACE, + sha1: env.SHA1, + cephRepo: env.CEPH_REPO, + releaseBuild: env.RELEASE_BUILD, + branch: env.BRANCH, + distros: env.DISTROS, + archs: env.ARCHS, + flavors: env.FLAVORS, + ) + env.SHA1 = out.sha1 + env.VERSION = out.version + out.extraEnv.each { k, v -> env."${k}" = v } + } } } stage("check for built packages") { @@ -618,7 +670,18 @@ pipeline { SHAMAN_API_KEY = credentials('shaman-api-key') } steps { - script { doCheckForBuiltPackagesStage() } + script { + doCheckForBuiltPackagesStage( + dist: env.DIST, + branch: env.BRANCH, + sha1: env.SHA1, + arch: env.ARCH, + flavor: env.FLAVOR, + home: env.HOME, + force: env.FORCE, + pulpUpload: env.PULP_UPLOAD, + ) + } } } stage("builder container") { @@ -630,7 +693,22 @@ pipeline { expression { return build_matrix["${DIST}_${ARCH}"] == true } } steps { - script { doBuilderContainerStage() } + script { + env.CEPH_BUILDER_IMAGE = doBuilderContainerStage( + containerRepoHostname: env.CONTAINER_REPO_HOSTNAME, + containerRepoOrganization: env.CONTAINER_REPO_ORGANIZATION, + containerRepoCredsUsr: env.CONTAINER_REPO_CREDS_USR, + containerRepoCredsPsw: env.CONTAINER_REPO_CREDS_PSW, + dockerHubCredsUsr: env.DOCKER_HUB_CREDS_USR, + dockerHubCredsPsw: env.DOCKER_HUB_CREDS_PSW, + branch: env.BRANCH, + dist: env.DIST, + arch: env.ARCH, + flavor: env.FLAVOR, + sha1: env.SHA1, + workspace: env.WORKSPACE, + ) + } } } stage("build") { @@ -642,11 +720,33 @@ pipeline { expression { return build_matrix["${DIST}_${ARCH}"] == true } } steps { - script { doBuildStage() } + script { + doBuildStage( + dist: env.DIST, + sccacheBucketCredsUsr: env.SCCACHE_BUCKET_CREDS_USR, + sccacheBucketCredsPsw: env.SCCACHE_BUCKET_CREDS_PSW, + sha1: env.SHA1, + branch: env.BRANCH, + arch: env.ARCH, + flavor: env.FLAVOR, + cephBuilderImage: env.CEPH_BUILDER_IMAGE, + version: env.VERSION, + dwz: env.DWZ, + sccache: env.SCCACHE, + workspace: env.WORKSPACE, + ) + } } post { always { - script { doBuildPostAlways() } + script { + doBuildPostAlways( + workspace: env.WORKSPACE, + dist: env.DIST, + arch: env.ARCH, + flavor: env.FLAVOR, + ) + } } unsuccessful { script { @@ -665,7 +765,21 @@ pipeline { expression { return build_matrix["${DIST}_${ARCH}"] == true } } steps { - script { doUploadPackagesStage() } + script { + doUploadPackagesStage( + dist: env.DIST, + test: env.TEST, + branch: env.BRANCH, + sha1: env.SHA1, + flavor: env.FLAVOR, + cephBuilderImage: env.CEPH_BUILDER_IMAGE, + arch: env.ARCH, + version: env.VERSION, + workspace: env.WORKSPACE, + throwaway: env.THROWAWAY, + pulpUpload: env.PULP_UPLOAD, + ) + } } post { success { @@ -690,7 +804,15 @@ pipeline { CONTAINER_REPO_CREDS = credentials('quay-ceph-io-ceph-ci') } steps { - script { doContainerStage() } + script { + doContainerStage( + dist: env.DIST, + version: env.VERSION, + containerRepoCredsUsr: env.CONTAINER_REPO_CREDS_USR, + containerRepoCredsPsw: env.CONTAINER_REPO_CREDS_PSW, + pulpRegistryUpload: env.PULP_REGISTRY_UPLOAD, + ) + } } } stage("registry upload") {