diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c82a25f920..99d751e010 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -118,7 +118,7 @@ jobs: - name: Remove 'content-ok' label uses: actions/github-script@v3 - if: ${{ steps.check_pr_content.outcome == 'failure'}} + if: ${{ steps.check_pr_content.outcome == 'failure' && contains( github.event.pull_request.labels.*.name, 'content-ok') }} continue-on-error: true with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -138,7 +138,7 @@ jobs: - name: Remove 'authorized-request' label from PR uses: actions/github-script@v3 - if: ${{ steps.check_build_required.outputs.run-build == 'true' }} + if: ${{ steps.check_build_required.outputs.run-build == 'true' && contains( github.event.pull_request.labels.*.name, 'authorized-request') }} continue-on-error: true with: github-token: ${{ secrets.GITHUB_TOKEN }} @@ -225,7 +225,7 @@ jobs: KUBECONFIG: /tmp/ci-kubeconfig run: | API_SERVER=$( echo -n ${{ secrets.API_SERVER }} | base64 -d) - oc login --token=${{ secrets.CLUSTER_TOKEN }} --server=${API_SERVER} + oc login --token=${{ secrets.CLUSTER_TOKEN }} --server=${API_SERVER} --insecure-skip-tls-verify=${{ steps.set-env.outputs.insecure_skip_tls_verify }} ve1/bin/sa-for-chart-testing --delete charts-${{ github.event.number }} - name: Save PR artifact diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c664a7bcf..b16d02b61b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -105,10 +105,10 @@ jobs: echo "Full test in pr : {{steps.check_request.outputs.full_tests_in_pr }}" if ${{steps.check_if_release_pr.outputs.charts_release_branch == 'true' || steps.check_request.outputs.full_tests_in_pr == 'true' }} ; then echo "Release PR from dev to charts, oer PR with new full test, so running full tests" - ve1/bin/pytest tests/ --log-cli-level=WARNING --ignore=tests/functional/step_defs/test_smoke_scenarios.py --ignore=tests/functional/step_defs/test_submitted_charts.py --tb=short + ve1/bin/behave tests/functional/behave_features/ --tags=full --logging-level=WARNING --no-capture --no-color else echo "Not a release PR from dev to charts, so running only smoke tests" - ve1/bin/pytest tests/functional/step_defs/test_smoke_scenarios.py --log-cli-level=WARNING --tb=short + ve1/bin/behave tests/functional/behave_features/ --tags=smoke --logging-level=WARNING --no-capture --no-color fi - name: (Manual) Test CI Workflow @@ -130,7 +130,7 @@ jobs: echo "[INFO] Notify ID '${{ env.NOTIFY_ID }}'" echo "[INFO] Software Name '${{ env.SOFTWARE_NAME }}'" echo "[INFO] Software Version '${{ env.SOFTWARE_VERSION }}'" - ve1/bin/pytest tests/functional/step_defs/test_submitted_charts.py --log-cli-level=WARNING --tb=short + ve1/bin/behave tests/functional/behave_features/ --tags=version-change --logging-level=WARNING --no-capture --no-color - name: Approve PR id: approve_pr diff --git a/.github/workflows/version_check.yml b/.github/workflows/version_check.yml index 64aa0503f0..33c788def7 100644 --- a/.github/workflows/version_check.yml +++ b/.github/workflows/version_check.yml @@ -10,9 +10,13 @@ on: # 2. Runs this workflow whether or not a version change has occurred # 3. By default dperaza and mmulholla are tagged in any issues raised. dry-run: - description: "Dry Run? (Run tests but create issues in sandbox) {true,false}" + description: "Dry Run? (Unconditionally run tests and create issues in sandbox) {true,false}" required: true default: "true" + update-version: + description: "Dry run also checks and updates software-version file if not charts repository" + required: true + default: "false" vendor-type: description: "Vendor type {all,partner,redhat,community}" required: true @@ -32,14 +36,28 @@ jobs: run: | echo "GITHUB_EVENT_NAME : $GITHUB_EVENT_NAME" echo "GITHUB_REPOSITORY : $GITHUB_REPOSITORY" + echo "dry-run : ${{ github.event.inputs.dry-run }}" + echo "update-version : ${{ github.event.inputs.update-version }}" if [ $GITHUB_EVENT_NAME == 'workflow_dispatch' ]; then echo '::set-output name=run-job::true' + if [ "${{ github.event.inputs.dry-run }}" == "true" ]; then + if [[ "${{ github.event.inputs.update-version }}" == "true" && $GITHUB_REPOSITORY != "openshift-helm-charts/charts" ]]; then + echo '::set-output name=check-version::true' + else + echo '::set-output name=check-version::false' + fi + else + echo '::set-output name=check-version::true' + fi elif [ $GITHUB_REPOSITORY == "openshift-helm-charts/charts" ]; then echo '::set-output name=run-job::true' + echo '::set-output name=check-version::true' else echo '::set-output name=run-job::false' + echo '::set-output name=check-version::false' fi + - name: Install oc if: steps.check_repo.outputs.run-job == 'true' run: | @@ -58,56 +76,67 @@ jobs: id: get_curr_ocp_version run: | OCP_VERSION=$(./oc version -o json | jq '.openshiftVersion') + OCP_VERSION=$(sed -e 's/^"//' -e 's/"$//' <<< $OCP_VERSION) printf "[INFO] Current OCP Version: %s\n" ${OCP_VERSION} echo "::set-output name=curr_ocp_version::${OCP_VERSION}" shell: bash - name: Checkout software-version branch - if: steps.check_repo.outputs.run-job == 'true' + if: steps.check_repo.outputs.check-version == 'true' uses: actions/checkout@v2 with: ref: "software-version" - repository: "openshift-helm-charts/charts" + repository: ${{ github.repository }} - name: Read previous OpenShift version id: get_prev_ocp_version - if: steps.check_repo.outputs.run-job == 'true' + if: steps.check_repo.outputs.check-version == 'true' uses: mikefarah/yq@master with: cmd: yq e '.openshift.release-client-version' software-version.yaml - - name: Compare OpenShift versions - id: compare_ocp_versions + - name: Check if test should run + id: check_test run: | set -euo pipefail if [ "${{ steps.check_repo.outputs.run-job }}" != "true" ]; then echo "::set-output name=run_tests::false" - elif [ "${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }}" = "${{ steps.get_prev_ocp_version.outputs.result }}" ]; then - # No change in the OpenShift version - do not run tests if a scheduled run or dry-run is not set - if [ "${{ github.event_name }}" == 'schedule' || "${{ github.event.inputs.dry-run }}" != 'true']; then + echo "::set-output name=update-version::false" + elif [ "${{ steps.check_repo.outputs.check-version }}" == "true" ]; then + if [ "${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }}" == "${{ steps.get_prev_ocp_version.outputs.result }}" ]; then + # No change in the OpenShift versions. printf "OpenShift version has not changed since last run: '%s' -> '%s'\n" "${{ steps.get_prev_ocp_version.outputs.result }}" "${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }}" - echo "::set-output name=run_tests::false" + echo "::set-output name=update-version::false" + if [ "${{ github.event.inputs.dry-run }}" == "true" ]; then + echo "Openshift version has not changed but run anyaway as dry-run is set" + echo "::set-output name=run_tests::true" + else + echo "Openshift version has not changed do not run tests" + echo "::set-output name=run_tests::false" + fi else + printf "OpenShift version has changed since last run: '%s' -> '%s'\n" "${{ steps.get_prev_ocp_version.outputs.result }}" "${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }}" echo "::set-output name=run_tests::true" + echo "::set-output name=update-version::true" fi else - # New OpenShift version is set - printf "OpenShift version has changed since last run: '%s' -> '%s'\n" "${{ steps.get_prev_ocp_version.outputs.result }}" "${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }}" + # Run whether open shift version has changed or not + echo "Run tests - version check skipped" + echo "::set-output name=update-version::false" echo "::set-output name=run_tests::true" fi shell: bash - name: Update software-version.yaml if: | - steps.compare_ocp_versions.outputs.run_tests == 'true' + steps.check_test.outputs.update-version == 'true' uses: mikefarah/yq@master with: - cmd: yq eval -i '.openshift.release-client-version = ${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }}' 'software-version.yaml' + cmd: yq eval -i '.openshift.release-client-version = "${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }}"' 'software-version.yaml' - name: Push software-version.yaml if: | - steps.compare_ocp_versions.outputs.run_tests == 'true' && - (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry-run != 'true')) + steps.check_test.outputs.update-version == 'true' run: | COMMIT_MESSAGE=$(printf "software-version.yaml: Update OpenShift version from '%s' to '%s'" "${{ steps.get_prev_ocp_version.outputs.result }}" "${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }}") git remote -v @@ -120,7 +149,7 @@ jobs: - name: Checkout main branch if: | - steps.compare_ocp_versions.outputs.run_tests == 'true' + steps.check_test.outputs.run_tests == 'true' uses: actions/checkout@v2 with: ref: "main" @@ -129,14 +158,14 @@ jobs: - name: Set up Python 3.x Part 1 if: | - steps.compare_ocp_versions.outputs.run_tests == 'true' + steps.check_test.outputs.run_tests == 'true' uses: actions/setup-python@v2 with: python-version: "3.9" - name: Set up Python 3.x Part 2 if: | - steps.compare_ocp_versions.outputs.run_tests == 'true' + steps.check_test.outputs.run_tests == 'true' run: | # set up python python3 -m venv ve1 @@ -145,7 +174,7 @@ jobs: - name: (Manual) Run tests on existing charts if: | - github.event_name == 'workflow_dispatch' && steps.compare_ocp_versions.outputs.run_tests == 'true' + github.event_name == 'workflow_dispatch' && steps.check_test.outputs.run_tests == 'true' env: CLUSTER_TOKEN: ${{ secrets.CLUSTER_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -162,19 +191,18 @@ jobs: printf "[INFO] Notify ID: '%s'\n" "${{ env.NOTIFY_ID }}" printf "[INFO] Software Name: '%s'\n" "${{ env.SOFTWARE_NAME }}" printf "[INFO] Software Version: '%s'\n" "${{ env.SOFTWARE_VERSION }}" - ve1/bin/pytest tests/functional/step_defs/test_submitted_charts.py --log-cli-level=INFO --tb=short + ve1/bin/behave tests/functional/behave_features/ --tags=version-change --logging-level=INFO --no-capture --no-color - name: (Schedule) Run tests on existing charts id: run-schedule-tests if: | - github.event_name == 'schedule' && steps.compare_ocp_versions.outputs.run_tests == 'true' + github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' env: CLUSTER_TOKEN: ${{ secrets.CLUSTER_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} BOT_NAME: ${{ secrets.BOT_NAME }} BOT_TOKEN: ${{ secrets.BOT_TOKEN }} - # XXX: set to false when ready to launch notifications - DRY_RUN: "true" + DRY_RUN: "false" VENDOR_TYPE: "all" NOTIFY_ID: "" SOFTWARE_NAME: "OpenShift" @@ -185,22 +213,24 @@ jobs: printf "[INFO] Notify ID: '%s'\n" "${{ env.NOTIFY_ID }}" printf "[INFO] Software Name: '%s'\n" "${{ env.SOFTWARE_NAME }}" printf "[INFO] Software Version: '%s'\n" "${{ env.SOFTWARE_VERSION }}" - ve1/bin/pytest tests/functional/step_defs/test_submitted_charts.py --log-cli-level=INFO --tb=short + ve1/bin/behave tests/functional/behave_features/ --tags=version-change --logging-level=INFO --no-capture --no-color - name: Send message to slack channel id: notify - if: always() && github.event_name == 'schedule' && steps.compare_cv_versions.outputs.run_tests == 'true' + if: always() && github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' uses: archive/github-actions-slack@v2.0.0 with: slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} slack-channel: C02979BDUPL - slack-text: ${{ steps.run-schedule-tests.conclusion }}! Nightly run after OCP version update detected. See '${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}' + slack-text: ${{ steps.run-schedule-tests.conclusion }}! Nightly run after an OpenShift version update to ${{ steps.get_curr_ocp_version.outputs.curr_ocp_version }} was detected. See '${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}' - name: Result from "Send Message to slack channel" run: echo "The result was ${{ steps.notify.outputs.slack-result }}" check-chart-verifier: + if: ${{ always() }} + needs: check-ocp name: Check Chart Verifier Version runs-on: ubuntu-20.04 steps: @@ -209,15 +239,25 @@ jobs: run: | echo "GITHUB_EVENT_NAME : $GITHUB_EVENT_NAME" echo "GITHUB_REPOSITORY : $GITHUB_REPOSITORY" + echo "dry-run : ${{ github.event.inputs.dry-run }}" + echo "update-version : ${{ github.event.inputs.update-version }}" if [ $GITHUB_EVENT_NAME == 'workflow_dispatch' ]; then echo '::set-output name=run-job::true' - echo "workflow_dispatch: set run-job to true" + if [ "${{ github.event.inputs.dry-run }}" == "true" ]; then + if [[ "${{ github.event.inputs.update-version }}" == "true" && $GITHUB_REPOSITORY != "openshift-helm-charts/charts" ]]; then + echo '::set-output name=check-version::true' + else + echo '::set-output name=check-version::false' + fi + else + echo '::set-output name=check-version::true' + fi elif [ $GITHUB_REPOSITORY == "openshift-helm-charts/charts" ]; then echo '::set-output name=run-job::true' - echo "charts repo: set run-job to true" + echo '::set-output name=check-version::true' else echo '::set-output name=run-job::false' - echo "set run-job to false" + echo '::set-output name=check-version::false' fi - name: Get current Chart Verifier version @@ -231,51 +271,63 @@ jobs: shell: bash - name: Checkout software-version branch - if: steps.check_repo.outputs.run-job == 'true' + if: steps.check_repo.outputs.check-version == 'true' uses: actions/checkout@v2 with: ref: "software-version" - repository: "openshift-helm-charts/charts" + repository: ${{ github.repository }} - name: Read previous Chart Verifier digest - if: steps.check_repo.outputs.run-job == 'true' + if: steps.check_repo.outputs.check-version == 'true' id: get_prev_cv_digest uses: mikefarah/yq@master with: cmd: yq e '.chart-verifier.latest-manifest-digest' software-version.yaml - name: Compare Chart Verifier versions - id: compare_cv_versions + id: check_test run: | set -euo pipefail if [ "${{ steps.check_repo.outputs.run-job }}" != "true" ]; then echo "::set-output name=run_tests::false" - elif [ "${{ steps.get_curr_cv_version.outputs.current_cv_digest }}" == "${{ steps.get_prev_cv_digest.outputs.result }}" ]; then - # No change in the Chart Verifier image - do not run tests if a scheduled run or dry-run is not set - if [ "${{ github.event_name }}" == 'schedule' || "${{ github.event.inputs.dry-run }}" != 'true']; then - printf "Chart Verifier has not changed since last run: '%s' -> '%s'\n" "${{ steps.get_prev_cv_digest.outputs.result }}" "${{ steps.get_curr_cv_version.outputs.current_cv_digest }}" - echo "::set-output name=run_tests::false" - else - echo "::set-output name=run_tests::true" - fi + echo "::set-output name=update-version::false" + elif [ "${{ steps.check_repo.outputs.check-version }}" == "true" ]; then + if [ "${{ steps.get_curr_cv_version.outputs.current_cv_digest }}" == "${{ steps.get_prev_cv_digest.outputs.result }}" ]; then + # No change in the Chart Verifier image - do not run tests if a scheduled run or dry-run is not set + printf "Chart Verifier has not changed since last run: '%s' -> '%s'\n" "${{ steps.get_prev_cv_digest.outputs.result }}" "${{ steps.get_curr_cv_version.outputs.current_cv_digest }}" + echo "::set-output name=update-version::false" + if [ "${{ github.event.inputs.dry-run }}" == "true" ]; then + echo "Chart Verifier image has not changed but run anyaway as dry-run is set" + echo "::set-output name=run_tests::true" + else + echo "Chart Verifier image has not changed do not run tests" + echo "::set-output name=run_tests::false" + fi + else + # New Chart Verifier image is found + printf "Chart Verifier has changed since last run: '%s' -> '%s'\n" "${{ steps.get_prev_cv_digest.outputs.result }}" "${{ steps.get_curr_cv_version.outputs.current_cv_digest }}" + echo "::set-output name=run_tests::true" + echo "::set-output name=update-version::true" + fi else - # New Chart Verifier image is found - printf "Chart Verifier has changed since last run: '%s' -> '%s'\n" "${{ steps.get_prev_cv_digest.outputs.result }}" "${{ steps.get_curr_cv_version.outputs.current_cv_digest }}" + # Run whether Chart Verifier image has changed or not + echo "Run tests - version check skipped" + echo "::set-output name=update-version::false" echo "::set-output name=run_tests::true" fi + shell: bash - name: Update software-version.yaml if: | - steps.compare_cv_versions.outputs.run_tests == 'true' + steps.check_test.outputs.update-version == 'true' uses: mikefarah/yq@master with: cmd: yq eval -i '.chart-verifier.latest-manifest-digest = ${{ steps.get_curr_cv_version.outputs.current_cv_digest }}' 'software-version.yaml' - name: Push software-version.yaml if: | - steps.compare_cv_versions.outputs.run_tests == 'true' && - (github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && github.event.inputs.dry-run != 'true')) + steps.check_test.outputs.update-version == 'true' run: | COMMIT_MESSAGE=$(printf "software-version.yaml: Update chart-verifier version from '%s' to '%s'" "${{ steps.get_prev_ocp_version.outputs.result }}" "${{ steps.get_curr_cv_version.outputs.current_cv_digest }}") git remote -v @@ -288,7 +340,7 @@ jobs: - name: Checkout charts main branch if: | - steps.compare_cv_versions.outputs.run_tests == 'true' + steps.check_test.outputs.run_tests == 'true' uses: actions/checkout@v2 with: ref: "main" @@ -297,14 +349,14 @@ jobs: - name: Set up Python 3.x Part 1 if: | - steps.compare_cv_versions.outputs.run_tests == 'true' + steps.check_test.outputs.run_tests == 'true' uses: actions/setup-python@v2 with: python-version: "3.9" - name: Set up Python 3.x Part 2 if: | - steps.compare_cv_versions.outputs.run_tests == 'true' + steps.check_test.outputs.run_tests == 'true' run: | # set up python pwd @@ -314,7 +366,7 @@ jobs: - name: (Manual) Run tests on existing charts if: | - github.event_name == 'workflow_dispatch' + github.event_name == 'workflow_dispatch' && steps.check_test.outputs.run_tests == 'true' env: CLUSTER_TOKEN: ${{ secrets.CLUSTER_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -331,12 +383,12 @@ jobs: printf "[INFO] Notify ID: '%s'\n" "${{ env.NOTIFY_ID }}" printf "[INFO] Software Name: '%s'\n" "${{ env.SOFTWARE_NAME }}" printf "[INFO] Software Version: '%s'\n" "${{ env.SOFTWARE_VERSION }}" - ve1/bin/pytest tests/functional/step_defs/test_submitted_charts.py --log-cli-level=INFO --tb=short + ve1/bin/behave tests/functional/behave_features/ --tags=version-change --logging-level=INFO --no-capture --no-color - name: (Schedule) Run tests on existing charts id: run-schedule-tests if: | - github.event_name == 'schedule' && steps.compare_cv_versions.outputs.run_tests == 'true' + github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' env: CLUSTER_TOKEN: ${{ secrets.CLUSTER_TOKEN }} GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -354,17 +406,21 @@ jobs: printf "[INFO] Notify ID: '%s'\n" "${{ env.NOTIFY_ID }}" printf "[INFO] Software Name: '%s'\n" "${{ env.SOFTWARE_NAME }}" printf "[INFO] Software Version: '%s'\n" "${{ env.SOFTWARE_VERSION }}" - ve1/bin/pytest tests/functional/step_defs/test_submitted_charts.py --log-cli-level=INFO --tb=short + ve1/bin/behave tests/functional/behave_features/ --tags=version-change --logging-level=INFO --no-capture --no-color - name: Send message to slack channel id: notify - if: always() && github.event_name == 'schedule' && steps.compare_cv_versions.outputs.run_tests == 'true' + if: always() && github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' uses: archive/github-actions-slack@v2.0.0 + env: + SOFTWARE_NAME: "chart-verifier" + SOFTWARE_VERSION: ${{ steps.get_curr_cv_version.outputs.current_cv_digest }} with: slack-bot-user-oauth-access-token: ${{ secrets.SLACK_BOT_USER_OAUTH_ACCESS_TOKEN }} slack-channel: C02979BDUPL - slack-text: ${{ steps.run-schedule-tests.conclusion }}! Nightly run after chartverifier version update detected. See '${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}' + slack-text: ${{ steps.run-schedule-tests.conclusion }}! Nightly run after a chart-verifier version update to ${{ steps.get_curr_cv_version.outputs.current_cv_digest }} was detected. See '${{github.server_url}}/${{github.repository}}/actions/runs/${{github.run_id}}' - name: Result from "Send Message to slack channel" + if: always() && github.event_name == 'schedule' && steps.check_test.outputs.run_tests == 'true' run: echo "The result was ${{ steps.notify.outputs.slack-result }}" diff --git a/scripts/requirements.txt b/scripts/requirements.txt index 36dd5b5a09..c9eb576899 100644 --- a/scripts/requirements.txt +++ b/scripts/requirements.txt @@ -34,3 +34,4 @@ toml==0.10.2 urllib3==1.26.5 websocket-client==1.2.1 analytics-python==1.4.0 +behave==1.2.6 diff --git a/scripts/src/chartrepomanager/indexannotations.py b/scripts/src/chartrepomanager/indexannotations.py index 1b8ee44758..d2b593cf9c 100644 --- a/scripts/src/chartrepomanager/indexannotations.py +++ b/scripts/src/chartrepomanager/indexannotations.py @@ -1,18 +1,22 @@ import sys import semantic_version +import requests +import yaml sys.path.append('../') from report import report_info -kubeOpenShiftVersionMap = {"1.13": "4.1", - "1.14": "4.2", - "1.16": "4.3", - "1.17": "4.4", - "1.18": "4.5", - "1.19": "4.6", - "1.20": "4.7", - "1.21": "4.8", - "1.22": "4.9"} +kubeOpenShiftVersionMap = {} + +def getKubVersionMap(): + + if not kubeOpenShiftVersionMap: + content = requests.get("https://github.com/redhat-certification/chart-verifier/blob/main/internal/tool/kubeOpenShiftVersionMap.yaml?raw=true") + version_data = yaml.safe_load(content.text) + for kubeVersion in version_data["versions"]: + kubeOpenShiftVersionMap[kubeVersion["kube-version"]] = kubeVersion["ocp-version"] + + return kubeOpenShiftVersionMap def getOCPVersions(kubeVersion): @@ -56,21 +60,26 @@ def getOCPVersions(kubeVersion): minOCP = "" maxOCP = "" + getKubVersionMap() for kubeVersionKey in kubeOpenShiftVersionMap : + #print(f"\n Map entry : {kubeVersionKey}: {kubeOpenShiftVersionMap[kubeVersionKey]}") + #print(f" MinOCP : {minOCP}, maxOCP: {maxOCP}") coercedKubeVersionKey = semantic_version.Version.coerce(kubeVersionKey) - if minOCP == "" and coercedKubeVersionKey in semantic_version.NpmSpec(checkKubeVersion): - minOCP = kubeOpenShiftVersionMap[kubeVersionKey] - print(f" Found min : {kubeVersion}: {minOCP}") - elif coercedKubeVersionKey in semantic_version.NpmSpec(checkKubeVersion): - maxOCP = kubeOpenShiftVersionMap[kubeVersionKey] - print(f" Found new Max : {kubeVersion}: {maxOCP}") + if coercedKubeVersionKey in semantic_version.NpmSpec(checkKubeVersion): + coercedOCPVersionValue = semantic_version.Version.coerce(kubeOpenShiftVersionMap[kubeVersionKey]) + if minOCP == "" or semantic_version.Version.coerce(minOCP) > coercedOCPVersionValue: + minOCP = kubeOpenShiftVersionMap[kubeVersionKey] + #print(f" Found new min : {checkKubeVersion}: {minOCP}") + if maxOCP == "" or semantic_version.Version.coerce(maxOCP) < coercedOCPVersionValue: + maxOCP = kubeOpenShiftVersionMap[kubeVersionKey] + #print(f" Found new Max : {checkKubeVersion}: {maxOCP}") # check if minOCP is open ended if minOCP != "" and semantic_version.Version("1.999.999") in semantic_version.NpmSpec(checkKubeVersion): ocp_versions = f">={minOCP}" elif minOCP == "": ocp_versions = "N/A" - elif maxOCP == "": + elif maxOCP == "" or maxOCP == minOCP: ocp_versions = minOCP else: ocp_versions = f"{minOCP} - {maxOCP}" diff --git a/scripts/src/checkprcontent/checkpr.py b/scripts/src/checkprcontent/checkpr.py index 7976d0e950..65fadcf9e3 100644 --- a/scripts/src/checkprcontent/checkpr.py +++ b/scripts/src/checkprcontent/checkpr.py @@ -84,6 +84,24 @@ def check_provider_delivery(report_in_pr,num_files_in_pr,report_file_match): print(f"::set-output name=providerDelivery::False") print(f"[INFO] providerDelivery is a no-go") +def get_file_match_compiled_patterns(): + """Return a tuple of patterns, where the first can be used to match any file in a chart PR + and the second can be used to match a valid report file within a chart PR. The patterns + match based on the relative path of a file to the base repository + + Both patterns capture chart type, chart vendor, chart name and chart version from the file path.. + + Examples of valid file paths are: + + charts/partners/hashicorp/vault/0.20.0/ + charts/partners/hashicorp/vault/0.20.0//report.yaml + """ + + pattern = re.compile(r"charts/"+TYPE_MATCH_EXPRESSION+"/([\w-]+)/([\w-]+)/([\w\.-]+)/.*") + reportpattern = re.compile(r"charts/"+TYPE_MATCH_EXPRESSION+"/([\w-]+)/([\w-]+)/([\w\.-]+)/report.yaml") + + return pattern,reportpattern + def ensure_only_chart_is_modified(api_url, repository, branch): # api_url https://api.github.com/repos///pulls/1 @@ -95,8 +113,7 @@ def ensure_only_chart_is_modified(api_url, repository, branch): files_api_url = f'{api_url}/files' headers = {'Accept': 'application/vnd.github.v3+json'} r = requests.get(files_api_url, headers=headers) - pattern = re.compile(r"charts/"+TYPE_MATCH_EXPRESSION+"/([\w-]+)/([\w-]+)/([\w\.-]+)/.*") - reportpattern = re.compile(r"charts/"+TYPE_MATCH_EXPRESSION+"/([\w-]+)/([\w-]+)/([\w\.-]+)/report.yaml") + pattern,reportpattern = get_file_match_compiled_patterns() page_number = 1 max_page_size,page_size = 100,100 matches_found = 0 @@ -129,9 +146,10 @@ def ensure_only_chart_is_modified(api_url, repository, branch): if matches_found == 1: pattern_match = match elif pattern_match.groups() != match.groups(): - msg = f"[ERROR] PR must only include one chart" + msg = "[ERROR] A PR must contain only one chart. Current PR includes files for multiple charts." print(msg) print(f"::set-output name=pr-content-error-message::{msg}") + exit(1) if none_chart_files: if file_count > 1 or "OWNERS" not in none_chart_files: #OWNERS not present or preset but not the only file diff --git a/scripts/src/indexfile/index.py b/scripts/src/indexfile/index.py index 97f20049ff..271620cc30 100644 --- a/scripts/src/indexfile/index.py +++ b/scripts/src/indexfile/index.py @@ -2,15 +2,16 @@ import json import requests import yaml +import semantic_version +import sys -def _make_http_request(method, url, body=None, params={}, headers={}, verbose=False): - method_map = {"get": requests.get, - "post": requests.post, - "put": requests.put, - "delete": requests.delete, - "patch": requests.patch} - request_method = method_map[method] - response = request_method(url, params=params, headers=headers, json=body) +sys.path.append('../') +from chartrepomanager import indexannotations + +INDEX_FILE = "https://charts.openshift.io/index.yaml" + +def _make_http_request(url, body=None, params={}, headers={}, verbose=False): + response = requests.get(url, params=params, headers=headers, json=body) if verbose: print(json.dumps(headers, indent=4, sort_keys=True)) print(json.dumps(body, indent=4, sort_keys=True)) @@ -18,14 +19,13 @@ def _make_http_request(method, url, body=None, params={}, headers={}, verbose=Fa print(response.text) return response.text -def _load_index_yaml(url): - - yaml_text = _make_http_request('get', url) +def _load_index_yaml(): + yaml_text = _make_http_request(INDEX_FILE) dct = yaml.safe_load(yaml_text) return dct def get_chart_info(tar_name): - index_dct = _load_index_yaml("https://charts.openshift.io/index.yaml") + index_dct = _load_index_yaml() for entry, charts in index_dct["entries"].items(): if tar_name.startswith(entry): for chart in charts: @@ -38,5 +38,97 @@ def get_chart_info(tar_name): print(f"[INFO] match not found: {tar_name}") return "","","","" +def get_charts_info(): + chart_info_list = [] + + index_dct = _load_index_yaml() + for entry, charts in index_dct["entries"].items(): + for chart in charts: + chart_info = {} + chart_info["name"] = chart['name'] + chart_info["version"] = chart["version"] + chart_info["providerType"] = chart["annotations"]["charts.openshift.io/providerType"] + chart_info["provider"] = entry.removesuffix(f'-{chart["name"]}') + #print(f'[INFO] found chart : {chart_info["provider"]} {chart["name"]} {chart["version"]} ') + if 'charts.openshift.io/supportedOpenShiftVersions' in chart["annotations"]: + chart_info["supportedOCP"] = chart["annotations"]["charts.openshift.io/supportedOpenShiftVersions"] + else: + chart_info["supportedOCP"] = "" + if "kubeVersion" in chart: + chart_info["kubeVersion"] = chart["kubeVersion"] + else: + chart_info["kubeVersion"] ="" + chart_info_list.append(chart_info) + + return chart_info_list + +def get_latest_charts(): + chart_list = get_charts_info() + + print(f"{len(chart_list)} charts found in Index file") + + chart_in_process = {"name" : ""} + chart_latest_version = "" + latest_charts = [] + + for index,chart in enumerate(chart_list): + chart_name = chart["name"] + #print(f'[INFO] look for latest chart : {chart_name} {chart["version"]}') + if chart_name == chart_in_process["name"]: + new_version = semantic_version.Version.coerce(chart["version"]) + #print(f' [INFO] compare chart versions : {new_version}({chart["version"]}) : {chart_latest_version}') + if new_version > chart_latest_version: + #print(f' [INFO] a new latest chart version : {new_version}') + chart_latest_version = new_version + chart_in_process = chart + else: + if chart_in_process["name"] != "": + #print(f' [INFO] chart completed : {chart_in_process["name"]} {chart_in_process["version"]}') + latest_charts.append(chart_in_process) + + #print(f'[INFO] new chart found : {chart_name} {chart["version"]}') + chart_in_process = chart + chart_version = chart["version"] + if chart_version.startswith("v"): + chart_version = chart_version[1:] + chart_latest_version = semantic_version.Version.coerce(chart_version) + else: + chart_in_process = chart + + if index+1 == len(chart_list): + #print(f' [INFO] last chart completed : {chart_in_process["name"]} {chart_in_process["version"]}') + latest_charts.append(chart_in_process) + + return latest_charts + + if __name__ == "__main__": - get_chart_info("redhat-dotnet-0.0.1") \ No newline at end of file + get_chart_info("redhat-dotnet-0.0.1") + + chart_list = get_latest_charts() + + for chart in chart_list: + print(f'[INFO] found latest chart : {chart["name"]} {chart["version"]}') + + + OCP_VERSION = semantic_version.Version.coerce("4.11") + + for chart in chart_list: + if "supportedOCP" in chart and chart["supportedOCP"] != "N/A" and chart["supportedOCP"] != "": + if OCP_VERSION in semantic_version.NpmSpec(chart["supportedOCP"]): + print(f'PASS: Chart supported OCP version {chart["supportedOCP"]} includes: {OCP_VERSION}') + else: + print(f' ERROR: Chart supported OCP version {chart["supportedOCP"]} does not include {OCP_VERSION}') + elif "kubeVersion" in chart and chart["kubeVersion"] != "": + supportedOCPVersion = indexannotations.getOCPVersions(chart["kubeVersion"]) + if OCP_VERSION in semantic_version.NpmSpec(supportedOCPVersion): + print(f'PASS: Chart kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) includes OCP version: {OCP_VERSION}') + else: + print(f' ERROR: Chart kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) does not include {OCP_VERSION}') + + + + + + + diff --git a/scripts/src/pullrequest/prartifact.py b/scripts/src/pullrequest/prartifact.py index a79174e650..f290a7c7b9 100644 --- a/scripts/src/pullrequest/prartifact.py +++ b/scripts/src/pullrequest/prartifact.py @@ -7,13 +7,15 @@ import requests +sys.path.append('../') +from checkprcontent import checkpr + # TODO(baijum): Move this code under chartsubmission.chart module def get_modified_charts(api_url): files_api_url = f'{api_url}/files' headers = {'Accept': 'application/vnd.github.v3+json'} r = requests.get(files_api_url, headers=headers) - pattern = re.compile(r"charts/(\w+)/([\w-]+)/([\w-]+)/([\w\.]+)/.*") - count = 0 + pattern,_ = checkpr.get_file_match_compiled_patterns() for f in r.json(): m = pattern.match(f["filename"]) if m: @@ -25,9 +27,11 @@ def get_modified_charts(api_url): def save_metadata(directory, vendor_label, chart, number): with open(os.path.join(directory, "vendor"), "w") as fd: + print(f"add {directory}/vendor as {vendor_label}") fd.write(vendor_label) with open(os.path.join(directory, "chart"), "w") as fd: + print(f"add {directory}/chart as {chart}") fd.write(chart) with open(os.path.join(directory, "NR"), "w") as fd: diff --git a/scripts/src/saforcharttesting/saforcharttesting.py b/scripts/src/saforcharttesting/saforcharttesting.py index dc80233219..5724995216 100644 --- a/scripts/src/saforcharttesting/saforcharttesting.py +++ b/scripts/src/saforcharttesting/saforcharttesting.py @@ -6,6 +6,7 @@ import argparse import subprocess import tempfile +import re from string import Template namespace_template = """\ @@ -211,7 +212,8 @@ def delete_clusterrolebinding(name): sys.exit(1) def write_sa_token(namespace, token): - sa_found = False + secret_found = False + secrets = [] for i in range(7): out = subprocess.run(["oc", "get", "serviceaccount", namespace, "-n", namespace, "-o", "json"], capture_output=True) stdout = out.stdout.decode("utf-8") @@ -223,15 +225,29 @@ def write_sa_token(namespace, token): else: sa = json.loads(stdout) if len(sa["secrets"]) >= 2: - sa_found = True + secrets = sa["secrets"] + secret_found = True break - time.sleep(10) + else: + pattern = r'Tokens:\s+([A-Za-z0-9-]+)' + dout = subprocess.run(["oc", "describe", "serviceaccount", namespace, "-n", namespace], capture_output=True) + dstdout = dout.stdout.decode("utf-8") + match = re.search(pattern, dstdout) + if match: + token_name = match.group(1) + else: + print("[ERROR] Token not found, Exiting") + sys.exit(1) + secrets.append({"name": token_name}) + secret_found = True + break + time.sleep(10) - if not sa_found: + if not secret_found: print("[ERROR] retrieving ServiceAccount:", namespace, stderr) sys.exit(1) - for secret in sa["secrets"]: + for secret in secrets: out = subprocess.run(["oc", "get", "secret", secret["name"], "-n", namespace, "-o", "json"], capture_output=True) stdout = out.stdout.decode("utf-8") if out.returncode != 0: diff --git a/tests/data/HC-16/dash-in-version/partner/report.yaml b/tests/data/HC-16/dash-in-version/partner/report.yaml new file mode 100644 index 0000000000..b4aec81305 --- /dev/null +++ b/tests/data/HC-16/dash-in-version/partner/report.yaml @@ -0,0 +1,89 @@ +apiversion: v1 +kind: verify-report +metadata: + tool: + verifier-version: 1.7.0 + profile: + VendorType: partner + version: v1.1 + chart-uri: https://github.com/openshift-helm-charts/development/blob/main/tests/data/psql-service-0.1.10-1.tgz?raw=true + digests: + chart: sha256:db482b4d90349c6b276ba27f581720cc62fa7bea05184b5bfd840844178a8da6 + package: c8635dcdc8f8493abbdef85305be55c9d5dbf3a44495a88a8285c9a0d0c63408 + lastCertifiedTimestamp: "2022-06-22T15:54:59.964823+00:00" + testedOpenShiftVersion: "4.10" + supportedOpenShiftVersions: '>=4.7' + providerControlledDelivery: false + chart: + name: psql-service + home: "" + sources: [] + version: 0.1.10-1 + description: A Helm chart for a RedHat Certified PSQL + keywords: [] + maintainers: [] + icon: "" + apiversion: v2 + condition: "" + tags: "" + appversion: 10.0.0 + deprecated: false + annotations: + charts.openshift.io/archs: x86_64 + charts.openshift.io/name: PSQL RedHat Demo Chart + charts.openshift.io/provider: RedHat + charts.openshift.io/supportURL: https://github.com/dperaza4dustbit/helm-chart + kubeversion: '>=1.20.0' + dependencies: [] + type: application + chart-overrides: "" +results: + - check: v1.0/contains-test + type: Mandatory + outcome: PASS + reason: Chart test files exist + - check: v1.0/contains-values + type: Mandatory + outcome: PASS + reason: Values file exist + - check: v1.1/has-kubeversion + type: Mandatory + outcome: PASS + reason: Kubernetes version specified + - check: v1.0/not-contains-crds + type: Mandatory + outcome: PASS + reason: Chart does not contain CRDs + - check: v1.0/not-contain-csi-objects + type: Mandatory + outcome: PASS + reason: CSI objects do not exist + - check: v1.0/chart-testing + type: Mandatory + outcome: PASS + reason: Chart tests have passed + - check: v1.0/has-readme + type: Mandatory + outcome: PASS + reason: Chart has a README + - check: v1.0/is-helm-v3 + type: Mandatory + outcome: PASS + reason: API version is V2, used in Helm 3 + - check: v1.0/contains-values-schema + type: Mandatory + outcome: PASS + reason: Values schema file exist + - check: v1.0/helm-lint + type: Mandatory + outcome: PASS + reason: Helm lint successful + - check: v1.0/images-are-certified + type: Mandatory + outcome: PASS + reason: 'Image is Red Hat certified : registry.access.redhat.com/rhscl/postgresql-10-rhel7:1-66' + - check: v1.0/required-annotations-present + type: Mandatory + outcome: PASS + reason: All required annotations present + diff --git a/tests/data/HC-16/dash-in-version/redhat/report.yaml b/tests/data/HC-16/dash-in-version/redhat/report.yaml new file mode 100644 index 0000000000..eff7b99361 --- /dev/null +++ b/tests/data/HC-16/dash-in-version/redhat/report.yaml @@ -0,0 +1,89 @@ +apiversion: v1 +kind: verify-report +metadata: + tool: + verifier-version: 1.7.0 + profile: + VendorType: redhat + version: v1.1 + chart-uri: https://github.com/openshift-helm-charts/development/blob/main/tests/data/psql-service-0.1.10-1.tgz?raw=true + digests: + chart: sha256:db482b4d90349c6b276ba27f581720cc62fa7bea05184b5bfd840844178a8da6 + package: c8635dcdc8f8493abbdef85305be55c9d5dbf3a44495a88a8285c9a0d0c63408 + lastCertifiedTimestamp: "2022-06-22T17:21:03.1478+00:00" + testedOpenShiftVersion: "4.10" + supportedOpenShiftVersions: '>=4.7' + providerControlledDelivery: false + chart: + name: psql-service + home: "" + sources: [] + version: 0.1.10-1 + description: A Helm chart for a RedHat Certified PSQL + keywords: [] + maintainers: [] + icon: "" + apiversion: v2 + condition: "" + tags: "" + appversion: 10.0.0 + deprecated: false + annotations: + charts.openshift.io/archs: x86_64 + charts.openshift.io/name: PSQL RedHat Demo Chart + charts.openshift.io/provider: RedHat + charts.openshift.io/supportURL: https://github.com/dperaza4dustbit/helm-chart + kubeversion: '>=1.20.0' + dependencies: [] + type: application + chart-overrides: "" +results: + - check: v1.0/has-readme + type: Mandatory + outcome: PASS + reason: Chart has a README + - check: v1.0/is-helm-v3 + type: Mandatory + outcome: PASS + reason: API version is V2, used in Helm 3 + - check: v1.0/not-contains-crds + type: Mandatory + outcome: PASS + reason: Chart does not contain CRDs + - check: v1.0/not-contain-csi-objects + type: Mandatory + outcome: PASS + reason: CSI objects do not exist + - check: v1.0/images-are-certified + type: Mandatory + outcome: PASS + reason: 'Image is Red Hat certified : registry.access.redhat.com/rhscl/postgresql-10-rhel7:1-66' + - check: v1.0/chart-testing + type: Mandatory + outcome: PASS + reason: Chart tests have passed + - check: v1.0/required-annotations-present + type: Mandatory + outcome: PASS + reason: All required annotations present + - check: v1.0/contains-test + type: Mandatory + outcome: PASS + reason: Chart test files exist + - check: v1.0/contains-values + type: Mandatory + outcome: PASS + reason: Values file exist + - check: v1.0/contains-values-schema + type: Mandatory + outcome: PASS + reason: Values schema file exist + - check: v1.1/has-kubeversion + type: Mandatory + outcome: PASS + reason: Kubernetes version specified + - check: v1.0/helm-lint + type: Mandatory + outcome: PASS + reason: Helm lint successful + diff --git a/tests/data/HC-17/dash-in-version/partner/report.yaml b/tests/data/HC-17/dash-in-version/partner/report.yaml new file mode 100644 index 0000000000..b4aec81305 --- /dev/null +++ b/tests/data/HC-17/dash-in-version/partner/report.yaml @@ -0,0 +1,89 @@ +apiversion: v1 +kind: verify-report +metadata: + tool: + verifier-version: 1.7.0 + profile: + VendorType: partner + version: v1.1 + chart-uri: https://github.com/openshift-helm-charts/development/blob/main/tests/data/psql-service-0.1.10-1.tgz?raw=true + digests: + chart: sha256:db482b4d90349c6b276ba27f581720cc62fa7bea05184b5bfd840844178a8da6 + package: c8635dcdc8f8493abbdef85305be55c9d5dbf3a44495a88a8285c9a0d0c63408 + lastCertifiedTimestamp: "2022-06-22T15:54:59.964823+00:00" + testedOpenShiftVersion: "4.10" + supportedOpenShiftVersions: '>=4.7' + providerControlledDelivery: false + chart: + name: psql-service + home: "" + sources: [] + version: 0.1.10-1 + description: A Helm chart for a RedHat Certified PSQL + keywords: [] + maintainers: [] + icon: "" + apiversion: v2 + condition: "" + tags: "" + appversion: 10.0.0 + deprecated: false + annotations: + charts.openshift.io/archs: x86_64 + charts.openshift.io/name: PSQL RedHat Demo Chart + charts.openshift.io/provider: RedHat + charts.openshift.io/supportURL: https://github.com/dperaza4dustbit/helm-chart + kubeversion: '>=1.20.0' + dependencies: [] + type: application + chart-overrides: "" +results: + - check: v1.0/contains-test + type: Mandatory + outcome: PASS + reason: Chart test files exist + - check: v1.0/contains-values + type: Mandatory + outcome: PASS + reason: Values file exist + - check: v1.1/has-kubeversion + type: Mandatory + outcome: PASS + reason: Kubernetes version specified + - check: v1.0/not-contains-crds + type: Mandatory + outcome: PASS + reason: Chart does not contain CRDs + - check: v1.0/not-contain-csi-objects + type: Mandatory + outcome: PASS + reason: CSI objects do not exist + - check: v1.0/chart-testing + type: Mandatory + outcome: PASS + reason: Chart tests have passed + - check: v1.0/has-readme + type: Mandatory + outcome: PASS + reason: Chart has a README + - check: v1.0/is-helm-v3 + type: Mandatory + outcome: PASS + reason: API version is V2, used in Helm 3 + - check: v1.0/contains-values-schema + type: Mandatory + outcome: PASS + reason: Values schema file exist + - check: v1.0/helm-lint + type: Mandatory + outcome: PASS + reason: Helm lint successful + - check: v1.0/images-are-certified + type: Mandatory + outcome: PASS + reason: 'Image is Red Hat certified : registry.access.redhat.com/rhscl/postgresql-10-rhel7:1-66' + - check: v1.0/required-annotations-present + type: Mandatory + outcome: PASS + reason: All required annotations present + diff --git a/tests/data/HC-17/dash-in-version/redhat/report.yaml b/tests/data/HC-17/dash-in-version/redhat/report.yaml new file mode 100644 index 0000000000..eff7b99361 --- /dev/null +++ b/tests/data/HC-17/dash-in-version/redhat/report.yaml @@ -0,0 +1,89 @@ +apiversion: v1 +kind: verify-report +metadata: + tool: + verifier-version: 1.7.0 + profile: + VendorType: redhat + version: v1.1 + chart-uri: https://github.com/openshift-helm-charts/development/blob/main/tests/data/psql-service-0.1.10-1.tgz?raw=true + digests: + chart: sha256:db482b4d90349c6b276ba27f581720cc62fa7bea05184b5bfd840844178a8da6 + package: c8635dcdc8f8493abbdef85305be55c9d5dbf3a44495a88a8285c9a0d0c63408 + lastCertifiedTimestamp: "2022-06-22T17:21:03.1478+00:00" + testedOpenShiftVersion: "4.10" + supportedOpenShiftVersions: '>=4.7' + providerControlledDelivery: false + chart: + name: psql-service + home: "" + sources: [] + version: 0.1.10-1 + description: A Helm chart for a RedHat Certified PSQL + keywords: [] + maintainers: [] + icon: "" + apiversion: v2 + condition: "" + tags: "" + appversion: 10.0.0 + deprecated: false + annotations: + charts.openshift.io/archs: x86_64 + charts.openshift.io/name: PSQL RedHat Demo Chart + charts.openshift.io/provider: RedHat + charts.openshift.io/supportURL: https://github.com/dperaza4dustbit/helm-chart + kubeversion: '>=1.20.0' + dependencies: [] + type: application + chart-overrides: "" +results: + - check: v1.0/has-readme + type: Mandatory + outcome: PASS + reason: Chart has a README + - check: v1.0/is-helm-v3 + type: Mandatory + outcome: PASS + reason: API version is V2, used in Helm 3 + - check: v1.0/not-contains-crds + type: Mandatory + outcome: PASS + reason: Chart does not contain CRDs + - check: v1.0/not-contain-csi-objects + type: Mandatory + outcome: PASS + reason: CSI objects do not exist + - check: v1.0/images-are-certified + type: Mandatory + outcome: PASS + reason: 'Image is Red Hat certified : registry.access.redhat.com/rhscl/postgresql-10-rhel7:1-66' + - check: v1.0/chart-testing + type: Mandatory + outcome: PASS + reason: Chart tests have passed + - check: v1.0/required-annotations-present + type: Mandatory + outcome: PASS + reason: All required annotations present + - check: v1.0/contains-test + type: Mandatory + outcome: PASS + reason: Chart test files exist + - check: v1.0/contains-values + type: Mandatory + outcome: PASS + reason: Values file exist + - check: v1.0/contains-values-schema + type: Mandatory + outcome: PASS + reason: Values schema file exist + - check: v1.1/has-kubeversion + type: Mandatory + outcome: PASS + reason: Kubernetes version specified + - check: v1.0/helm-lint + type: Mandatory + outcome: PASS + reason: Helm lint successful + diff --git a/tests/data/psql-service-0.1.10-1.tgz b/tests/data/psql-service-0.1.10-1.tgz new file mode 100644 index 0000000000..7e36739391 Binary files /dev/null and b/tests/data/psql-service-0.1.10-1.tgz differ diff --git a/tests/data/vault-test-timeout-0.17.0.tgz b/tests/data/vault-test-timeout-0.17.0.tgz new file mode 100644 index 0000000000..46472f2859 Binary files /dev/null and b/tests/data/vault-test-timeout-0.17.0.tgz differ diff --git a/tests/functional/behave_features/HC-01_chart_src_without_report.feature b/tests/functional/behave_features/HC-01_chart_src_without_report.feature new file mode 100644 index 0000000000..01c83bfffa --- /dev/null +++ b/tests/functional/behave_features/HC-01_chart_src_without_report.feature @@ -0,0 +1,33 @@ +Feature: Chart source submission without report + Partners, redhat and community users can publish their chart by submitting + error-free chart in source format without a report. + + Scenario Outline: [HC-01-001] A partner or redhat associate submits an error-free chart source + Given the vendor "" has a valid identity as "" + And an error-free chart source is used in "" + When the user sends a pull request with the chart + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + And a release is published with corresponding report and chart tarball + + @partners @smoke @full + Examples: + | vendor_type | vendor | chart_path | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | + + @redhat @full + Examples: + | vendor_type | vendor | chart_path | + | redhat | redhat | tests/data/vault-0.17.0.tgz | + + Scenario Outline: [HC-01-002] A community user submits an error-free chart source without report + Given the vendor "" has a valid identity as "" + And an error-free chart source is used in "" + When the user sends a pull request with the chart + Then the pull request is not merged + And user gets the "" in the pull request comment + + @community @full + Examples: + | vendor_type | vendor | chart_path | message | + | community | redhat | tests/data/vault-0.17.0.tgz | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/behave_features/HC-02_chart_tar_without_report.feature b/tests/functional/behave_features/HC-02_chart_tar_without_report.feature new file mode 100644 index 0000000000..4fd4a9a4c7 --- /dev/null +++ b/tests/functional/behave_features/HC-02_chart_tar_without_report.feature @@ -0,0 +1,33 @@ +Feature: Chart tarball submission without report + Partners, redhat and community users can publish their chart by submitting + error-free chart in tarball format without a report. + + Scenario Outline: [HC-02-001] A partner or redhat associate submits an error-free chart tarball + Given the vendor "" has a valid identity as "" + And an error-free chart tarball is used in "" + When the user sends a pull request with the chart + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + And a release is published with corresponding report and chart tarball + + @partners @full + Examples: + | vendor_type | vendor | chart_path | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | + + @redhat @smoke @full + Examples: + | vendor_type | vendor | chart_path | + | redhat | redhat | tests/data/vault-0.17.0.tgz | + + Scenario Outline: [HC-02-002] A community user submits an error-free chart tarball without report + Given the vendor "" has a valid identity as "" + And an error-free chart tarball is used in "" + When the user sends a pull request with the chart + Then the pull request is not merged + And user gets the "" in the pull request comment + + @community @full + Examples: + | vendor_type | vendor | chart_path | message | + | community | redhat | tests/data/vault-0.17.0.tgz | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/behave_features/HC-03_chart_verifier_comes_back_with_failures.feature b/tests/functional/behave_features/HC-03_chart_verifier_comes_back_with_failures.feature new file mode 100644 index 0000000000..d1afebec42 --- /dev/null +++ b/tests/functional/behave_features/HC-03_chart_verifier_comes_back_with_failures.feature @@ -0,0 +1,35 @@ +Feature: Chart verifier comes back with a failure + Partners, redhat or community user submit charts which does not contain README file + + Scenario Outline: [HC-03-001] A partner or community user submits a chart which does not contain a readme file + Given the vendor "" has a valid identity as "" + And chart source is used in "" + And README file is missing in the chart + When the user pushed the chart and created pull request + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @smoke @full + Examples: + | vendor_type | vendor | chart_path | message | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | Chart does not have a README | + + @community @full + Examples: + | vendor_type | vendor | chart_path | message | + | community | redhat | tests/data/vault-0.17.0.tgz | Community charts require maintainer review and approval | + + Scenario Outline: [HC-03-002] A redhat user submits a chart which does not contain a readme file + Given the vendor "" has a valid identity as "" + And chart source is used in "" + And README file is missing in the chart + When the user pushed the chart and created pull request + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart with correct providerType + And a release is published with corresponding report and chart tarball + + @redhat @full + Examples: + | vendor_type | vendor | chart_path | + | redhat | redhat | tests/data/vault-0.17.0.tgz | + diff --git a/tests/functional/behave_features/HC-04_invalid_url_in_the_report.feature b/tests/functional/behave_features/HC-04_invalid_url_in_the_report.feature new file mode 100644 index 0000000000..884d326fe7 --- /dev/null +++ b/tests/functional/behave_features/HC-04_invalid_url_in_the_report.feature @@ -0,0 +1,44 @@ +Feature: Report contains an invalid URL + Partners, redhat and community users submits only report with an invalid URL + + Scenario Outline: [HC-04-001] A user submits a report with an invalid url + Given the vendor "" has a valid identity as "" + And a "" is provided + And the report contains an "" + When the user sends a pull request with the report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @smoke @full + Examples: + | vendor_type | vendor | report_path | invalid_url | message | + | partners | hashicorp | tests/data/report.yaml | example.com/vault-0.13.0.tgz | Missing schema in URL | + + @redhat @smoke @full + Examples: + | vendor_type | vendor | report_path | invalid_url | message | + | redhat | redhat | tests/data/report.yaml | htts://example.com/vault-0.13.0.tgz | Invalid schema | + + @community @smoke @full + Examples: + | vendor_type | vendor | report_path | invalid_url | message | + | community | redhat | tests/data/report.yaml | https:example.comvault-0.13.0.tgz | Invalid URL | + + @partners @full + Examples: + | vendor_type | vendor | report_path | invalid_url | message | + | partners | hashicorp | tests/data/report.yaml | htts://example.com/vault-0.13.0.tgz | Invalid schema | + | partners | hashicorp | tests/data/report.yaml | https:example.comvault-0.13.0.tgz | Invalid URL | + + @redhat @full + Examples: + | vendor_type | vendor | report_path | invalid_url | message | + | redhat | redhat | tests/data/report.yaml | example.com/vault-0.13.0.tgz | Missing schema in URL | + | redhat | redhat | tests/data/report.yaml | https:example.comvault-0.13.0.tgz | Invalid URL | + + @community @full + Examples: + | vendor_type | vendor | report_path | invalid_url | message | + | community | redhat | tests/data/report.yaml | example.com/vault-0.13.0.tgz | Missing schema in URL | + | community | redhat | tests/data/report.yaml | htts://example.com/vault-0.13.0.tgz | Invalid schema | + diff --git a/tests/functional/behave_features/HC-05_pr_includes_a_file_which_is_not_chart_related.feature b/tests/functional/behave_features/HC-05_pr_includes_a_file_which_is_not_chart_related.feature new file mode 100644 index 0000000000..567da35184 --- /dev/null +++ b/tests/functional/behave_features/HC-05_pr_includes_a_file_which_is_not_chart_related.feature @@ -0,0 +1,25 @@ +Feature: PR includes a non chart related file + Partners, redhat or community user submit charts which includes a file which is not part of the chart + + Scenario Outline: [HC-05-001] A user submits a chart with non chart related file + Given the vendor "" has a valid identity as "" + And chart source is used in "" + And user adds a non chart related file + When the user sends a pull request with both chart and non related file + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @smoke @full + Examples: + | vendor_type | vendor | chart_path | message | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | PR includes one or more files not related to charts | + + @redhat @full + Examples: + | vendor_type | vendor | chart_path | message | + | redhat | redhat | tests/data/vault-0.17.0.tgz | PR includes one or more files not related to charts | + + @community @full + Examples: + | vendor_type | vendor | chart_path | message | + | community | redhat | tests/data/vault-0.17.0.tgz | PR includes one or more files not related to charts | diff --git a/tests/functional/behave_features/HC-06_provider_delivery_control.feature b/tests/functional/behave_features/HC-06_provider_delivery_control.feature new file mode 100644 index 0000000000..9f9a5dc9e0 --- /dev/null +++ b/tests/functional/behave_features/HC-06_provider_delivery_control.feature @@ -0,0 +1,51 @@ +Feature: Report only submission with provider control settings + Partners can prevent publication of their chart by submitting + error-free report that was generated by chart-verifier and with + prvider controlled delivery set in the report and the OWNERS file. + + @external-feedback + Scenario Outline: [HC-06-001] A partner associate submits an error-free report with provider controlled delivery + Given the vendor "" has a valid identity as "" + And provider delivery control is set to "" in the OWNERS file + And a "" is provided + And provider delivery control is set to "" in the report + When the user sends a pull request with the report + Then the user sees the pull request is merged + And the "" is updated with an entry for the submitted chart + + @partners @smoke @full + Examples: + | vendor_type | vendor | report_path | index_file | provider_control_owners | provider_control_report | + | partners | hashicorp | tests/data/report.yaml | unpublished-certified-charts.yaml | true | true | + + @external-feedback + Scenario Outline: [HC-06-002] A partner associate submits an error-free report and chart with provider controlled delivery + Given the vendor "" has a valid identity as "" + And provider delivery control is set to "" in the OWNERS file + And an error-free chart tarball used in "" and report in "" + And provider delivery control is set to "" in the report + When the user sends a pull request with the report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @full + Examples: + | vendor_type | vendor | chart_path | report_path | provider_control_owners | provider_control_report | message | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | true | true | OWNERS file and/or report indicate provider controlled delivery but pull request is not report only. | + + @external-feedback + Scenario Outline: [HC-06-003] A partner associate submits an error-free report with inconsistent provider controlled delivery setting + Given the vendor "" has a valid identity as "" + And provider delivery control is set to "" in the OWNERS file + And a "" is provided + And provider delivery controls is set to "" and a package digest is "" in the report + When the user sends a pull request with the report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @full + Examples: + | vendor_type | vendor | report_path | provider_control_owners | provider_control_report | package_digest_set | message | + | partners | hashicorp | tests/data/report.yaml | true | false | true | OWNERS file indicates provider controlled delivery but report does not. | + | partners | hashicorp | tests/data/report.yaml | false | true | true | Report indicates provider controlled delivery but OWNERS file does not. | + | partners | hashicorp | tests/data/report.yaml | true | true | false | Provider delivery control requires a package digest in the report. | diff --git a/tests/functional/behave_features/HC-07_report_and_chart_src.feature b/tests/functional/behave_features/HC-07_report_and_chart_src.feature new file mode 100644 index 0000000000..dd69ada41c --- /dev/null +++ b/tests/functional/behave_features/HC-07_report_and_chart_src.feature @@ -0,0 +1,33 @@ +Feature: Chart source submission with report + Partners, redhat and community users can publish their chart by submitting + error-free chart in source format with a report. + + Scenario Outline: [HC-07-001] A partner or redhat associate submits an error-free chart source with report + Given the vendor "" has a valid identity as "" + And an error-free chart source used in "" and report in "" + When the user sends a pull request with the chart and report + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + And a release is published with corresponding report and chart tarball + + @partners @smoke @full + Examples: + | vendor_type | vendor | chart_path | report_path | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | + + @redhat @full + Examples: + | vendor_type | vendor | chart_path | report_path | + | redhat | redhat | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | + + Scenario Outline: [HC-07-002] A community user submits an error-free chart source with report + Given the vendor "" has a valid identity as "" + And an error-free chart source used in "" and report in "" + When the user sends a pull request with the chart and report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @community @smoke @full + Examples: + | vendor_type | vendor | chart_path | report_path | message | + | community | redhat | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/behave_features/HC-08_report_and_chart_tar.feature b/tests/functional/behave_features/HC-08_report_and_chart_tar.feature new file mode 100644 index 0000000000..69e9211fa9 --- /dev/null +++ b/tests/functional/behave_features/HC-08_report_and_chart_tar.feature @@ -0,0 +1,33 @@ +Feature: Chart tarball submission with report + Partners, redhat and community users can publish their chart by submitting + error-free chart in tarball format with a report. + + Scenario Outline: [HC-08-001] A partner or redhat associate submits an error-free chart tarball with report + Given the vendor "" has a valid identity as "" + And an error-free chart tarball used in "" and report in "" + When the user sends a pull request with the chart and report + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + And a release is published with corresponding report and chart tarball + + @partners @smoke @full + Examples: + | vendor_type | vendor | chart_path | report_path | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | + + @redhat @full + Examples: + | vendor_type | vendor | chart_path | report_path | + | redhat | redhat | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | + + Scenario Outline: [HC-08-002] A community user submits an error-free chart tarball with report + Given the vendor "" has a valid identity as "" + And an error-free chart tarball used in "" and report in "" + When the user sends a pull request with the chart and report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @community @smoke @full + Examples: + | vendor_type | vendor | chart_path | report_path | message | + | community | redhat | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | Community charts require maintainer review and approval, a review will be conducted shortly | diff --git a/tests/functional/behave_features/HC-09_report_in_json_format.feature b/tests/functional/behave_features/HC-09_report_in_json_format.feature new file mode 100644 index 0000000000..2de4a15c14 --- /dev/null +++ b/tests/functional/behave_features/HC-09_report_in_json_format.feature @@ -0,0 +1,27 @@ +Feature: Report only submission in json format + If partners, redhat and community users try to publish chart by submitting report + in json format then will receive an error message + + Scenario Outline: [HC-09-001] An user submits a report in json format + Given the vendor "" has a valid identity as "" + And report is used in "" + When the user sends a pull request with the report + Then the pull request is not merged + #this step is failing currently https://issues.redhat.com/browse/HELM-396 + And user gets the "" in the pull request comment + + @partners + Examples: + | vendor_type | vendor | report_path | message | + | partners | hashicorp | tests/data/report.json | One of these must be modified: report, chart source, or tarball | + + @redhat + Examples: + | vendor_type | vendor | report_path | message | + | redhat | redhat | tests/data/report.json | One of these must be modified: report, chart source, or tarball | + + @community + Examples: + | vendor_type | vendor | report_path | message | + | community | redhat | tests/data/report.json | One of these must be modified: report, chart source, or tarball | + diff --git a/tests/functional/behave_features/HC-10_report_only_edited.feature b/tests/functional/behave_features/HC-10_report_only_edited.feature new file mode 100644 index 0000000000..a922d1c69f --- /dev/null +++ b/tests/functional/behave_features/HC-10_report_only_edited.feature @@ -0,0 +1,47 @@ +Feature: Edited report only submission + Partners, redhat and community users attempt to publish their chart by submitting + report that was edited after it was generated by chart-verifier. + + Scenario Outline: [HC-10-001] A partner or redhat associate submits an edited report + Given the vendor "" has a valid identity as "" + And a "" is provided + And the report includes "" and "" OpenshiftVersion values and chart "" value + When the user sends a pull request with the report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @smoke @full + Examples: + | vendor_type | vendor | report_path | tested | supported | kubeversion | message | + | partners | hashicorp | tests/data/report.yaml | 4.9 | 4.6-4.9 | >=1.20.0 | is not a valid semantic version | + | partners | hashicorp | tests/data/report.yaml | 4.8 | >=4.7 | >=1.21.0 | does not match supportedOpenShiftVersions | + + @partners @full + Examples: + | vendor_type | vendor | report_path | tested | supported | kubeversion | message | + | partners | hashicorp | tests/data/report.yaml | 4.0 | >=4.7 | >=1.20.0 | is not a supported OpenShift version | + | partners | hashicorp | tests/data/report.yaml | 4.6 | >=4.7 | >=1.20.0 | not within specified kube-versions | + + @redhat @smoke @full + Examples: + | vendor_type | vendor | report_path | tested | supported | kubeversion | message | + | redhat | redhat | tests/data/report.yaml | 4.0 | >=4.7 | >=1.20.0 | is not a supported OpenShift version | + + @redhat @full + Examples: + | vendor_type | vendor | report_path | tested | supported | kubeversion | message | + | redhat | redhat | tests/data/report.yaml | 4.9 | 4.6-4.9 | >=1.20.0 | is not a valid semantic version | + | redhat | redhat | tests/data/report.yaml | 4.6 | >=4.7 | >=1.20.0 | not within specified kube-versions | + | redhat | redhat | tests/data/report.yaml | 4.8 | >=4.7 | >=1.21.0 | does not match supportedOpenShiftVersions | + + @community @smoke @full + Examples: + | vendor_type | vendor | report_path | tested | supported | kubeversion | message | + | community | redhat | tests/data/report.yaml | 4.6 | >=4.7 | >=1.20.0 | not within specified kube-versions | + + @community @full + Examples: + | vendor_type | vendor | report_path | tested | supported | kubeversion | message | + | community | redhat | tests/data/report.yaml | 4.9 | 4.6-4.9 | >=1.20.0 | is not a valid semantic version | + | community | redhat | tests/data/report.yaml | 4.0 | >=4.7 | >=1.20.0 | is not a supported OpenShift version | + | community | redhat | tests/data/report.yaml | 4.8 | >=4.7 | >=1.21.0 | does not match supportedOpenShiftVersions | \ No newline at end of file diff --git a/tests/functional/behave_features/HC-11_report_with_missing_checks.feature b/tests/functional/behave_features/HC-11_report_with_missing_checks.feature new file mode 100644 index 0000000000..012a8aa6cc --- /dev/null +++ b/tests/functional/behave_features/HC-11_report_with_missing_checks.feature @@ -0,0 +1,27 @@ +Feature: Report does not include a check + Partners, redhat and community users submits only report which does not include full set of checks + + Scenario Outline: [HC-11-001] A user submits a report with missing checks + Given the vendor "" has a valid identity as "" + And a "" is provided + And the report has a "" missing + When the user sends a pull request with the report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @smoke @full + Examples: + | vendor_type | vendor | report_path | check | message | + | partners | hashicorp | tests/data/report.yaml | v1.0/helm-lint | Missing mandatory check : v1.0/helm-lint | + + @community @full + Examples: + | vendor_type | vendor | report_path | check | message | + | community | redhat | tests/data/report.yaml | v1.0/helm-lint | Missing mandatory check : v1.0/helm-lint | + + @partners @full + Examples: + | vendor_type | vendor | report_path | check | message | + | partners | hashicorp | tests/data/report.yaml | v1.0/not-contains-crds | Missing mandatory check : v1.0/not-contains-crds | + # Commented this scenario, since it is failing , raised bug : https://issues.redhat.com/browse/HELM-289 , we can uncomment again when the issue fixed + #| redhat | redhat | tests/data/report.yaml |v1.0/helm-lint | Missing mandatory check : v1.0/helm-lint | \ No newline at end of file diff --git a/tests/functional/behave_features/HC-12_report_without_chart.feature b/tests/functional/behave_features/HC-12_report_without_chart.feature new file mode 100644 index 0000000000..d74dec2743 --- /dev/null +++ b/tests/functional/behave_features/HC-12_report_without_chart.feature @@ -0,0 +1,32 @@ +Feature: Report only submission + Partners, redhat and community users can publish their chart by submitting + error-free report that was generated by chart-verifier. + + Scenario Outline: [HC-12-001] A partner or redhat associate submits an error-free report + Given the vendor "" has a valid identity as "" + And an error-free report is used in "" + When the user sends a pull request with the report + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + + @partners @smoke @full + Examples: + | vendor_type | vendor | report_path | + | partners | hashicorp | tests/data/report.yaml | + + @redhat @full + Examples: + | vendor_type | vendor | report_path | + | redhat | redhat | tests/data/report.yaml | + + Scenario Outline: [HC-12-002] A community user submits an error-free report + Given the vendor "" has a valid identity as "" + And an error-free report is used in "" + When the user sends a pull request with the report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @community @smoke @full + Examples: + | vendor_type | vendor | report_path | message | + | community | redhat | tests/data/report.yaml | Community charts require maintainer review and approval, a review will be conducted shortly | \ No newline at end of file diff --git a/tests/functional/behave_features/HC-13_sha_value_does_not_match.feature b/tests/functional/behave_features/HC-13_sha_value_does_not_match.feature new file mode 100644 index 0000000000..05d6e1d9de --- /dev/null +++ b/tests/functional/behave_features/HC-13_sha_value_does_not_match.feature @@ -0,0 +1,26 @@ +Feature: SHA value in the report does not match + Partners, redhat and community users submits chart tar with report + where chart sha does not match with sha value digests.chart in the report + + Scenario Outline: [HC-13-001] A user submits a chart tarball with report + Given the vendor "" has a valid identity as "" + And a chart tarball is used in "" and report in "" + And the report contains "" + When the user sends a pull request with the chart tar and report + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @full + Examples: + | vendor_type | vendor | error | message | chart_path | report_path | + | partners | hashicorp | sha_mismatch | Digest is not matching | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | + + @redhat @full + Examples: + | vendor_type | vendor | error | message | chart_path | report_path | + | redhat | redhat | sha_mismatch | Digest is not matching | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | + + @community @full + Examples: + | vendor_type | vendor | error | message | chart_path | report_path | + | community | redhat | sha_mismatch | Digest is not matching | tests/data/vault-0.17.0.tgz | tests/data/report.yaml | diff --git a/tests/functional/behave_features/HC-14_user_submits_chart_with_errors.feature b/tests/functional/behave_features/HC-14_user_submits_chart_with_errors.feature new file mode 100644 index 0000000000..96d07ade97 --- /dev/null +++ b/tests/functional/behave_features/HC-14_user_submits_chart_with_errors.feature @@ -0,0 +1,49 @@ +Feature: Chart submission with errors + Partners, redhat or community user submit charts which result in errors + + Scenario Outline: [HC-14-001] An unauthorized user submits a chart + Given the vendor "" has a valid identity as "" + And A "" wants to submit a chart in "" + And the user creates a branch to add a new chart version + When the user sends a pull request with the chart + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @smoke @full + Examples: + | vendor_type | vendor | chart_path | message | user | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | is not allowed to submit the chart on behalf of | unauthorized | + + @redhat @full + Examples: + | vendor_type | vendor | chart_path | message | user | + | redhat | redhat | tests/data/vault-0.17.0.tgz | is not allowed to submit the chart on behalf of | unauthorized | + + @community @full + Examples: + | vendor_type | vendor | chart_path | message | user | + | community | redhat | tests/data/vault-0.17.0.tgz | is not allowed to submit the chart on behalf of | unauthorized | + + Scenario Outline: [HC-14-002] An authorized user submits a chart with incorrect version + Given the vendor "" has a valid identity as "" + And An authorized user wants to submit a chart in "" + And Chart.yaml specifies a "" + And the user creates a branch to add a new chart version + When the user sends a pull request with the chart + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @smoke @full + Examples: + | vendor_type | vendor | chart_path | message | bad_version | + | partners | hashicorp | tests/data/vault-0.17.0.tgz | doesn't match the directory structure | 9.9.9 | + + @redhat @full + Examples: + | vendor_type | vendor | chart_path | message | bad_version | + | redhat | redhat | tests/data/vault-0.17.0.tgz | doesn't match the directory structure | 9.9.9 | + + @community @full + Examples: + | vendor_type | vendor | chart_path | message | bad_version | + | community | redhat | tests/data/vault-0.17.0.tgz | doesn't match the directory structure | 9.9.9 | diff --git a/tests/functional/behave_features/HC-15_check_submitted_charts.feature b/tests/functional/behave_features/HC-15_check_submitted_charts.feature new file mode 100644 index 0000000000..8fea2fe2f2 --- /dev/null +++ b/tests/functional/behave_features/HC-15_check_submitted_charts.feature @@ -0,0 +1,16 @@ +Feature: Check submitted charts + New Openshift or chart-verifier will trigger (automatically or manually) a recursive checking on + existing submitted charts under `charts/` directory with the specified Openshift and chart-verifier + version. + + Besides, during workflow development, engineers would like to check if the changes will break checks + on existing submitted charts. + + @version-change + Scenario: [HC-15-001] A new Openshift or chart-verifier version is specified either by a cron job or manually + Given there is a github workflow for testing existing charts + When workflow for testing existing charts is triggered + And a new Openshift or chart-verifier version is specified + And the vendor type is specified, e.g. partner, and/or redhat + Then submission tests are run for existing charts + And all results are reported back to the caller diff --git a/tests/functional/behave_features/HC-16_chart_test_takes_more_than_30mins.feature b/tests/functional/behave_features/HC-16_chart_test_takes_more_than_30mins.feature new file mode 100644 index 0000000000..1084f2dcd4 --- /dev/null +++ b/tests/functional/behave_features/HC-16_chart_test_takes_more_than_30mins.feature @@ -0,0 +1,33 @@ +Feature: Chart test takes longer time and exceeds default timeout + Partners, redhat or community user submit charts which result in errors + + Scenario Outline: [HC-16-001] A partner or community user submits chart that takes more than 30 mins + Given the vendor "" has a valid identity as "" + And an error-free chart tarball is used in "" + When the user sends a pull request with the chart + Then the pull request is not merged + And user gets the "" in the pull request comment + + @partners @full + Examples: + | vendor_type | vendor | chart_path | message | + | partners | hashicorp | tests/data/vault-test-timeout-0.17.0.tgz | Chart test failure: timed out waiting for the condition | + + @community @full + Examples: + | vendor_type | vendor | chart_path | message | + | community | redhat | tests/data/vault-test-timeout-0.17.0.tgz | Community charts require maintainer review and approval, a review will be conducted shortly | + + Scenario Outline: [HC-16-002] A redhat associate submits a chart that takes more than 30 mins + Given the vendor "" has a valid identity as "" + And an error-free chart tarball is used in "" + When the user sends a pull request with the chart + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + And a release is published with corresponding report and chart tarball + + @redhat @full + Examples: + | vendor_type | vendor | chart_path | + | redhat | redhat | tests/data/vault-test-timeout-0.17.0.tgz | + diff --git a/tests/functional/behave_features/HC-17_dash_in_version.feature b/tests/functional/behave_features/HC-17_dash_in_version.feature new file mode 100644 index 0000000000..7431fce613 --- /dev/null +++ b/tests/functional/behave_features/HC-17_dash_in_version.feature @@ -0,0 +1,21 @@ +Feature: Report only submission + Partners, redhat and community users can publish their chart by submitting + error-free report that was generated by chart-verifier. + + Scenario Outline: [HC-17-001] A partner or redhat associate submits report only with dash in chart version + Given the vendor "" has a valid identity as "" + And an error-free report is used in "" + When the user sends a pull request with the report + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + + @partners @full + Examples: + | vendor_type | vendor | report_path | + | partners | redhat | tests/data/HC-17/dash-in-version/partner/report.yaml | + + @redhat @full + Examples: + | vendor_type | vendor | report_path | + | redhat | redhat | tests/data/HC-17/dash-in-version/redhat/report.yaml | + diff --git a/tests/functional/behave_features/common/__init__.py b/tests/functional/behave_features/common/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/functional/behave_features/common/utils/__init__.py b/tests/functional/behave_features/common/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/functional/behave_features/common/utils/chart.py b/tests/functional/behave_features/common/utils/chart.py new file mode 100644 index 0000000000..45d5938a67 --- /dev/null +++ b/tests/functional/behave_features/common/utils/chart.py @@ -0,0 +1,142 @@ +# -*- coding: utf-8 -*- +"""Utility module for processing chart files.""" + +import os +import tarfile +import yaml +import shutil +import json + +def get_name_and_version_from_report(path): + """ + Parameters: + path (str): path to the report.yaml + + Returns: + str: chart name + str: chart version + """ + if path.endswith('yaml'): + with open(path, 'r') as fd: + try: + report = yaml.safe_load(fd) + except yaml.YAMLError as err: + raise AssertionError(f"error parsing '{path}': {err}") + elif path.endswith('json'): + with open(path, 'r') as fd: + try: + report = json.load(fd) + except Exception as err: + raise AssertionError(f"error parsing '{path}': {err}") + else: + raise AssertionError("Unknown report type") + chart = report['metadata']['chart'] + return chart['name'], chart['version'] + + +def get_name_and_version_from_chart_tar(path): + """ + Parameters: + path (str): path to the chart tar file + + Returns: + str: chart name + str: chart version + """ + tar = tarfile.open(path) + for member in tar.getmembers(): + if member.name.split('/')[-1] == 'Chart.yaml': + chart = tar.extractfile(member) + if chart is not None: + content = chart.read() + try: + chart_yaml = yaml.safe_load(content) + return chart_yaml['name'], chart_yaml['version'] + except yaml.YAMLError as err: + raise AssertionError(f"error parsing '{path}': {err}") + else: + raise AssertionError(f"Chart.yaml not in {path}") + + +def get_name_and_version_from_chart_src(path): + """ + Parameters: + path (str): path to the chart src directory + + Returns: + str: chart name + str: chart version + """ + chart_path = os.path.join(path, 'Chart.yaml') + with open(chart_path, 'r') as fd: + try: + chart_yaml = yaml.safe_load(fd) + except yaml.YAMLError as err: + raise AssertionError(f"error parsing '{path}': {err}") + return chart_yaml['name'], chart_yaml['version'] + +def extract_chart_tgz(src, dst, secrets, logger): + """Extracts the chart tgz file into the target location under 'charts/' for PR submission tests + + Parameters: + src (str): path to the test chart tgz + dst (str): path to the extract destination, e.g. 'charts/partners/hashicorp/vault/0.13.0' + """ + try: + logger.info(f"Remove existing local '{dst}/src'") + shutil.rmtree(f'{dst}/src') + except FileNotFoundError: + logger.info(f"'{dst}/src' does not exist") + finally: + with tarfile.open(src, 'r') as fd: + fd.extractall(dst) + os.rename(f'{dst}/{secrets.chart_name}', f'{dst}/src') + +def get_all_charts(charts_path: str, vendor_types: str) -> list: + # TODO: Support `community` as vendor_type. + """Gets charts with src or tgz under `charts/` given vendor_types and without report. + + Parameters: + charts_path (str): path to the `charts/` directory + vendor_types (str): vendor type to look for, any combination of `partner`, `redhat`, separated + by commas, or `all` to run both `partner` and `redhat`. + + Returns: + list: list of (vendor_type, vendor, chart_name, chart_version) tuples + """ + ret = [] + # Pre-process vendor types + vendor_types = vendor_types.replace('partner', 'partners') + vendor_types = [vt.strip() for vt in vendor_types.split(',')] + vendor_types = list( + {'partners', 'redhat', 'all'}.intersection(set(vendor_types))) + vendor_types = ['partners', + 'redhat'] if 'all' in vendor_types else vendor_types + + # Iterate through `charts/` to find chart submission with src or tgz + for vt in vendor_types: + charts_path_vt = f'{charts_path}/{vt}' + vendor_names = [name for name in os.listdir( + charts_path_vt) if os.path.isdir(f'{charts_path_vt}/{name}')] + for vn in vendor_names: + charts_path_vt_vn = f'{charts_path_vt}/{vn}' + chart_names = [name for name in os.listdir( + charts_path_vt_vn) if os.path.isdir(f'{charts_path_vt_vn}/{name}')] + for cn in chart_names: + charts_path_vt_vn_cn = f'{charts_path_vt_vn}/{cn}' + file_names = [name for name in os.listdir( + charts_path_vt_vn_cn)] + if 'OWNERS' not in file_names: + continue + chart_versions = [name for name in os.listdir( + charts_path_vt_vn_cn) if os.path.isdir(f'{charts_path_vt_vn_cn}/{name}')] + # Only interest in latest chart version + if len(chart_versions) == 0: + continue + cv = max(chart_versions) + charts_path_vt_vn_cn_cv = f'{charts_path_vt_vn_cn}/{cv}' + file_names = [name for name in os.listdir( + charts_path_vt_vn_cn_cv)] + if 'report.yaml' not in file_names and (f'{cn}-{cv}.tgz' in file_names or 'src' in file_names): + ret.append((vt, vn, cn, cv)) + return ret diff --git a/tests/functional/behave_features/common/utils/chart_certification.py b/tests/functional/behave_features/common/utils/chart_certification.py new file mode 100644 index 0000000000..6b023d64fa --- /dev/null +++ b/tests/functional/behave_features/common/utils/chart_certification.py @@ -0,0 +1,1036 @@ +# -*- coding: utf-8 -*- +"""Utility class for setting up and manipulating certification workflow tests.""" + +import os +import json +import pathlib +import shutil +import logging +import time +import uuid +from tempfile import TemporaryDirectory +from dataclasses import dataclass +from string import Template +from pathlib import Path + +import git +import yaml + +from common.utils.notifier import * +from common.utils.index import * +from common.utils.github import * +from common.utils.secret import * +from common.utils.set_directory import SetDirectory +from common.utils.setttings import * +from common.utils.chart import * + +@dataclass +class ChartCertificationE2ETest: + owners_file_content: str = """\ +chart: + name: ${chart_name} + shortDescription: Test chart for testing chart submission workflows. +publicPgpKey: null +providerDelivery: ${provider_delivery} +users: +- githubUsername: ${bot_name} +vendor: + label: ${vendor} + name: ${vendor} +""" + secrets: E2ETestSecret = E2ETestSecret() + + old_cwd: str = os.getcwd() + repo: git.Repo = git.Repo() + temp_dir: TemporaryDirectory = None + temp_repo: git.Repo = None + github_actions: str = os.environ.get("GITHUB_ACTIONS") + def set_git_username_email(self, repo, username, email): + """ + Parameters: + repo (git.Repo): git.Repo instance of the local directory + username (str): git username to set + email (str): git email to set + """ + repo.config_writer().set_value("user", "name", username).release() + repo.config_writer().set_value("user", "email", email).release() + + def get_bot_name_and_token(self): + bot_name = os.environ.get("BOT_NAME") + logging.debug(f"Enviroment variable value BOT_NAME: {bot_name}") + bot_token = os.environ.get("BOT_TOKEN") + if not bot_name and not bot_token: + bot_name = "github-actions[bot]" + bot_token = os.environ.get("GITHUB_TOKEN") + if not bot_token: + raise Exception("BOT_TOKEN environment variable not defined") + elif not bot_name: + raise Exception("BOT_TOKEN set but BOT_NAME not specified") + elif not bot_token: + raise Exception("BOT_NAME set but BOT_TOKEN not specified") + return bot_name, bot_token + + def remove_chart(self, chart_directory, chart_version, remote_repo, base_branch, bot_token): + # Remove chart files from base branch + logging.info( + f"Remove {chart_directory}/{chart_version} from {remote_repo}:{base_branch}") + try: + self.temp_repo.git.rm('-rf', '--cached', f'{chart_directory}/{chart_version}') + self.temp_repo.git.commit( + '-m', f'Remove {chart_directory}/{chart_version}') + self.temp_repo.git.push(f'https://x-access-token:{bot_token}@github.com/{remote_repo}', + f'HEAD:refs/heads/{base_branch}') + except git.exc.GitCommandError: + logging.info( + f"{chart_directory}/{chart_version} not exist on {remote_repo}:{base_branch}") + + def remove_owners_file(self, chart_directory, remote_repo, base_branch, bot_token): + # Remove the OWNERS file from base branch + logging.info( + f"Remove {chart_directory}/OWNERS from {remote_repo}:{base_branch}") + try: + self.temp_repo.git.rm('-rf', '--cached', f'{chart_directory}/OWNERS') + self.temp_repo.git.commit( + '-m', f'Remove {chart_directory}/OWNERS') + self.temp_repo.git.push(f'https://x-access-token:{bot_token}@github.com/{remote_repo}', + f'HEAD:refs/heads/{base_branch}') + except git.exc.GitCommandError: + logging.info( + f"{chart_directory}/OWNERS not exist on {remote_repo}:{base_branch}") + + def create_test_gh_pages_branch(self, remote_repo, base_branch, bot_token): + # Get SHA from 'dev-gh-pages' branch + logging.info( + f"Create '{remote_repo}:{base_branch}-gh-pages' from '{remote_repo}:dev-gh-pages'") + r = github_api( + 'get', f'repos/{remote_repo}/git/ref/heads/dev-gh-pages', bot_token) + j = json.loads(r.text) + sha = j['object']['sha'] + + # Create a new gh-pages branch for testing + data = {'ref': f'refs/heads/{base_branch}-gh-pages', 'sha': sha} + r = github_api( + 'post', f'repos/{remote_repo}/git/refs', bot_token, json=data) + + logging.info(f'gh-pages branch created: {base_branch}-gh-pages') + + def setup_git_context(self, repo: git.Repo): + self.set_git_username_email(repo, self.secrets.bot_name, GITHUB_ACTIONS_BOT_EMAIL) + if os.environ.get('WORKFLOW_DEVELOPMENT'): + logging.info("Wokflow development enabled") + repo.git.add(A=True) + repo.git.commit('-m', 'Checkpoint') + + def send_pull_request(self, remote_repo, base_branch, pr_branch, bot_token): + pr_body = os.environ.get('PR_BODY') + data = {'head': pr_branch, 'base': base_branch, + 'title': base_branch, 'body': pr_body} + logging.debug(f"PR_BODY Content: {pr_body}") + logging.info( + f"Create PR from '{remote_repo}:{pr_branch}'") + r = github_api( + 'post', f'repos/{remote_repo}/pulls', bot_token, json=data) + j = json.loads(r.text) + if not 'number' in j: + raise AssertionError(f"error sending pull request, response was: {r.text}") + return j['number'] + + def create_and_push_owners_file(self, chart_directory, base_branch, vendor_name, vendor_type, chart_name, provider_delivery=False): + with SetDirectory(Path(self.temp_dir.name)): + # Create the OWNERS file from the string template + values = {'bot_name': self.secrets.bot_name, + 'vendor': vendor_name, 'chart_name': chart_name, + "provider_delivery" : provider_delivery} + content = Template(self.secrets.owners_file_content).substitute(values) + logging.debug(f"OWNERS File Content: {content}") + with open(f'{chart_directory}/OWNERS', 'w') as fd: + fd.write(content) + + # Push OWNERS file to the test_repo + logging.info( + f"Push OWNERS file to '{self.secrets.test_repo}:{base_branch}'") + self.temp_repo.git.add(f'{chart_directory}/OWNERS') + self.temp_repo.git.commit( + '-m', f"Add {vendor_type} {vendor_name} {chart_name} OWNERS file") + self.temp_repo.git.push(f'https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}', + f'HEAD:refs/heads/{base_branch}', '-f') + + def check_index_yaml(self,base_branch, vendor, chart_name, chart_version, index_file="index.yaml", check_provider_type=False, failure_type='error'): + old_branch = self.repo.active_branch.name + self.repo.git.fetch(f'https://github.com/{self.secrets.test_repo}.git', + '{0}:{0}'.format(f'{base_branch}-gh-pages'), '-f') + + self.repo.git.checkout(f'{base_branch}-gh-pages') + + with open(index_file, 'r') as fd: + try: + index = yaml.safe_load(fd) + except yaml.YAMLError as err: + if failure_type == 'error': + raise AssertionError(f"error parsing index.yaml: {err}") + else: + logging.warning(f"error parsing index.yaml: {err}") + return False + + if index: + entry = f"{vendor}-{chart_name}" + if "entries" not in index or entry not in index['entries']: + if failure_type == 'error': + raise AssertionError(f"{entry} not added in entries to {index_file} & Found index.yaml entries: {index['entries']}") + else: + logging.warning(f"{chart_version} not added to {index_file}") + logging.warning(f"Index.yaml entry content: {index['entries'][entry]}") + return False + + version_list = [release['version'] for release in index['entries'][entry]] + if chart_version not in version_list: + raise AssertionError(f"{chart_version} not added to {index_file} & Found index.yaml entry content: {index['entries'][entry]}") + + #This check is applicable for charts submitted in redhat path when one of the chart-verifier check fails + #Check whether providerType annotations is community in index.yaml when vendor_type is redhat + if check_provider_type and self.secrets.vendor_type == 'redhat': + provider_type_in_index_yaml = index['entries'][entry][0]['annotations']['charts.openshift.io/providerType'] + if provider_type_in_index_yaml != 'community': + if failure_type == 'error': + raise AssertionError(f"{provider_type_in_index_yaml} is not correct as providerType in index.yaml") + else: + logging.warning(f"{provider_type_in_index_yaml} is not correct as providerType in index.yaml") + + logging.info("Index updated correctly, cleaning up local branch") + self.repo.git.checkout(old_branch) + self.repo.git.branch('-D', f'{base_branch}-gh-pages') + return True + else: + return False + + def check_release_result(self, vendor, chart_name, chart_version, chart_tgz, failure_type='error'): + expected_tag = f'{vendor}-{chart_name}-{chart_version}' + try: + release = get_release_by_tag(self.secrets, expected_tag) + logging.info(f"Released '{expected_tag}' successfully") + + expected_chart_asset = f'{vendor}-{chart_tgz}' + required_assets = [expected_chart_asset] + logging.info(f"Check '{required_assets}' is in release assets") + release_id = release['id'] + get_release_assets(self.secrets, release_id, required_assets) + return True + except Exception as e: + if failure_type == 'error': + raise AssertionError(e) + else: + logging.warning(e) + return False + finally: + logging.info(f"Delete release '{expected_tag}'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/releases/{release_id}', self.secrets.bot_token) + + logging.info(f"Delete release tag '{expected_tag}'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/tags/{expected_tag}', self.secrets.bot_token) + + # expect_result: a string representation of expected result, e.g. 'success' + def check_workflow_conclusion(self, pr_number, expect_result: str, failure_type='error'): + try: + # Check workflow conclusion + run_id = get_run_id(self.secrets, pr_number) + conclusion = get_run_result(self.secrets, run_id) + if conclusion == expect_result: + logging.info(f"PR{pr_number} Workflow run was '{expect_result}' which is expected") + else: + if failure_type == 'warning': + logging.warning(f"PR{pr_number if pr_number else self.secrets.pr_number} Workflow run was '{conclusion}' which is unexpected, run id: {run_id}") + else: + raise AssertionError( + f"PR{pr_number if pr_number else self.secrets.pr_number} Workflow run was '{conclusion}' which is unexpected, run id: {run_id}") + + return run_id, conclusion + except Exception as e: + if failure_type == 'error': + raise AssertionError(e) + else: + logging.warning(e) + return None, None + + # expect_merged: boolean representing whether the PR should be merged + def check_pull_request_result(self, pr_number, expect_merged: bool, failure_type='error'): + # Check if PR merged + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/pulls/{pr_number}/merge', self.secrets.bot_token) + logging.info(f"PR{pr_number} result status_code : {r.status_code}") + if r.status_code == 204 and expect_merged: + logging.info(f"PR{pr_number} merged sucessfully as expected") + return True + elif r.status_code == 404 and not expect_merged: + logging.info(f"PR{pr_number} not merged, which is expected") + return True + elif r.status_code == 204 and not expect_merged: + if failure_type == 'error': + raise AssertionError(f"PR{pr_number} Expecting not merged but PR was merged") + else: + logging.warning(f"PR{pr_number} Expecting not merged but PR was merged") + return False + elif r.status_code == 404 and expect_merged: + if failure_type == 'error': + raise AssertionError(f"PR{pr_number} Expecting PR merged but PR was not merged") + else: + logging.warning(f"PR{pr_number} Expecting PR merged but PR was not merged") + return False + else: + if failure_type == 'error': + raise AssertionError(f"PR{pr_number} Got unexpected status code from PR: {r.status_code}") + else: + logging.warning(f"PR{pr_number} Got unexpected status code from PR: {r.status_code}") + return False + + def cleanup_release(self, expected_tag): + """Cleanup the release and release tag. + + Releases might be left behind if check_index_yam() ran before check_release_result() and fails the test. + """ + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/releases', self.secrets.bot_token) + releases = json.loads(r.text) + logging.debug(f"List of releases: {releases}") + for release in releases: + if release['tag_name'] == expected_tag: + release_id = release['id'] + logging.info(f"Delete release '{expected_tag}'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/releases/{release_id}', self.secrets.bot_token) + + logging.info(f"Delete release tag '{expected_tag}'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/tags/{expected_tag}', self.secrets.bot_token) + + def check_pull_request_labels(self, pr_number): + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/issues/{pr_number}/labels', self.secrets.bot_token) + labels = json.loads(r.text) + authorized_request = False + content_ok = False + for label in labels: + logging.info(f"PR{pr_number} found label {label['name']}") + if label['name'] == "authorized-request": + authorized_request = True + if label['name'] == "content-ok": + content_ok = True + + if authorized_request and content_ok: + logging.info(f"PR{pr_number} authorized request and content-ok labels were found as expected") + return True + else: + raise AssertionError(f"PR{pr_number} authorized request and/or content-ok labels were not found as expected") + +@dataclass +class ChartCertificationE2ETestSingle(ChartCertificationE2ETest): + test_name: str = '' # Meaningful test name for this test, displayed in PR title + test_chart: str = '' + test_report: str = '' + chart_directory: str = '' + secrets: E2ETestSecretOneShot = E2ETestSecretOneShot() + + def __post_init__(self) -> None: + # unique string based on uuid.uuid4(), not using timestamp here because even + # in nanoseconds there are chances of collisions among first test cases of + # different processes. + self.uuid = uuid.uuid4().hex + + bot_name, bot_token = self.get_bot_name_and_token() + test_repo = TEST_REPO + + #Storing current branch to checkout after scenario execution + if os.environ.get('LOCAL_RUN'): + self.secrets.active_branch = self.repo.active_branch.name + logging.debug(f"Active branch name : {self.secrets.active_branch}") + + # Create a new branch locally from detached HEAD + head_sha = self.repo.git.rev_parse('--short', 'HEAD') + unique_branch = f'{head_sha}-{self.uuid}' + logging.debug(f"Unique branch name : {unique_branch}") + local_branches = [h.name for h in self.repo.heads] + logging.debug(f"Local branch names : {local_branches}") + if unique_branch not in local_branches: + self.repo.git.checkout('-b', f'{unique_branch}') + + current_branch = self.repo.active_branch.name + logging.debug(f"Current active branch name : {current_branch}") + + r = github_api( + 'get', f'repos/{test_repo}/branches', bot_token) + branches = json.loads(r.text) + branch_names = [branch['name'] for branch in branches] + logging.debug(f"Remote test repo branch names : {branch_names}") + if current_branch not in branch_names: + logging.info( + f"{test_repo}:{current_branch} does not exists, creating with local branch") + self.repo.git.push(f'https://x-access-token:{bot_token}@github.com/{test_repo}', + f'HEAD:refs/heads/{current_branch}', '-f') + + pretty_test_name = self.test_name.strip().lower().replace(' ', '-') + base_branch = f'{self.uuid}-{pretty_test_name}-{current_branch}' if pretty_test_name else f'{self.uuid}-test-{current_branch}' + logging.debug(f"Base branch name : {base_branch}") + pr_branch = base_branch + '-pr-branch' + + self.secrets.owners_file_content = self.owners_file_content + self.secrets.test_repo = test_repo + self.secrets.bot_name = bot_name + self.secrets.bot_token = bot_token + self.secrets.base_branch = base_branch + self.secrets.pr_branch = pr_branch + self.secrets.index_file = "index.yaml" + self.secrets.provider_delivery = False + + + def cleanup (self): + # Cleanup releases and release tags + self.cleanup_release() + # Teardown step to cleanup branches + if self.temp_dir is not None: + self.temp_dir.cleanup() + self.repo.git.worktree('prune') + + head_sha = self.repo.git.rev_parse('--short', 'HEAD') + current_branch = f'{head_sha}-{self.uuid}' + logging.info(f"Delete remote '{current_branch}' branch") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/heads/{current_branch}', self.secrets.bot_token) + + logging.info(f"Delete '{self.secrets.test_repo}:{self.secrets.base_branch}'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/heads/{self.secrets.base_branch}', self.secrets.bot_token) + + logging.info(f"Delete '{self.secrets.test_repo}:{self.secrets.base_branch}-gh-pages'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/heads/{self.secrets.base_branch}-gh-pages', self.secrets.bot_token) + + logging.info(f"Delete '{self.secrets.test_repo}:{self.secrets.pr_branch}'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/heads/{self.secrets.pr_branch}', self.secrets.bot_token) + + logging.info(f"Delete local '{self.secrets.base_branch}'") + try: + self.repo.git.branch('-D', self.secrets.base_branch) + except git.exc.GitCommandError: + logging.info(f"Local '{self.secrets.base_branch}' does not exist") + + logging.info(f"Delete local '{current_branch}'") + try: + if os.environ.get('LOCAL_RUN'): + self.repo.git.checkout(f'{self.secrets.active_branch}') + self.repo.git.branch('-D', current_branch) + except git.exc.GitCommandError: + logging.info(f"Local '{current_branch}' does not exist") + + def update_bot_name(self, bot_name): + logging.debug(f"Updating bot name: {bot_name}") + self.secrets.bot_name = bot_name + + def update_bad_version(self, bad_version): + logging.debug(f"Updating bad version: {bad_version}") + self.secrets.bad_version = bad_version + + def update_chart_directory(self): + base_branch_without_uuid = "-".join(self.secrets.base_branch.split("-")[:-1]) + vendor_without_suffix = self.secrets.vendor.split("-")[0] + self.secrets.base_branch = f'{base_branch_without_uuid}-{self.secrets.vendor_type}-{vendor_without_suffix}-{self.secrets.chart_name}-{self.secrets.chart_version}' + self.secrets.pr_branch = f'{self.secrets.base_branch}-pr-branch' + self.chart_directory = f'charts/{self.secrets.vendor_type}/{self.secrets.vendor}/{self.secrets.chart_name}' + logging.debug(f"Updating chart_directory: {self.chart_directory}") + + def update_test_chart(self, test_chart): + logging.debug(f"Updating test chart: {test_chart}") + self.test_chart = test_chart + chart_name, chart_version = self.get_chart_name_version() + logging.debug(f"Got chart_name: {chart_name} and chart_version: {chart_version} from the chart") + self.secrets.test_chart = self.test_chart + self.secrets.chart_name = chart_name + self.secrets.chart_version = chart_version + self.update_chart_directory() + + def update_test_report(self, test_report): + logging.debug(f"Updating test report: {test_report}") + self.test_report = test_report + chart_name, chart_version = self.get_chart_name_version() + logging.debug(f"Got chart_name: {chart_name} and chart_version: {chart_version} from the report") + self.secrets.test_report = self.test_report + self.secrets.chart_name = chart_name + self.secrets.chart_version = chart_version + self.update_chart_directory() + + def get_unique_vendor(self, vendor): + """Set unique vendor name. + Note that release tag is generated with this vendor name. + """ + # unique string based on uuid.uuid4() + suffix = self.uuid + if "PR_NUMBER" in os.environ: + pr_num = os.environ["PR_NUMBER"] + suffix = f"{suffix}-{pr_num}" + return f"{vendor}-{suffix}" + + def get_chart_name_version(self): + if not self.test_report and not self.test_chart: + raise AssertionError("Provide at least one of test report or test chart.") + if self.test_report: + chart_name, chart_version = get_name_and_version_from_report(self.test_report) + else: + chart_name, chart_version = get_name_and_version_from_chart_tar(self.test_chart) + return chart_name, chart_version + + def set_vendor(self, vendor, vendor_type): + # use unique vendor id to avoid collision between tests + logging.debug(f"Setting vendor: {vendor} vendor_type: {vendor_type}") + self.secrets.vendor = self.get_unique_vendor(vendor) + logging.debug(f"Unique vendor value: {self.secrets.vendor}") + self.secrets.vendor_type = vendor_type + + def setup_git_context(self): + super().setup_git_context(self.repo) + + def setup_gh_pages_branch(self): + self.create_test_gh_pages_branch(self.secrets.test_repo, self.secrets.base_branch, self.secrets.bot_token) + + def setup_temp_dir(self): + self.temp_dir = TemporaryDirectory(prefix='tci-') + with SetDirectory(Path(self.temp_dir.name)): + # Make PR's from a temporary directory + logging.info(f'Worktree directory: {self.temp_dir.name}') + self.repo.git.worktree('add', '--detach', self.temp_dir.name, f'HEAD') + self.temp_repo = git.Repo(self.temp_dir.name) + + self.set_git_username_email(self.temp_repo, self.secrets.bot_name, GITHUB_ACTIONS_BOT_EMAIL) + self.temp_repo.git.checkout('-b', self.secrets.base_branch) + pathlib.Path( + f'{self.chart_directory}/{self.secrets.chart_version}').mkdir(parents=True, exist_ok=True) + + self.remove_chart(self.chart_directory, self.secrets.chart_version, self.secrets.test_repo, self.secrets.base_branch, self.secrets.bot_token) + self.remove_owners_file(self.chart_directory, self.secrets.test_repo, self.secrets.base_branch, self.secrets.bot_token) + + def update_chart_version_in_chart_yaml(self, new_version): + with SetDirectory(Path(self.temp_dir.name)): + path = f'{self.chart_directory}/{self.secrets.chart_version}/src/Chart.yaml' + with open(path, 'r') as fd: + try: + chart = yaml.safe_load(fd) + except yaml.YAMLError as err: + raise AssertionError(f"error parsing '{path}': {err}") + current_version = chart['version'] + + if current_version != new_version: + chart['version'] = new_version + try: + with open(path, 'w') as fd: + fd.write(yaml.dump(chart)) + except Exception as e: + raise AssertionError("Failed to update version in yaml file") + + def remove_readme_file(self): + with SetDirectory(Path(self.temp_dir.name)): + path = f'{self.chart_directory}/{self.secrets.chart_version}/src/README.md' + try: + os.remove(path) + except Exception as e: + raise AssertionError(f"Failed to remove readme file : {e}") + + def process_owners_file(self): + super().create_and_push_owners_file(self.chart_directory, self.secrets.base_branch, self.secrets.vendor, self.secrets.vendor_type, self.secrets.chart_name,self.secrets.provider_delivery) + + def process_chart(self, is_tarball: bool): + with SetDirectory(Path(self.temp_dir.name)): + if is_tarball: + # Copy the chart tar into temporary directory for PR submission + chart_tar = self.secrets.test_chart.split('/')[-1] + shutil.copyfile(f'{self.old_cwd}/{self.secrets.test_chart}', + f'{self.chart_directory}/{self.secrets.chart_version}/{chart_tar}') + else: + # Unzip files into temporary directory for PR submission + extract_chart_tgz(self.secrets.test_chart, f'{self.chart_directory}/{self.secrets.chart_version}', self.secrets, logging) + + + def process_report(self, update_chart_sha=False, update_url=False, url=None, + update_versions=False,supported_versions=None,tested_version=None,kube_version=None, + update_provider_delivery=False, provider_delivery=False, missing_check=None,unset_package_digest=False): + + with SetDirectory(Path(self.temp_dir.name)): + # Copy report to temporary location and push to test_repo:pr_branch + logging.info( + f"Push report to '{self.secrets.test_repo}:{self.secrets.pr_branch}'") + + if self.secrets.test_report.endswith('json'): + logging.debug("Report type is json") + report_path = f'{self.chart_directory}/{self.secrets.chart_version}/' + self.secrets.test_report.split('/')[-1] + with open(self.secrets.test_report, 'r') as fd: + try: + report = json.load(fd) + except Exception as e: + raise AssertionError("Failed to read json file") + + with open(report_path, 'w') as fd: + try: + fd.write(json.dumps(report, indent=4)) + except Exception as e: + raise AssertionError("Failed to write report in json format") + + elif self.secrets.test_report.endswith('yaml'): + logging.debug("Report type is yaml") + tmpl = open(self.secrets.test_report).read() + values = {'repository': self.secrets.test_repo, + 'branch': self.secrets.base_branch} + content = Template(tmpl).substitute(values) + + report_path = f'{self.chart_directory}/{self.secrets.chart_version}/' + self.secrets.test_report.split('/')[-1] + + try: + report = yaml.safe_load(content) + except yaml.YAMLError as err: + raise AssertionError(f"error parsing '{report_path}': {err}") + + if self.secrets.vendor_type != "partners": + report["metadata"]["tool"]["profile"]["VendorType"] = self.secrets.vendor_type + logging.info(f'VendorType set to {report["metadata"]["tool"]["profile"]["VendorType"]} in report.yaml') + + if update_chart_sha or update_url or update_versions or update_provider_delivery or unset_package_digest: + #For updating the report.yaml, for chart sha mismatch scenario + if update_chart_sha: + new_sha_value = 'sha256:5b85ae00b9ca2e61b2d70a59f98fd72136453b1a185676b29d4eb862981c1xyz' + logging.info(f"Current SHA Value in report: {report['metadata']['tool']['digests']['chart']}") + report['metadata']['tool']['digests']['chart'] = new_sha_value + logging.info(f"Updated sha value in report: {new_sha_value}") + + #For updating the report.yaml, for invalid_url sceanrio + if update_url: + logging.info(f"Current chart-uri in report: {report['metadata']['tool']['chart-uri']}") + report['metadata']['tool']['chart-uri'] = url + logging.info(f"Updated chart-uri value in report: {url}") + + if update_versions: + report['metadata']['tool']['testedOpenShiftVersion'] = tested_version + report['metadata']['tool']['supportedOpenShiftVersions'] = supported_versions + report['metadata']['chart']['kubeversion'] = kube_version + logging.info(f"Updated testedOpenShiftVersion value in report: {tested_version}") + logging.info(f"Updated supportedOpenShiftVersions value in report: {supported_versions}") + logging.info(f"Updated kubeversion value in report: {kube_version}") + + if update_provider_delivery: + report['metadata']['tool']['providerControlledDelivery'] = provider_delivery + + if unset_package_digest: + del report['metadata']['tool']['digests']['package'] + + with open(report_path, 'w') as fd: + try: + fd.write(yaml.dump(report)) + logging.info("Report updated with new values") + except Exception as e: + raise AssertionError("Failed to update report yaml with new values") + + #For removing the check for missing check scenario + if missing_check: + logging.info(f"Updating report with {missing_check}") + with open(report_path, 'r+') as fd: + report_content = yaml.safe_load(fd) + results = report_content["results"] + new_results = filter(lambda x: x['check'] != missing_check, results) + report_content["results"] = list(new_results) + fd.seek(0) + yaml.dump(report_content, fd) + fd.truncate() + else: + raise AssertionError("Unknown report type") + + self.temp_repo.git.add(report_path) + self.temp_repo.git.commit( + '-m', f"Add {self.secrets.vendor} {self.secrets.chart_name} {self.secrets.chart_version} report") + self.temp_repo.git.push(f'https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}', + f'HEAD:refs/heads/{self.secrets.pr_branch}', '-f') + + def add_non_chart_related_file(self): + with SetDirectory(Path(self.temp_dir.name)): + path = f'{self.chart_directory}/Notes.txt' + with open(path, 'w') as fd: + fd.write("This is a test file") + + def push_chart(self, is_tarball: bool, add_non_chart_file=False): + # Push chart to test_repo:pr_branch + if is_tarball: + chart_tar = self.secrets.test_chart.split('/')[-1] + self.temp_repo.git.add(f'{self.chart_directory}/{self.secrets.chart_version}/{chart_tar}') + else: + if add_non_chart_file: + self.temp_repo.git.add(f'{self.chart_directory}/') + else: + self.temp_repo.git.add(f'{self.chart_directory}/{self.secrets.chart_version}/src') + self.temp_repo.git.commit( + '-m', f"Add {self.secrets.vendor} {self.secrets.chart_name} {self.secrets.chart_version} chart") + + self.temp_repo.git.push(f'https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}', + f'HEAD:refs/heads/{self.secrets.pr_branch}', '-f') + + def send_pull_request(self): + self.secrets.pr_number = super().send_pull_request(self.secrets.test_repo, self.secrets.base_branch, self.secrets.pr_branch, self.secrets.bot_token) + logging.info(f"[INFO] PR number: {self.secrets.pr_number}") + + # expect_result: a string representation of expected result, e.g. 'success' + def check_workflow_conclusion(self, expect_result: str): + # Check workflow conclusion + super().check_workflow_conclusion(None, expect_result) + + # expect_merged: boolean representing whether the PR should be merged + def check_pull_request_result(self, expect_merged: bool): + super().check_pull_request_result(self.secrets.pr_number, expect_merged) + + def check_pull_request_labels(self): + super().check_pull_request_labels(self.secrets.pr_number) + + def check_pull_request_comments(self, expect_message: str): + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/issues/{self.secrets.pr_number}/comments', self.secrets.bot_token) + logging.info(f'STATUS_CODE: {r.status_code}') + + response = json.loads(r.text) + logging.debug(f"CHECK PULL_REQUEST COMMENT RESPONSE: {response}") + if len(response) == 0: + raise AssertionError("No comment found in the PR") + complete_comment = response[0]['body'] + + if expect_message in complete_comment: + logging.info("Found the expected comment in the PR") + else: + raise AssertionError(f"Was expecting '{expect_message}' in the comment {complete_comment}") + + def check_index_yaml(self, check_provider_type=False): + super().check_index_yaml(self.secrets.base_branch, self.secrets.vendor, self.secrets.chart_name, self.secrets.chart_version, self.secrets.index_file,check_provider_type) + + def check_release_result(self): + chart_tgz = self.secrets.test_chart.split('/')[-1] + super().check_release_result(self.secrets.vendor, self.secrets.chart_name, self.secrets.chart_version, chart_tgz) + + def cleanup_release(self): + expected_tag = f'{self.secrets.vendor}-{self.secrets.chart_name}-{self.secrets.chart_version}' + super().cleanup_release(expected_tag) + +@dataclass +class ChartCertificationE2ETestMultiple(ChartCertificationE2ETest): + secrets: E2ETestSecretRecursive = E2ETestSecretRecursive() + + def __post_init__(self) -> None: + bot_name, bot_token = self.get_bot_name_and_token() + dry_run = self.get_dry_run() + notify_id = self.get_notify_id() + software_name, software_version = self.get_software_name_version() + vendor_type = self.get_vendor_type() + + test_repo = TEST_REPO + base_branches = [] + pr_branches = [] + + pr_base_branch = self.repo.active_branch.name + r = github_api( + 'get', f'repos/{test_repo}/branches', bot_token) + branches = json.loads(r.text) + branch_names = [branch['name'] for branch in branches] + if pr_base_branch not in branch_names: + logging.info( + f"{test_repo}:{pr_base_branch} does not exists, creating with local branch") + self.repo.git.push(f'https://x-access-token:{bot_token}@github.com/{test_repo}', + f'HEAD:refs/heads/{pr_base_branch}', '-f') + + self.secrets = E2ETestSecretRecursive() + self.secrets.software_name = software_name + self.secrets.software_version = software_version + self.secrets.test_repo = test_repo + self.secrets.bot_name = bot_name + self.secrets.bot_token = bot_token + self.secrets.vendor_type = vendor_type + self.secrets.pr_base_branch = pr_base_branch + self.secrets.base_branches = base_branches + self.secrets.pr_branches = pr_branches + self.secrets.dry_run = dry_run + self.secrets.notify_id = notify_id + self.secrets.owners_file_content = self.owners_file_content + self.secrets.release_tags = list() + + def cleanup (self): + # Teardown step to cleanup releases and branches + for release_tag in self.secrets.release_tags: + self.cleanup_release(release_tag) + + self.repo.git.worktree('prune') + for base_branch in self.secrets.base_branches: + logging.info(f"Delete '{self.secrets.test_repo}:{base_branch}'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/heads/{base_branch}', self.secrets.bot_token) + + logging.info(f"Delete '{self.secrets.test_repo}:{base_branch}-gh-pages'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/heads/{base_branch}-gh-pages', self.secrets.bot_token) + + logging.info(f"Delete local '{base_branch}'") + try: + self.repo.git.branch('-D', base_branch) + except git.exc.GitCommandError: + logging.info(f"Local '{base_branch}' does not exist") + + for pr_branch in self.secrets.pr_branches: + logging.info(f"Delete '{self.secrets.test_repo}:{pr_branch}'") + github_api( + 'delete', f'repos/{self.secrets.test_repo}/git/refs/heads/{pr_branch}', self.secrets.bot_token) + + try: + logging.info("Delete local 'tmp' branch") + self.temp_repo.git.branch('-D', 'tmp') + except git.exc.GitCommandError: + logging.info(f"Local 'tmp' branch does not exist") + + def get_dry_run(self): + # Accepts 'true' or 'false', depending on whether we want to notify + # Don't notify on dry runs, default to True + dry_run = False if os.environ.get("DRY_RUN") == 'false' else True + # Don't notify if not triggerd on PROD_REPO and PROD_BRANCH + if not dry_run: + triggered_branch = os.environ.get("GITHUB_REF").split('/')[-1] + triggered_repo = os.environ.get("GITHUB_REPOSITORY") + if triggered_repo != PROD_REPO or triggered_branch != PROD_BRANCH: + dry_run = True + return dry_run + + def get_notify_id(self): + # Accepts comma separated Github IDs or empty strings to override people to tag in notifications + notify_id = os.environ.get("NOTIFY_ID") + if notify_id: + notify_id = [vt.strip() for vt in notify_id.split(',')] + else: + notify_id = ["dperaza","mmulholla"] + return notify_id + + def get_software_name_version(self): + software_name = os.environ.get("SOFTWARE_NAME") + if not software_name: + raise Exception("SOFTWARE_NAME environment variable not defined") + + software_version = os.environ.get("SOFTWARE_VERSION").strip('\"') + if not software_version: + raise Exception("SOFTWARE_VERSION environment variable not defined") + elif software_version.startswith("sha256"): + software_version = software_version[-8:] + + return software_name, software_version + + def get_vendor_type(self): + vendor_type = os.environ.get("VENDOR_TYPE") + if not vendor_type: + logging.info( + f"VENDOR_TYPE environment variable not defined, default to `all`") + vendor_type = 'all' + return vendor_type + + def setup_temp_dir(self): + self.temp_dir = TemporaryDirectory(prefix='tci-') + with SetDirectory(Path(self.temp_dir.name)): + # Make PR's from a temporary directory + logging.info(f'Worktree directory: {self.temp_dir.name}') + self.repo.git.worktree('add', '--detach', self.temp_dir.name, f'HEAD') + self.temp_repo = git.Repo(self.temp_dir.name) + + # Run submission flow test with charts in PROD_REPO:PROD_BRANCH + self.set_git_username_email(self.temp_repo, self.secrets.bot_name, GITHUB_ACTIONS_BOT_EMAIL) + self.temp_repo.git.checkout(PROD_BRANCH, 'charts') + self.temp_repo.git.restore('--staged', 'charts') + self.secrets.submitted_charts = get_all_charts( + 'charts', self.secrets.vendor_type) + logging.info( + f"Found charts for {self.secrets.vendor_type}: {self.secrets.submitted_charts}") + self.temp_repo.git.checkout('-b', 'tmp') + + def get_owner_ids(self, chart_directory, owners_table): + + with open(f'{chart_directory}/OWNERS', 'r') as fd: + try: + owners = yaml.safe_load(fd) + # Pick owner ids for notification + owners_table[chart_directory] = [ + owner.get('githubUsername', '') for owner in owners['users']] + except yaml.YAMLError as err: + logging.warning( + f"Error parsing OWNERS of {chart_directory}: {err}") + + def push_chart(self, chart_directory, chart_name, chart_version, vendor_name, vendor_type, pr_branch): + # Push chart files to test_repo:pr_branch + self.temp_repo.git.add(f'{chart_directory}/{chart_version}') + self.temp_repo.git.commit( + '-m', f"Add {vendor_type} {vendor_name} {chart_name} {chart_version} chart files") + self.temp_repo.git.push(f'https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}', + f'HEAD:refs/heads/{pr_branch}', '-f') + + def report_failure(self,chart,chart_owners,failure_type,pr_html_url=None,run_html_url=None): + + os.environ['GITHUB_REPO'] = PROD_REPO.split('/')[1] + os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token + if not self.secrets.dry_run: + os.environ['GITHUB_REPO'] = PROD_REPO.split('/')[1] + os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token + os.environ['GITHUB_ORGANIZATION'] = PROD_REPO.split('/')[0] + logging.info(f"Send notification to '{self.secrets.notify_id}' about verification result of '{chart}'") + create_verification_issue(chart, chart_owners, failure_type,self.secrets.notify_id, pr_html_url, run_html_url, self.secrets.software_name, + self.secrets.software_version, self.secrets.bot_token, self.secrets.dry_run) + else: + os.environ['GITHUB_ORGANIZATION'] = PROD_REPO.split('/')[0] + os.environ['GITHUB_REPO'] = "sandbox" + os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token + logging.info(f"Send notification to '{self.secrets.notify_id}' about dry run verification result of '{chart}'") + create_verification_issue(chart, chart_owners, failure_type,self.secrets.notify_id, pr_html_url, run_html_url, self.secrets.software_name, + self.secrets.software_version, self.secrets.bot_token, self.secrets.dry_run) + logging.info(f"Dry Run - send sandbox notification to '{chart_owners}' about verification result of '{chart}'") + + + def check_single_chart_result(self, vendor_type, vendor_name, chart_name, chart_version, pr_number, owners_table): + base_branch = f'{self.secrets.software_name}-{self.secrets.software_version}-{self.secrets.pr_base_branch}-{vendor_type}-{vendor_name}-{chart_name}-{chart_version}' + + # Check workflow conclusion + chart = f'{vendor_name} {chart_name} {chart_version}' + run_id, conclusion = super().check_workflow_conclusion(pr_number, 'success', failure_type='warning') + + if conclusion and run_id: + if conclusion != 'success': + # Send notification to owner through GitHub issues + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/actions/runs/{run_id}', self.secrets.bot_token) + run = r.json() + run_html_url = run['html_url'] + + pr = get_pr(self.secrets,pr_number) + pr_html_url = pr["html_url"] + chart_directory = f'charts/{vendor_type}/{vendor_name}/{chart_name}' + chart_owners = owners_table[chart_directory] + + self.report_failure(chart,chart_owners,CHECKS_FAILED,pr_html_url,run_html_url) + + logging.warning(f"PR{pr_number} workflow failed: {vendor_name}, {chart_name}, {chart_version}") + return + else: + logging.info(f"PR{pr_number} workflow passed: {vendor_name}, {chart_name}, {chart_version}") + else: + logging.warning(f"PR{pr_number} workflow did not complete: {vendor_name}, {chart_name}, {chart_version}") + return + + + # Check PRs are merged + if not super().check_pull_request_result(pr_number, True, failure_type='warning'): + logging.warning(f"PR{pr_number} pull request was not merged: {vendor_name}, {chart_name}, {chart_version}") + return + logging.info(f"PR{pr_number} pull request was merged: {vendor_name}, {chart_name}, {chart_version}") + + # Check index.yaml is updated + if not super().check_index_yaml(base_branch, vendor_name, chart_name, chart_version, check_provider_type=False, failure_type='warning'): + logging.warning(f"PR{pr_number} - Chart was not found in Index file: {vendor_name}, {chart_name}, {chart_version}") + logging.info(f"PR{pr_number} - Chart was found in Index file: {vendor_name}, {chart_name}, {chart_version}") + + # Check release is published + chart_tgz = f'{chart_name}-{chart_version}.tgz' + if not super().check_release_result(vendor_name, chart_name, chart_version, chart_tgz, failure_type='warning'): + logging.warning(f"PR{pr_number} - Release was not created: {vendor_name}, {chart_name}, {chart_version}") + logging.info(f"PR{pr_number} - Release was created: {vendor_name}, {chart_name}, {chart_version}") + + def process_single_chart(self, vendor_type, vendor_name, chart_name, chart_version, pr_number_list, owners_table): + # Get SHA from 'pr_base_branch' branch + logging.info(f"Process chart: {vendor_type}/{vendor_name}/{chart_name}/{chart_version}") + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/git/ref/heads/{self.secrets.pr_base_branch}', self.secrets.bot_token) + j = json.loads(r.text) + pr_base_branch_sha = j['object']['sha'] + + chart_directory = f'charts/{vendor_type}/{vendor_name}/{chart_name}' + base_branch = f'{self.secrets.software_name}-{self.secrets.software_version}-{self.secrets.pr_base_branch}-{vendor_type}-{vendor_name}-{chart_name}-{chart_version}' + base_branch = base_branch.replace(":","-") + pr_branch = f'{base_branch}-pr-branch' + + self.secrets.base_branches.append(base_branch) + self.secrets.pr_branches.append(pr_branch) + self.temp_repo.git.checkout('tmp') + self.temp_repo.git.checkout('-b', base_branch) + + # Create test gh-pages branch for checking index.yaml + self.create_test_gh_pages_branch(self.secrets.test_repo, base_branch, self.secrets.bot_token) + + # Create a new base branch for testing current chart + logging.info( + f"Create {self.secrets.test_repo}:{base_branch} for testing") + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/branches', self.secrets.bot_token) + branches = json.loads(r.text) + branch_names = [branch['name'] for branch in branches] + if base_branch in branch_names: + logging.warning( + f"{self.secrets.test_repo}:{base_branch} already exists") + return + data = {'ref': f'refs/heads/{base_branch}', + 'sha': pr_base_branch_sha} + r = github_api( + 'post', f'repos/{self.secrets.test_repo}/git/refs', self.secrets.bot_token, json=data) + + # Remove chart and owners file from git + self.remove_chart(chart_directory, chart_version, self.secrets.test_repo, base_branch, self.secrets.bot_token) + self.remove_owners_file(chart_directory, self.secrets.test_repo, base_branch, self.secrets.bot_token) + + # Get owners id for notifications + self.get_owner_ids(chart_directory, owners_table) + + # Create and push test owners file + super().create_and_push_owners_file(chart_directory, base_branch, vendor_name, vendor_type, chart_name) + + # Push test chart to pr_branch + self.push_chart(chart_directory, chart_name, chart_version, vendor_name, vendor_type, pr_branch) + + # Create PR from pr_branch to base_branch + logging.info("sleep for 5 seconds to avoid secondary api limit") + time.sleep(5) + pr_number = super().send_pull_request(self.secrets.test_repo, base_branch, pr_branch, self.secrets.bot_token) + pr_number_list.append((vendor_type, vendor_name, chart_name, chart_version, pr_number)) + logging.info(f"PR{pr_number} created in {self.secrets.test_repo} into {base_branch} from {pr_branch}") + + # Record expected release tags + self.secrets.release_tags.append(f'{vendor_name}-{chart_name}-{chart_version}') + + def process_all_charts(self): + self.setup_git_context(self.repo) + self.setup_temp_dir() + + owners_table = dict() + pr_number_list = list() + + skip_charts = list() + + logging.info(f"Running tests for : {self.secrets.software_name} {self.secrets.software_version} :") + # First look for charts in index.yaml to see if kubeVersion is good: + if self.secrets.software_name == "OpenShift": + logging.info("check index file for invalid kubeVersions") + failed_charts = check_index_entries(self.secrets.software_version) + if failed_charts: + for chart in failed_charts: + providerDir = chart["providerType"].replace("partner","partners") + chart_directory = f'charts/{providerDir}/{chart["provider"]}/{chart["name"]}' + self.get_owner_ids(chart_directory,owners_table) + chart_owners = owners_table[chart_directory] + chart_id = f'{chart["provider"]} {chart["name"]} {chart["version"]}' + self.report_failure(chart_id,chart_owners,chart["message"],"","") + skip_charts.append(f'{chart["name"]}-{chart["version"]}') + + + # Process test charts and send PRs from temporary directory + with SetDirectory(Path(self.temp_dir.name)): + for vendor_type, vendor_name, chart_name, chart_version in self.secrets.submitted_charts: + if f'{chart_name}-{chart_version}' in skip_charts: + logging.info(f"Skip already failed chart: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}") + else: + logging.info(f"Process chart: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}") + self.process_single_chart(vendor_type, vendor_name, chart_name, chart_version, pr_number_list, owners_table) + logging.info("sleep for 5 seconds to avoid secondary api limit") + time.sleep(5) + + for vendor_type, vendor_name, chart_name, chart_version, pr_number in pr_number_list: + logging.info(f"PR{pr_number} Check result: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}") + self.check_single_chart_result(vendor_type, vendor_name, chart_name, chart_version, pr_number, owners_table) + + diff --git a/tests/functional/behave_features/common/utils/github.py b/tests/functional/behave_features/common/utils/github.py new file mode 100644 index 0000000000..39a9aa21c5 --- /dev/null +++ b/tests/functional/behave_features/common/utils/github.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +"""Utility class for setting up and manipulating GitHub operations.""" + +import json +import requests +from retrying import retry + +from common.utils.setttings import * + +@retry(stop_max_delay=30_000, wait_fixed=1000) +def get_run_id(secrets, pr_number=None): + + pr = get_pr(secrets, pr_number) + r = github_api( + 'get', f'repos/{secrets.test_repo}/actions/runs', secrets.bot_token) + runs = json.loads(r.text) + + for run in runs['workflow_runs']: + if run['head_sha'] == pr['head']['sha'] and run['name'] == CERTIFICATION_CI_NAME: + return run['id'] + else: + raise Exception("Workflow for the submitted PR did not run.") + + +@retry(stop_max_delay=60_000*40, wait_fixed=2000) +def get_run_result(secrets, run_id): + r = github_api( + 'get', f'repos/{secrets.test_repo}/actions/runs/{run_id}', secrets.bot_token) + run = json.loads(r.text) + + if run['conclusion'] is None: + raise Exception(f"Workflow {run_id} is still running, PR: {secrets.pr_number} ") + + return run['conclusion'] + + +@retry(stop_max_delay=10_000, wait_fixed=1000) +def get_release_assets(secrets, release_id, required_assets): + r = github_api( + 'get', f'repos/{secrets.test_repo}/releases/{release_id}/assets', secrets.bot_token) + asset_list = json.loads(r.text) + asset_names = [asset['name'] for asset in asset_list] + missing_assets = list() + for asset in required_assets: + if asset not in asset_names: + missing_assets.append(asset) + if len(missing_assets) > 0: + raise Exception(f"Missing release asset: {missing_assets}") + + +@retry(stop_max_delay=15_000, wait_fixed=1000) +def get_release_by_tag(secrets, release_tag): + r = github_api( + 'get', f'repos/{secrets.test_repo}/releases', secrets.bot_token) + releases = json.loads(r.text) + for release in releases: + if release['tag_name'] == release_tag: + return release + raise Exception("Release not published") + + +def get_pr(secrets, pr_number=None): + pr_number = secrets.pr_number if pr_number is None else pr_number + r = github_api( + 'post', f'repos/{secrets.test_repo}/pulls/{pr_number}', secrets.bot_token) + pr = json.loads(r.text) + return pr + + +def github_api_get(endpoint, bot_token, headers={}): + if not headers: + headers = {'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'Bearer {bot_token}'} + r = requests.get(f'{GITHUB_BASE_URL}/{endpoint}', headers=headers) + + return r + + +def github_api_delete(endpoint, bot_token, headers={}): + if not headers: + headers = {'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'Bearer {bot_token}'} + r = requests.delete(f'{GITHUB_BASE_URL}/{endpoint}', headers=headers) + + return r + + +def github_api_post(endpoint, bot_token, headers={}, json={}): + if not headers: + headers = {'Accept': 'application/vnd.github.v3+json', + 'Authorization': f'Bearer {bot_token}'} + r = requests.post(f'{GITHUB_BASE_URL}/{endpoint}', + headers=headers, json=json) + + return r + + +def github_api(method, endpoint, bot_token, headers={}, data={}, json={}): + if method == 'get': + return github_api_get(endpoint, bot_token, headers=headers) + elif method == 'post': + return github_api_post(endpoint, bot_token, headers=headers, json=json) + elif method == 'delete': + return github_api_delete(endpoint, bot_token, headers=headers) + else: + raise ValueError( + "Github API method not implemented in helper function") diff --git a/tests/functional/behave_features/common/utils/index.py b/tests/functional/behave_features/common/utils/index.py new file mode 100644 index 0000000000..a6e6d97e6b --- /dev/null +++ b/tests/functional/behave_features/common/utils/index.py @@ -0,0 +1,43 @@ + +import logging +import semantic_version +import sys + +sys.path.append('../../../../../scripts/src') +from chartrepomanager import indexannotations +from indexfile import index + + + +def check_index_entries(ocpVersion): + + all_chart_list = index.get_latest_charts() + failed_chart_list = [] + + OCP_VERSION = semantic_version.Version.coerce(ocpVersion) + + for chart in all_chart_list: + if "supportedOCP" in chart and chart["supportedOCP"] != "N/A" and chart["supportedOCP"] != "": + if OCP_VERSION in semantic_version.NpmSpec(chart["supportedOCP"]): + logging.info(f'PASS: Chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} includes: {OCP_VERSION}') + else: + chart["message"] = f'chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} does not include latest OCP version {OCP_VERSION}' + logging.info(f' ERROR: Chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} does not include {OCP_VERSION}') + failed_chart_list.append(chart) + elif "kubeVersion" in chart and chart["kubeVersion"] != "": + supportedOCPVersion = indexannotations.getOCPVersions(chart["kubeVersion"]) + if OCP_VERSION in semantic_version.NpmSpec(supportedOCPVersion): + logging.info(f'PASS: Chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) includes OCP version: {OCP_VERSION}') + else: + chart["message"] = f'chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) does not include latest OCP version {OCP_VERSION}' + logging.info(f' ERROR: Chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) does not include {OCP_VERSION}') + failed_chart_list.append(chart) + + return failed_chart_list + + + + + + + diff --git a/tests/functional/behave_features/common/utils/notifier.py b/tests/functional/behave_features/common/utils/notifier.py new file mode 100755 index 0000000000..fc1b433d1c --- /dev/null +++ b/tests/functional/behave_features/common/utils/notifier.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +"""Utility module for sending chart owners notifications.""" + +import json +import os +import sys + +import requests + +from common.utils.setttings import * + +endpoint_data = {} + +CHECKS_FAILED = "checks failed" + +def _set_endpoint_key(key, env_var): + if key not in endpoint_data: + if env_var in os.environ: + endpoint_data[key] = os.environ[env_var] + else: + raise Exception( + f"Environment variables {env_var} is required to connect to github") + + +def _set_endpoint(): + _set_endpoint_key("access_token", "GITHUB_AUTH_TOKEN") + _set_endpoint_key("organization", "GITHUB_ORGANIZATION") + _set_endpoint_key("repo", "GITHUB_REPO") + + +def _make_gihub_request(method, uri, body=None, params={}, headers={}, verbose=False): + headers.update({"Authorization": f'Bearer {endpoint_data["access_token"]}', + "Accept": "application/vnd.github.v3+json"}) + + url = f'{GITHUB_BASE_URL}/repos/{endpoint_data["organization"]}/{endpoint_data["repo"]}/{uri}' + + print(f"API url: {url}") + method_map = {"get": requests.get, + "post": requests.post, + "put": requests.put, + "delete": requests.delete, + "patch": requests.patch} + request_method = method_map[method] + response = request_method(url, params=params, headers=headers, json=body) + if verbose: + print(json.dumps(headers, indent=4, sort_keys=True)) + print(json.dumps(body, indent=4, sort_keys=True)) + print(json.dumps(params, indent=4, sort_keys=True)) + print(response.text) + response.raise_for_status() + try: + resp_json = response.json() + except Exception: + resp_json = None + if resp_json and verbose: + print(json.dumps(resp_json, indent=4, sort_keys=True)) + return resp_json + +# Call this method directly if you are not creating a verification issue nor a version change issue. +def create_an_issue(title, description, assignees=[], labels=[]): + uri = "issues" + method = "post" + body = {"title": title, + "body": description, + "assignees": assignees, + "labels": labels} + _make_gihub_request(method, uri, body=body, verbose=False) + + +def _verify_endpoint(access_token): + if "repo" not in endpoint_data: + raise Exception("GITHUB_REPO environment variable not defined") + + if "organization" not in endpoint_data: + raise Exception("GITHUB_ORGANIZATION environment variable not defined") + + if access_token: + endpoint_data["access_token"] = access_token + + +def create_verification_issue(chart, chart_owners, failure_type, notify_developers, pr_url, report_url, software_name, software_version, access_token=None, dry_run=False): + """Create and issue with chart-verifier findings after a version change trigger. + + chart_name -- Name of the chart that was verified. Include version for more verbose information\n + chart_owners -- Github IDs of the chart owners\n + failure_type - Indication of the type of failure + report_url -- URL or the report resulting from verification if applicable\n + kube-version -- The kubeVersion attribute of the chart if it is bade.\n + software_name -- Name of the software dependency that changed e.g, OCP and Chart Verifier\n + software_version -- The softwared dependency version used\n + access_token -- An optional github access token secret. If not passed will try to get from GITHUB_AUTH_TOKEN environment variable\ + dry-run -- Set if the test run is a dry-run. + """ + + + title = f"Chart {chart}" + if dry_run: + title = f"Dry Run: Chart {chart}" + + if failure_type == CHECKS_FAILED: + title = f"{title} has failures with {software_name} version {software_version}" + report_result = "some chart checks have failed. Please review the failures and, if required, consider submitting a new chart version with the appropriate additions/corrections." + body = (f"FYI @{' @'.join(notify_developers)}, in PR {pr_url} we triggered the chart certification workflow against chart {chart} because the workflow " + f"now supports {software_name} version {software_version}. We have found that {report_result}. Check details in the report: " + f"{report_url}, Chart owners are: {chart_owners}") + else: + title = f"{title} does not support {software_name} version {software_version}" + body = (f"FYI @{' @'.join(notify_developers)}, we checked the OCP versions supported by {chart} because the workflow " + f"now supports {software_name} version {software_version}. We have found that {failure_type}. Chart owners are: {chart_owners}") + + _set_endpoint() + _verify_endpoint(access_token) + create_an_issue(title, body) + + + + +def create_version_change_issue(chart_name, chart_owners, software_name, software_version, access_token=None): + """Create and issue with new version of software dependencies supported by certitifcation program. + + chart_name -- Name of the chart afected. Include version for more verbose information + chart_owners -- Github IDs of the chart owners\n + software_name -- Name of the software dependency that changed e.g, OCP and Chart Verifier\n + software_version -- The softwared dependency version used\n + access_token -- An optional github access token secret. If not passed will try to get from GITHUB_AUTH_TOKEN environment variable\n + """ + + title = f"Action needed for {chart_name} after a certification dependency change" + + body = (f"FYI @{' @'.join(chart_owners)}, {software_name} {software_version} is now supported by the certification program. " + "Consider submiting a new chart version.") + + _set_endpoint() + _verify_endpoint(access_token) + create_an_issue(title, body) + + +if __name__ == "__main__": + # Collecting info interactively + print("Enter the chart name: ") + chart_name = sys.stdin.readline().strip() + print("Enter chart owners: ") + chart_owners = sys.stdin.readline().strip().split() + print("Enter the github organization: ") + organization = sys.stdin.readline().strip() + print("Enter the github repo: ") + repo = sys.stdin.readline().strip() + + # setting endpoint + print(f"Creating custom issue in https://github.com/{organization}/{repo}") + endpoint_data["organization"] = organization + endpoint_data["repo"] = repo + + print("Enter the name of software dependency that changed: ") + software_name = sys.stdin.readline().strip() + print("Enter the version of software dependency that changed: ") + software_version = sys.stdin.readline().strip() + + print("What type of issue are you creating (verification/version-change)?: ") + issue_type = sys.stdin.readline().strip() + + if issue_type == "verification": + print("Enter the report url: ") + report_url = sys.stdin.readline().strip() + print("Did the chart verification pass (yes/no)?: ") + pass_answer = sys.stdin.readline().strip() + pass_verification = pass_answer == "yes" + create_verification_issue(chart_name, chart_owners, report_url, + software_name, software_version, pass_verification=pass_verification) + else: + create_version_change_issue( + chart_name, chart_owners, software_name, software_version) diff --git a/tests/functional/behave_features/common/utils/secret.py b/tests/functional/behave_features/common/utils/secret.py new file mode 100644 index 0000000000..d3240134cc --- /dev/null +++ b/tests/functional/behave_features/common/utils/secret.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +"""Utility class for storing test specific settings.""" + +from dataclasses import dataclass + +@dataclass +class E2ETestSecret: + # common secrets between one-shot and recursive tests + test_repo: str = '' + bot_name: str = '' + bot_token: str = '' + pr_number: int = -1 + vendor_type: str = '' + owners_file_content: str = '' + test_chart: str = '' + test_report: str = '' + chart_name: str = '' + chart_version: str = '' + +@dataclass +class E2ETestSecretOneShot(E2ETestSecret): + # one-shot testing + active_branch: str = '' + base_branch: str = '' + pr_branch: str = '' + pr_number: int = -1 + vendor: str = '' + bad_version: str = '' + provider_delivery: bool = False + index_file: str = "index.yaml" + +@dataclass +class E2ETestSecretRecursive(E2ETestSecret): + # recursive testing + software_name: str = '' + software_version: str = '' + pr_base_branch: str = '' + base_branches: list = None + pr_branches: list = None + dry_run: bool = True + notify_id: list = None + submitted_charts: list = None + release_tags: list = None diff --git a/tests/functional/behave_features/common/utils/set_directory.py b/tests/functional/behave_features/common/utils/set_directory.py new file mode 100644 index 0000000000..921c295f74 --- /dev/null +++ b/tests/functional/behave_features/common/utils/set_directory.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +"""Sets the cwd within the context. + +Reference: https://dev.to/teckert/changing-directory-with-a-python-context-manager-2bj8 +""" + +import os +from dataclasses import dataclass +from pathlib import Path + +@dataclass +class SetDirectory(object): + """ + Args: + path (Path): The path to the cwd + """ + path: Path + origin: Path = Path().absolute() + + def __enter__(self): + os.chdir(self.path) + def __exit__(self, exc_type, exc_value, traceback): + os.chdir(self.origin) diff --git a/tests/functional/behave_features/common/utils/setttings.py b/tests/functional/behave_features/common/utils/setttings.py new file mode 100644 index 0000000000..5f2439f173 --- /dev/null +++ b/tests/functional/behave_features/common/utils/setttings.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +"""Settings and global variables for e2e tests""" + +GITHUB_BASE_URL = 'https://api.github.com' +# The sandbox repository where we run all our tests on +TEST_REPO = 'openshift-helm-charts/sandbox' +# The prod repository where we create notification issues +PROD_REPO = 'openshift-helm-charts/charts' +# The prod branch where we store all chart files +PROD_BRANCH = 'main' +# This is used to find chart certification workflow run id +CERTIFICATION_CI_NAME = 'CI' +# GitHub actions bot email for git email +GITHUB_ACTIONS_BOT_EMAIL = '41898282+github-actions[bot]@users.noreply.github.com' diff --git a/tests/functional/behave_features/environment.py b/tests/functional/behave_features/environment.py new file mode 100644 index 0000000000..f34a5d2eba --- /dev/null +++ b/tests/functional/behave_features/environment.py @@ -0,0 +1,23 @@ +from behave import fixture, use_fixture +from common.utils.chart_certification import ChartCertificationE2ETestSingle +from common.utils.chart_certification import ChartCertificationE2ETestMultiple + +@fixture +def workflow_test(context): + context.workflow_test = ChartCertificationE2ETestSingle(test_name=context.test_name) + yield context.workflow_test + context.workflow_test.cleanup() + +@fixture +def submitted_chart_test(context): + context.chart_test = ChartCertificationE2ETestMultiple() + yield context.chart_test + context.chart_test.cleanup() + +def before_scenario(context, scenario): + if 'version-change' in scenario.tags: + print("[INFO] Using submitted charts fixture") + use_fixture(submitted_chart_test, context) + else: + context.test_name = scenario.name.split('@')[0][:-4].split(']')[1] + use_fixture(workflow_test, context) diff --git a/tests/functional/behave_features/steps/implementation.py b/tests/functional/behave_features/steps/implementation.py new file mode 100644 index 0000000000..8830d5af58 --- /dev/null +++ b/tests/functional/behave_features/steps/implementation.py @@ -0,0 +1,249 @@ +from behave import given, when, then + +############### Common step definitions ############### +@given(u'the vendor "{vendor}" has a valid identity as "{vendor_type}"') +def vendor_has_valid_identity(context, vendor, vendor_type): + context.workflow_test.set_vendor(vendor, vendor_type) + +@given(u'an error-free chart source is used in "{chart_path}"') +def chart_source_is_used(context, chart_path): + context.workflow_test.update_test_chart(chart_path) + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + context.workflow_test.process_chart(is_tarball=False) + context.workflow_test.push_chart(is_tarball=False) + +@given(u'chart source is used in "{chart_path}"') +def user_has_used_chart_src(context, chart_path): + context.workflow_test.update_test_chart(chart_path) + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + context.workflow_test.process_chart(is_tarball=False) + +@given(u'an error-free chart tarball is used in "{chart_path}"') +def user_has_created_error_free_chart_tarball(context, chart_path): + context.workflow_test.update_test_chart(chart_path) + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + context.workflow_test.process_chart(is_tarball=True) + context.workflow_test.push_chart(is_tarball=True) + +@given(u'an error-free chart tarball used in "{chart_path}" and report in "{report_path}"') +def user_has_created_error_free_chart_tarball_and_report(context, chart_path, report_path): + context.workflow_test.update_test_chart(chart_path) + context.workflow_test.update_test_report(report_path) + + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + context.workflow_test.process_chart(is_tarball=True) + context.workflow_test.process_report() + context.workflow_test.push_chart(is_tarball=True) + +@given(u'a chart tarball is used in "{chart_path}" and report in "{report_path}"') +def user_has_created_a_chart_tarball_and_report(context, chart_path, report_path): + context.workflow_test.update_test_chart(chart_path) + context.workflow_test.update_test_report(report_path) + + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + context.workflow_test.process_chart(is_tarball=True) + +@given(u'an error-free chart source used in "{chart_path}" and report in "{report_path}"') +def user_has_created_error_free_chart_src_and_report(context, chart_path, report_path): + context.workflow_test.update_test_chart(chart_path) + context.workflow_test.update_test_report(report_path) + + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + context.workflow_test.process_chart(is_tarball=False) + context.workflow_test.process_report() + context.workflow_test.push_chart(is_tarball=False) + +@given(u'report is used in "{report_path}"') +@given(u'an error-free report is used in "{report_path}"') +def user_has_created_error_free_report(context, report_path): + context.workflow_test.update_test_report(report_path) + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + context.workflow_test.process_report() + +@given(u'a "{report_path}" is provided') +def user_generated_a_report(context, report_path): + context.workflow_test.update_test_report(report_path) + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + +@when(u'the user sends a pull request with the report') +@when(u'the user sends a pull request with the chart') +@when(u'the user sends a pull request with the chart and report') +def user_sends_a_pull_request(context): + context.workflow_test.send_pull_request() + +@when(u'the user pushed the chart and created pull request') +def user_pushed_the_chart_and_created_pull_request(context): + context.workflow_test.push_chart(is_tarball=False) + context.workflow_test.send_pull_request() + +@then(u'the user sees the pull request is merged') +def pull_request_is_merged(context): + context.workflow_test.check_workflow_conclusion(expect_result='success') + context.workflow_test.check_pull_request_result(expect_merged=True) + context.workflow_test.check_pull_request_labels() + +@then(u'the index.yaml file is updated with an entry for the submitted chart') +def index_yaml_updated_with_submitted_chart(context): + context.workflow_test.check_index_yaml() + +@then(u'a release is published with corresponding report and chart tarball') +def release_is_published(context): + context.workflow_test.check_release_result() + +@then(u'the pull request is not merged') +def pull_request_is_not_merged(context): + context.workflow_test.check_workflow_conclusion(expect_result='failure') + context.workflow_test.check_pull_request_result(expect_merged=False) + +@then(u'user gets the "{message}" in the pull request comment') +def user_gets_a_message(context, message): + context.workflow_test.check_pull_request_comments(expect_message=message) + +########## Unique step definitions ################# + +@given(u'README file is missing in the chart') +def readme_file_is_missing(context): + context.workflow_test.remove_readme_file() + +@then(u'the index.yaml file is updated with an entry for the submitted chart with correct providerType') +def index_yaml_is_updated_with_new_entry_with_correct_provider_type(context): + context.workflow_test.check_index_yaml(check_provider_type=True) + +@given(u'the report contains an "{invalid_url}"') +def invalid_url_in_the_report(context, invalid_url): + context.workflow_test.process_report(update_url=True, url=invalid_url) + +@given(u'user adds a non chart related file') +def user_adds_a_non_chart_related_file(context): + context.workflow_test.add_non_chart_related_file() + +@when(u'the user sends a pull request with both chart and non related file') +def user_sends_pull_request_with_chart_and_non_related_file(context): + context.workflow_test.push_chart(is_tarball=False, add_non_chart_file=True) + context.workflow_test.send_pull_request() + +@given(u'provider delivery control is set to "{provider_control_owners}" in the OWNERS file') +def provider_delivery_control_set_in_owners(context, provider_control_owners): + if provider_control_owners == "true": + context.workflow_test.secrets.provider_delivery=True + else: + context.workflow_test.secrets.provider_delivery=False + +@given(u'provider delivery control is set to "{provider_control_report}" in the report') +def provider_delivery_control_set_in_report(context, provider_control_report): + if provider_control_report == "true": + context.workflow_test.process_report(update_provider_delivery=True, provider_delivery=True) + else: + context.workflow_test.process_report(update_provider_delivery=True, provider_delivery=False) + +@given(u'provider delivery controls is set to "{provider_control_report}" and a package digest is "{package_digest_set}" in the report') +def provider_delivery_control_and_package_digest_set_in_report(context, provider_control_report, package_digest_set=True): + if package_digest_set == "true": + no_package_digest = False + else: + no_package_digest = True + + if provider_control_report == "true": + context.workflow_test.process_report(update_provider_delivery=True, provider_delivery=True, unset_package_digest=no_package_digest) + else: + context.workflow_test.process_report(update_provider_delivery=True, provider_delivery=False, unset_package_digest=no_package_digest) + +@then(u'the "{index_file}" is updated with an entry for the submitted chart') +def index_file_is_updated(context, index_file): + context.workflow_test.secrets.index_file = index_file + context.workflow_test.check_index_yaml(True) + +@given(u'the report includes "{tested}" and "{supported}" OpenshiftVersion values and chart "{kubeversion}" value') +def report_includes_specified_versions(context, tested, supported, kubeversion): + context.workflow_test.process_report(update_versions=True, supported_versions=supported, tested_version=tested, kube_version=kubeversion) + +@given(u'the report has a "{check}" missing') +def report_has_a_check_missing(context, check): + context.workflow_test.process_report(missing_check=check) + +@given(u'A "{user}" wants to submit a chart in "{chart_path}"') +def user_wants_to_submit_a_chart(context, user, chart_path): + context.workflow_test.update_test_chart(chart_path) + context.workflow_test.update_bot_name(user) + +@given(u'An authorized user wants to submit a chart in "{chart_path}"') +def authorized_user_wants_to_submit_a_chart(context, chart_path): + context.workflow_test.update_test_chart(chart_path) + +@given(u'the user creates a branch to add a new chart version') +def the_user_creates_a_branch_to_add_a_new_chart_version(context): + context.workflow_test.setup_git_context() + context.workflow_test.setup_gh_pages_branch() + context.workflow_test.setup_temp_dir() + context.workflow_test.process_owners_file() + context.workflow_test.process_chart(is_tarball=False) + if context.workflow_test.secrets.bad_version: + context.workflow_test.update_chart_version_in_chart_yaml(context.workflow_test.secrets.bad_version) + context.workflow_test.push_chart(is_tarball=False) + +@given(u'Chart.yaml specifies a "{bad_version}"') +def chart_yaml_specifies_bad_version(context, bad_version): + if bad_version != '': + context.workflow_test.update_bad_version(bad_version) + +@given(u'the report contains "{error}"') +def sha_value_does_not_match(context, error): + if error == 'sha_mismatch': + context.workflow_test.process_report(update_chart_sha=True) + else: + raise AssertionError(f"This {error} handling is not implemented yet") + +@when(u'the user sends a pull request with the chart tar and report') +def user_sends_pull_request_with_chart_tarball_and_report(context): + context.workflow_test.push_chart(is_tarball=True) + context.workflow_test.send_pull_request() + +######## Test Submitted Charts Step definitions ########## +@given(u'there is a github workflow for testing existing charts') +def theres_github_workflow_for_testing_charts(context): + print("[INFO] Running step: there is a github workflow for testing existing charts") + +@when(u'a new Openshift or chart-verifier version is specified') +def new_openshift_or_verifier_version_is_specified(context): + print("[INFO] Running step: a new Openshift or chart-verifier version is specified") + +@when(u'the vendor type is specified, e.g. partner, and/or redhat') +def vendor_type_is_specified(context): + print("[INFO] Running step: the vendor type is specified, e.g. partner, and/or redhat") + +@when(u'workflow for testing existing charts is triggered') +def workflow_is_triggered(context): + print("[INFO] Running step: workflow for testing existing charts is triggered") + +@then(u'submission tests are run for existing charts') +def submission_tests_run_for_submitted_charts(context): + print("[INFO] Running step: submission tests are run for existing charts") + context.chart_test.process_all_charts() + +@then(u'all results are reported back to the caller') +def all_results_report_back_to_caller(context): + print("[INFO] Running step: all results are reported back to the caller") \ No newline at end of file diff --git a/tests/functional/features/HC-16_chart_test_takes_more_than_30mins.feature b/tests/functional/features/HC-16_chart_test_takes_more_than_30mins.feature new file mode 100644 index 0000000000..75083899fd --- /dev/null +++ b/tests/functional/features/HC-16_chart_test_takes_more_than_30mins.feature @@ -0,0 +1,31 @@ +Feature: Chart test takes longer time and exceeds default timeout + Partners, redhat or community user submit charts which result in errors + + Examples: + | chart_path | + | tests/data/vault-test-timeout-0.17.0.tgz | + + Scenario Outline: [HC-16-001] A partner or community user submits chart that takes more than 30 mins + Given the vendor has a valid identity as + And an error-free chart tarball is used in + When the user sends a pull request with the chart + Then the pull request is not merged + And user gets the in the pull request comment + + Examples: + | vendor_type | vendor | message | + | partners | hashicorp | Chart test failure: timed out waiting for the condition | + | community | redhat | Community charts require maintainer review and approval, a review will be conducted shortly | + + Scenario Outline: [HC-16-002] A redhat associate submits a chart that takes more than 30 mins + Given the vendor has a valid identity as + And an error-free chart tarball is used in + When the user sends a pull request with the chart + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + And a release is published with corresponding report and chart tarball + + Examples: + | vendor_type | vendor | + | redhat | redhat | + diff --git a/tests/functional/features/HC-16_dash_in_version.feature b/tests/functional/features/HC-16_dash_in_version.feature new file mode 100644 index 0000000000..9d65c0c8fe --- /dev/null +++ b/tests/functional/features/HC-16_dash_in_version.feature @@ -0,0 +1,16 @@ +Feature: Report only submission + Partners, redhat and community users can publish their chart by submitting + error-free report that was generated by chart-verifier. + + Scenario Outline: [HC-16-001] A partner or redhat associate submits report only with dash in chart version + Given the vendor has a valid identity as + And an error-free report is used in + When the user sends a pull request with the report + Then the user sees the pull request is merged + And the index.yaml file is updated with an entry for the submitted chart + + Examples: + | vendor_type | vendor | report_path | + | partners | redhat | tests/data/HC-16/dash-in-version/partner/report.yaml | + | redhat | redhat | tests/data/HC-16/dash-in-version/redhat/report.yaml | + diff --git a/tests/functional/step_defs/HC-16_test_dash_in_version.py b/tests/functional/step_defs/HC-16_test_dash_in_version.py new file mode 100644 index 0000000000..4eaee44adc --- /dev/null +++ b/tests/functional/step_defs/HC-16_test_dash_in_version.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +"""Report only submission + +Partners, redhat and community users can publish their chart by submitting +error-free report that was generated by chart-verifier. +""" +import pytest +from pytest_bdd import scenario + +from functional.utils.chart_certification import ChartCertificationE2ETestSingle + +@pytest.fixture +def workflow_test(): + test_name = 'Test Chart Report Only' + workflow_test = ChartCertificationE2ETestSingle(test_name=test_name) + yield workflow_test + workflow_test.cleanup() + + +@scenario('../features/HC-16_dash_in_version.feature', "[HC-16-001] A partner or redhat associate submits report only with dash in chart version") +def test_partner_or_redhat_user_submits_report_dash_in_version(): + """A community user submits an error-free report""" diff --git a/tests/functional/step_defs/conftest.py b/tests/functional/step_defs/conftest.py index 4f9b4ecac5..83de389817 100644 --- a/tests/functional/step_defs/conftest.py +++ b/tests/functional/step_defs/conftest.py @@ -194,6 +194,7 @@ def user_should_see_pull_request_getting_merged(workflow_test): """the user sees the pull request is merged.""" workflow_test.check_workflow_conclusion(expect_result='success') workflow_test.check_pull_request_result(expect_merged=True) + workflow_test.check_pull_request_labels() @then("the pull request is not merged") def the_pull_request_is_not_getting_merged(workflow_test): diff --git a/tests/functional/step_defs/test_chart_test_takes_more_than_30mins.py b/tests/functional/step_defs/test_chart_test_takes_more_than_30mins.py new file mode 100644 index 0000000000..473d26281c --- /dev/null +++ b/tests/functional/step_defs/test_chart_test_takes_more_than_30mins.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" Chart test takes longer time and exceeds default timeout + Partners, redhat or community user submit charts which result in errors +""" +import logging +import datetime +import pytest +from pytest_bdd import scenario + +from functional.utils.chart_certification import ChartCertificationE2ETestSingle + +@pytest.fixture +def workflow_test(): + test_name = 'Chart test takes more than 30mins' + test_chart = 'tests/data/vault-test-timeout-0.17.0.tgz' + workflow_test = ChartCertificationE2ETestSingle(test_name=test_name, test_chart=test_chart) + start_time = datetime.datetime.now() + yield workflow_test + workflow_test.cleanup() + end_time = datetime.datetime.now() + time_diff = end_time - start_time + total_diff_seconds = time_diff.total_seconds() + if not int(total_diff_seconds) >= 1800: + pytest.fail(f"Timeout is not as expected: {total_diff_seconds}") + + +@scenario('../features/HC-16_chart_test_takes_more_than_30mins.feature', "[HC-16-001] A partner or community user submits chart that takes more than 30 mins") +def test_partner_or_community_chart_test_takes_more_than_30mins(): + """ A partner or community submitted chart takes more than 30 mins""" + +@scenario('../features/HC-16_chart_test_takes_more_than_30mins.feature', "[HC-16-002] A redhat associate submits a chart that takes more than 30 mins") +def test_redhat_chart_test_takes_more_than_30mins(): + """ A redhat submitted chart takes more than 30 mins""" \ No newline at end of file diff --git a/tests/functional/utils/chart_certification.py b/tests/functional/utils/chart_certification.py index d50a970de4..b9b4e4247b 100644 --- a/tests/functional/utils/chart_certification.py +++ b/tests/functional/utils/chart_certification.py @@ -16,8 +16,8 @@ import git import yaml import pytest -from functional.utils.notifier import create_verification_issue - +from functional.utils.notifier import * +from functional.utils.index import * from functional.utils.github import * from functional.utils.secret import * from functional.utils.set_directory import SetDirectory @@ -257,6 +257,28 @@ def check_pull_request_result(self, pr_number, expect_merged: bool, logger=pytes logger(f"PR{pr_number} Got unexpected status code from PR: {r.status_code}") return False + def check_pull_request_labels(self,pr_number,logger=pytest.fail): + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/issues/{pr_number}/labels', self.secrets.bot_token) + labels = json.loads(r.text) + authorized_request = False + content_ok = False + for label in labels: + logging.info(f"PR{pr_number} found label {label['name']}") + if label['name'] == "authorized-request": + authorized_request = True + if label['name'] == "content-ok": + content_ok = True + + + if authorized_request and content_ok: + logging.info(f"PR{pr_number} authorized request and content-ok labels were found as expected") + return True + else: + logger(f"PR{pr_number} authorized request and/or content-ok labels were not found as expected") + return False + + def cleanup_release(self, expected_tag): """Cleanup the release and release tag. @@ -290,7 +312,10 @@ def __post_init__(self) -> None: # different processes. self.uuid = uuid.uuid4().hex - chart_name, chart_version = self.get_chart_name_version() + if self.test_report or self.test_chart: + self.secrets.chart_name, self.secrets.chart_version = self.get_chart_name_version() + self.chart_directory = f'charts/{self.secrets.vendor_type}/{self.secrets.vendor}/{self.secrets.chart_name}' + bot_name, bot_token = self.get_bot_name_and_token() test_repo = TEST_REPO @@ -324,12 +349,9 @@ def __post_init__(self) -> None: self.secrets.bot_token = bot_token self.secrets.base_branch = base_branch self.secrets.pr_branch = pr_branch - self.secrets.chart_name = chart_name - self.secrets.chart_version = chart_version self.secrets.index_file = "index.yaml" self.secrets.provider_delivery = False - def cleanup (self): # Cleanup releases and release tags self.cleanup_release() @@ -410,6 +432,7 @@ def set_vendor(self, vendor, vendor_type): self.secrets.pr_branch = f'{self.secrets.base_branch}-pr-branch' self.chart_directory = f'charts/{self.secrets.vendor_type}/{self.secrets.vendor}/{self.secrets.chart_name}' + def setup_git_context(self): super().setup_git_context(self.repo) @@ -585,6 +608,11 @@ def check_workflow_conclusion(self, expect_result: str): def check_pull_request_result(self, expect_merged: bool): super().check_pull_request_result(self.secrets.pr_number, expect_merged, pytest.fail) + # expect_merged: boolean representing whether the PR should be merged + def check_pull_request_labels(self): + super().check_pull_request_labels(self.secrets.pr_number, pytest.fail) + + def check_pull_request_comments(self, expect_message: str): r = github_api( 'get', f'repos/{self.secrets.test_repo}/issues/{self.secrets.pr_number}/comments', self.secrets.bot_token) @@ -762,50 +790,57 @@ def push_chart(self, chart_directory, chart_name, chart_version, vendor_name, ve self.temp_repo.git.push(f'https://x-access-token:{self.secrets.bot_token}@github.com/{self.secrets.test_repo}', f'HEAD:refs/heads/{pr_branch}', '-f') + def report_failure(self,chart,chart_owners,failure_type,pr_html_url=None,run_html_url=None): + + os.environ['GITHUB_REPO'] = PROD_REPO.split('/')[1] + os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token + if not self.secrets.dry_run: + os.environ['GITHUB_REPO'] = PROD_REPO.split('/')[1] + os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token + os.environ['GITHUB_ORGANIZATION'] = PROD_REPO.split('/')[0] + logging.info(f"Send notification to '{self.secrets.notify_id}' about verification result of '{chart}'") + create_verification_issue(chart, chart_owners, failure_type,self.secrets.notify_id, pr_html_url, run_html_url, self.secrets.software_name, + self.secrets.software_version, self.secrets.bot_token, self.secrets.dry_run) + else: + os.environ['GITHUB_ORGANIZATION'] = PROD_REPO.split('/')[0] + os.environ['GITHUB_REPO'] = "sandbox" + os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token + logging.info(f"Send notification to '{self.secrets.notify_id}' about dry run verification result of '{chart}'") + create_verification_issue(chart, chart_owners, failure_type,self.secrets.notify_id, pr_html_url, run_html_url, self.secrets.software_name, + self.secrets.software_version, self.secrets.bot_token, self.secrets.dry_run) + logging.info(f"Dry Run - send sandbox notification to '{chart_owners}' about verification result of '{chart}'") + + def check_single_chart_result(self, vendor_type, vendor_name, chart_name, chart_version, pr_number, owners_table): base_branch = f'{self.secrets.software_name}-{self.secrets.software_version}-{self.secrets.pr_base_branch}-{vendor_type}-{vendor_name}-{chart_name}-{chart_version}' # Check workflow conclusion - chart = f'{vendor_type} {vendor_name} {chart_name} {chart_version}' + chart = f'{vendor_name} {chart_name} {chart_version}' run_id, conclusion = super().check_workflow_conclusion(pr_number, 'success', logging.warning) if conclusion and run_id: - # Send notification to owner through GitHub issues - r = github_api( - 'get', f'repos/{self.secrets.test_repo}/actions/runs/{run_id}', self.secrets.bot_token) - run = r.json() - run_html_url = run['html_url'] - chart_directory = f'charts/{vendor_type}/{vendor_name}/{chart_name}' - pass_verification = conclusion == 'success' - os.environ['GITHUB_REPO'] = PROD_REPO.split('/')[1] - os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token - if not self.secrets.dry_run: - chart_owners = owners_table[chart_directory] - os.environ['GITHUB_REPO'] = PROD_REPO.split('/')[1] - os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token - os.environ['GITHUB_ORGANIZATION'] = PROD_REPO.split('/')[0] - logging.info(f"PR{pr_number} Send notification to '{self.secrets.notify_id}' about verification result of '{chart}'") - create_verification_issue(f"charts/{vendor_name}/{chart_name}/{chart_version}", chart_owners, self.secrets.notify_id, run_html_url, self.secrets.software_name, - self.secrets.software_version, pass_verification, self.secrets.bot_token, self.secrets.dry_run) - else: + if conclusion != 'success': + # Send notification to owner through GitHub issues + r = github_api( + 'get', f'repos/{self.secrets.test_repo}/actions/runs/{run_id}', self.secrets.bot_token) + run = r.json() + run_html_url = run['html_url'] + + pr = get_pr(self.secrets,pr_number) + pr_html_url = pr["html_url"] + chart_directory = f'charts/{vendor_type}/{vendor_name}/{chart_name}' chart_owners = owners_table[chart_directory] - os.environ['GITHUB_ORGANIZATION'] = PROD_REPO.split('/')[0] - os.environ['GITHUB_REPO'] = "sandbox" - os.environ['GITHUB_AUTH_TOKEN'] = self.secrets.bot_token - logging.info(f"Send notification to '{self.secrets.notify_id}' about dry run verification result of '{chart}'") - create_verification_issue(f"charts/{vendor_name}/{chart_name}/{chart_version}", chart_owners, self.secrets.notify_id, run_html_url, self.secrets.software_name, - self.secrets.software_version, pass_verification, self.secrets.bot_token, self.secrets.dry_run) - logging.info(f"PR{pr_number} Dry Run - send sandbox notification to '{chart_owners}' about verification result of '{chart}'") + self.report_failure(chart,chart_owners,CHECKS_FAILED,pr_html_url,run_html_url) - if conclusion != 'success': logging.warning(f"PR{pr_number} workflow failed: {vendor_name}, {chart_name}, {chart_version}") return + else: + logging.info(f"PR{pr_number} workflow passed: {vendor_name}, {chart_name}, {chart_version}") else: logging.warning(f"PR{pr_number} workflow did not complete: {vendor_name}, {chart_name}, {chart_version}") return - logging.info(f"PR{pr_number} workflow passed: {vendor_name}, {chart_name}, {chart_version}") # Check PRs are merged if not super().check_pull_request_result(pr_number, True, logging.warning): @@ -891,13 +926,34 @@ def process_all_charts(self): owners_table = dict() pr_number_list = list() + skip_charts = list() + + logging.info(f"Running tests for : {self.secrets.software_name} {self.secrets.software_version} :") + # First look for charts in index.yaml to see if kubeVersion is good: + if self.secrets.software_name == "OpenShift": + logging.info("check index file for invalid kubeVersions") + failed_charts = check_index_entries(self.secrets.software_version) + if failed_charts: + for chart in failed_charts: + providerDir = chart["providerType"].replace("partner","partners") + chart_directory = f'charts/{providerDir}/{chart["provider"]}/{chart["name"]}' + self.get_owner_ids(chart_directory,owners_table) + chart_owners = owners_table[chart_directory] + chart_id = f'{chart["provider"]} {chart["name"]} {chart["version"]}' + self.report_failure(chart_id,chart_owners,chart["message"],"","") + skip_charts.append(f'{chart["name"]}-{chart["version"]}') + + # Process test charts and send PRs from temporary directory with SetDirectory(Path(self.temp_dir.name)): for vendor_type, vendor_name, chart_name, chart_version in self.secrets.submitted_charts: - logging.info(f"Process chart: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}") - self.process_single_chart(vendor_type, vendor_name, chart_name, chart_version, pr_number_list, owners_table) - logging.info("sleep for 5 seconds to avoid secondary api limit") - time.sleep(5) + if f'{chart_name}-{chart_version}' in skip_charts: + logging.info(f"Skip already failed chart: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}") + else: + logging.info(f"Process chart: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}") + self.process_single_chart(vendor_type, vendor_name, chart_name, chart_version, pr_number_list, owners_table) + logging.info("sleep for 5 seconds to avoid secondary api limit") + time.sleep(5) for vendor_type, vendor_name, chart_name, chart_version, pr_number in pr_number_list: logging.info(f"PR{pr_number} Check result: {vendor_type}, {vendor_name}, {chart_name}, {chart_version}") diff --git a/tests/functional/utils/github.py b/tests/functional/utils/github.py index c815381a10..73a816d5c8 100644 --- a/tests/functional/utils/github.py +++ b/tests/functional/utils/github.py @@ -10,11 +10,8 @@ @retry(stop_max_delay=30_000, wait_fixed=1000) def get_run_id(secrets, pr_number=None): - pr_number = secrets.pr_number if pr_number is None else pr_number - r = github_api( - 'post', f'repos/{secrets.test_repo}/pulls/{pr_number}', secrets.bot_token) - pr = json.loads(r.text) + pr = get_pr(secrets, pr_number) r = github_api( 'get', f'repos/{secrets.test_repo}/actions/runs', secrets.bot_token) runs = json.loads(r.text) @@ -26,7 +23,7 @@ def get_run_id(secrets, pr_number=None): raise Exception("Workflow for the submitted PR did not run.") -@retry(stop_max_delay=60_000*10, wait_fixed=2000) +@retry(stop_max_delay=60_000*40, wait_fixed=2000) def get_run_result(secrets, run_id): r = github_api( 'get', f'repos/{secrets.test_repo}/actions/runs/{run_id}', secrets.bot_token) @@ -62,6 +59,15 @@ def get_release_by_tag(secrets, release_tag): return release raise Exception("Release not published") + +def get_pr(secrets, pr_number=None): + pr_number = secrets.pr_number if pr_number is None else pr_number + r = github_api( + 'post', f'repos/{secrets.test_repo}/pulls/{pr_number}', secrets.bot_token) + pr = json.loads(r.text) + return pr + + def github_api_get(endpoint, bot_token, headers={}): if not headers: headers = {'Accept': 'application/vnd.github.v3+json', diff --git a/tests/functional/utils/index.py b/tests/functional/utils/index.py new file mode 100644 index 0000000000..8ca0148862 --- /dev/null +++ b/tests/functional/utils/index.py @@ -0,0 +1,43 @@ + +import logging +import semantic_version +import sys + +sys.path.append('../../../scripts/src') +from chartrepomanager import indexannotations +from indexfile import index + + + +def check_index_entries(ocpVersion): + + all_chart_list = index.get_latest_charts() + failed_chart_list = [] + + OCP_VERSION = semantic_version.Version.coerce(ocpVersion) + + for chart in all_chart_list: + if "supportedOCP" in chart and chart["supportedOCP"] != "N/A" and chart["supportedOCP"] != "": + if OCP_VERSION in semantic_version.NpmSpec(chart["supportedOCP"]): + logging.info(f'PASS: Chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} includes: {OCP_VERSION}') + else: + chart["message"] = f'chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} does not include latest OCP version {OCP_VERSION}' + logging.info(f' ERROR: Chart {chart["name"]} {chart["version"]} supported OCP version {chart["supportedOCP"]} does not include {OCP_VERSION}') + failed_chart_list.append(chart) + elif "kubeVersion" in chart and chart["kubeVersion"] != "": + supportedOCPVersion = indexannotations.getOCPVersions(chart["kubeVersion"]) + if OCP_VERSION in semantic_version.NpmSpec(supportedOCPVersion): + logging.info(f'PASS: Chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) includes OCP version: {OCP_VERSION}') + else: + chart["message"] = f'chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) does not include latest OCP version {OCP_VERSION}' + logging.info(f' ERROR: Chart {chart["name"]} {chart["version"]} kubeVersion {chart["kubeVersion"]} (OCP: {supportedOCPVersion}) does not include {OCP_VERSION}') + failed_chart_list.append(chart) + + return failed_chart_list + + + + + + + diff --git a/tests/functional/utils/notifier.py b/tests/functional/utils/notifier.py index 9adeebc7c5..d44279e795 100755 --- a/tests/functional/utils/notifier.py +++ b/tests/functional/utils/notifier.py @@ -11,6 +11,7 @@ endpoint_data = {} +CHECKS_FAILED = "checks failed" def _set_endpoint_key(key, env_var): if key not in endpoint_data: @@ -77,35 +78,39 @@ def _verify_endpoint(access_token): endpoint_data["access_token"] = access_token -def create_verification_issue(chart_name, chart_owners, notify_developers, report_url, software_name, software_version, pass_verification, access_token=None, dry_run=False): +def create_verification_issue(chart, chart_owners, failure_type, notify_developers, pr_url, report_url, software_name, software_version, access_token=None, dry_run=False): """Create and issue with chart-verifier findings after a version change trigger. chart_name -- Name of the chart that was verified. Include version for more verbose information\n chart_owners -- Github IDs of the chart owners\n - report_url -- URL or the report resulting from verification\n + failure_type - Indication of the type of failure + report_url -- URL or the report resulting from verification if applicable\n + kube-version -- The kubeVersion attribute of the chart if it is bade.\n software_name -- Name of the software dependency that changed e.g, OCP and Chart Verifier\n software_version -- The softwared dependency version used\n - pass_verification -- A boolean indicating whether the verification passed\n - access_token -- An optional github access token secret. If not passed will try to get from GITHUB_AUTH_TOKEN environment variable\n + access_token -- An optional github access token secret. If not passed will try to get from GITHUB_AUTH_TOKEN environment variable\ + dry-run -- Set if the test run is a dry-run. """ - if not pass_verification: - title = f"Chart {chart_name}" - if dry_run: - title = f"Dry Run: Chart {chart_name}" - + title = f"Chart {chart}" + if dry_run: + title = f"Dry Run: Chart {chart}" + if failure_type == CHECKS_FAILED: title = f"{title} has failures with {software_name} version {software_version}" report_result = "some chart checks have failed. Please review the failures and, if required, consider submitting a new chart version with the appropriate additions/corrections." + body = (f"FYI @{' @'.join(notify_developers)}, in PR {pr_url} we triggered the chart certification workflow against chart {chart} because the workflow " + f"now supports {software_name} version {software_version}. We have found that {report_result}. Check details in the report: " + f"{report_url}, Chart owners are: {chart_owners}") + else: + title = f"{title} does not support {software_name} version {software_version}" + body = (f"FYI @{' @'.join(notify_developers)}, we checked the OCP versions supported by {chart} because the workflow " + f"now supports {software_name} version {software_version}. We have found that {failure_type}. Chart owners are: {chart_owners}") - body = (f"FYI @{' @'.join(notify_developers)}, we have triggered the chart certification workflow against chart {chart_name} because the workflow " - f"now supports {software_name} version {software_version}. We have found that {report_result}. Check details in the report: " - f"{report_url}, Chart owners are: {chart_owners}") - - _set_endpoint() - _verify_endpoint(access_token) - create_an_issue(title, body) + _set_endpoint() + _verify_endpoint(access_token) + create_an_issue(title, body)