From d9113f8f60f1c4a45264ac34ff6f446b11395d01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:02:44 +0200 Subject: [PATCH 01/17] testing gitlab ci translated to github actions --- .github/workflows/main.yml | 214 +++++++++++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..ef64c9f --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,214 @@ +name: 'testing ${{ env.TESTED_PARSER }}-parser on ${{ env.TESTED_BRANCH }} branch' + +# This workflow is triggered on pushes, pull requests, or can be run manually. +on: + push: + branches: [ main, master ] # Adjust to your default branch + pull_request: + branches: [ main, master ] # Adjust to your default branch + workflow_dispatch: + +# Environment variables available to all jobs in the workflow +env: + TESTED_BRANCH: master + TESTED_PARSER: jsoniq + +jobs: + build: + name: Build + runs-on: ubuntu-latest + steps: + - name: Set up JDK 11 and Maven + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + maven-version: '3.9.9' + + - name: Clone and Build Rumble Repository + run: | + echo "cloning ${{ env.TESTED_BRANCH }}" + git clone --single-branch --branch "${{ env.TESTED_BRANCH }}" "https://github.com/gfourny/rumble.git" + cd rumble + mvn clean compile assembly:single -quiet + cd .. + + - name: Upload build artifact + uses: actions/upload-artifact@v4 + with: + name: rumble-build + path: rumble/target + + test: + name: Test - ${{ matrix.test_name }} + needs: build + runs-on: ubuntu-latest + # Use a matrix to run all test jobs in parallel + strategy: + fail-fast: false # Allows all tests to finish even if one fails + matrix: + test_name: [ 'app', 'array', 'fn1', 'fn2', 'map', 'math', 'misc', 'op', 'prod1', 'prod2', 'ser', 'xs' ] + + # The job will be marked with a warning but won't fail the workflow. + continue-on-error: true + + steps: + - name: Set up JDK 11 and Maven + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + maven-version: '3.9.9' + + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: rumble-build + path: . # download to the root + + - name: Run Maven Tests + id: run-tests + run: | + # Capitalize the first letter of the test_name to form the class name + TEST_CLASS=$(echo ${{ matrix.test_name }} | awk '{print toupper(substr($0,1,1))substr($0,2)}')Test + + if [ "${{ env.TESTED_PARSER }}" = "jsoniq" ]; then + mvn -Dtest=${TEST_CLASS} test -quiet + else + mvn -Dtest=XQuery${TEST_CLASS} test -quiet + fi + + - name: Upload Surefire reports + # Corresponds to `when: always` + if: always() + uses: actions/upload-artifact@v4 + with: + name: surefire-report-${{ matrix.test_name }} + path: target/surefire-reports/*.xml + retention-days: 7 + + spotless-check: + name: Spotless Check + needs: build + runs-on: ubuntu-latest + steps: + - name: Set up JDK 11 and Maven + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + maven-version: '3.9.9' + + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: rumble-build + path: . + + - name: Run Spotless Check + run: mvn spotless:check + + publish-test-results: + name: Publish Test Results + # This job runs after all jobs in the `test` matrix have completed. + needs: test + runs-on: ubuntu-latest + # This job should always run to report on which tests passed or failed. + if: always() + steps: + - name: Download all surefire reports + uses: actions/download-artifact@v4 + with: + # Download all artifacts that match this pattern + pattern: surefire-report-* + path: all-surefire-reports + merge-multiple: true # merge all downloaded artifacts into one directory + + - name: Publish Unit Test Results + uses: EnricoMi/publish-unit-test-results-action@v2 + with: + files: all-surefire-reports/**/*.xml + + collect: + name: Collect Artifacts + needs: [test, spotless-check] + runs-on: ubuntu-latest + # Corresponds to `when: always` + if: always() + steps: + # Checkout the repository to get access to analytics scripts etc. + - uses: actions/checkout@v4 + + - name: Set up JDK 11 and Maven + uses: actions/setup-java@v4 + with: + java-version: '11' + distribution: 'temurin' + maven-version: '3.9.9' + + - name: Download build artifact + uses: actions/download-artifact@v4 + with: + name: rumble-build + path: . + + - name: Download all surefire reports + uses: actions/download-artifact@v4 + with: + pattern: surefire-report-* + path: all-surefire-reports + merge-multiple: true + + - name: Collect and process artifacts + run: | + mkdir -p collected-artifacts/surefire-reports + # Copy all downloaded reports into the target directory for processing + cp all-surefire-reports/*.xml target/surefire-reports/ + cp -r target/surefire-reports collected-artifacts/ + + # Run the collection/analytics Java code + mvn compile + mvn exec:java + + - name: Upload collected data + uses: actions/upload-artifact@v4 + with: + name: collected-data + path: | + collected-artifacts + analytics-results + + plot: + name: Analytics and Plotting + needs: collect + runs-on: ubuntu-latest + # Corresponds to `when: always` + if: always() + steps: + # Checkout the repository to get the analytics scripts + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Download collected data + uses: actions/download-artifact@v4 + with: + name: collected-data + path: . + + - name: Install dependencies and run plotting script + run: | + pip install -r analytics/requirements.txt + python analytics/plot.py + echo "TO SEE THE PLOTS, CHECK THE ARTIFACTS OF THIS WORKFLOW RUN" + + - name: Upload plots + uses: actions/upload-artifact@v4 + with: + name: plots-and-results + path: | + plots + analytics-results From b4152766f1b097e459a491f48d517f10b335fe94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:04:49 +0200 Subject: [PATCH 02/17] Update main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ef64c9f..3532cc0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: 'testing ${{ env.TESTED_PARSER }}-parser on ${{ env.TESTED_BRANCH }} branch' +name: 'testing TODO-parser on TODO branch' # This workflow is triggered on pushes, pull requests, or can be run manually. on: From 6868798a8e51822aa2dbaa2b8409f830a858753e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:08:41 +0200 Subject: [PATCH 03/17] Update main.yml --- .github/workflows/main.yml | 74 ++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 39 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 3532cc0..e991303 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,53 +18,55 @@ jobs: name: Build runs-on: ubuntu-latest steps: - - name: Set up JDK 11 and Maven + # This is the correct way to check out your repository code + - name: Checkout Repository + uses: actions/checkout@v4 + with: + # If you truly need to build a different branch than the one triggering the workflow + ref: ${{ env.TESTED_BRANCH }} + + # The 'maven-version' input is no longer valid and has been removed. + - name: Set up JDK 11 uses: actions/setup-java@v4 with: java-version: '11' distribution: 'temurin' - maven-version: '3.9.9' + # Cache Maven dependencies for faster builds + cache: 'maven' - - name: Clone and Build Rumble Repository - run: | - echo "cloning ${{ env.TESTED_BRANCH }}" - git clone --single-branch --branch "${{ env.TESTED_BRANCH }}" "https://github.com/gfourny/rumble.git" - cd rumble - mvn clean compile assembly:single -quiet - cd .. + - name: Build with Maven + run: mvn clean compile assembly:single -quiet - name: Upload build artifact uses: actions/upload-artifact@v4 with: - name: rumble-build - path: rumble/target + name: rumble-target + path: target test: name: Test - ${{ matrix.test_name }} needs: build runs-on: ubuntu-latest - # Use a matrix to run all test jobs in parallel strategy: - fail-fast: false # Allows all tests to finish even if one fails + fail-fast: false matrix: test_name: [ 'app', 'array', 'fn1', 'fn2', 'map', 'math', 'misc', 'op', 'prod1', 'prod2', 'ser', 'xs' ] - # The job will be marked with a warning but won't fail the workflow. continue-on-error: true steps: - - name: Set up JDK 11 and Maven + - name: Set up JDK 11 uses: actions/setup-java@v4 with: java-version: '11' distribution: 'temurin' - maven-version: '3.9.9' + cache: 'maven' - name: Download build artifact uses: actions/download-artifact@v4 with: - name: rumble-build - path: . # download to the root + name: rumble-target + path: target - name: Run Maven Tests id: run-tests @@ -79,7 +81,6 @@ jobs: fi - name: Upload Surefire reports - # Corresponds to `when: always` if: always() uses: actions/upload-artifact@v4 with: @@ -92,37 +93,34 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - name: Set up JDK 11 and Maven + - name: Set up JDK 11 uses: actions/setup-java@v4 with: java-version: '11' distribution: 'temurin' - maven-version: '3.9.9' + cache: 'maven' - name: Download build artifact uses: actions/download-artifact@v4 with: - name: rumble-build - path: . + name: rumble-target + path: target - name: Run Spotless Check run: mvn spotless:check publish-test-results: name: Publish Test Results - # This job runs after all jobs in the `test` matrix have completed. needs: test runs-on: ubuntu-latest - # This job should always run to report on which tests passed or failed. if: always() steps: - name: Download all surefire reports uses: actions/download-artifact@v4 with: - # Download all artifacts that match this pattern pattern: surefire-report-* path: all-surefire-reports - merge-multiple: true # merge all downloaded artifacts into one directory + merge-multiple: true - name: Publish Unit Test Results uses: EnricoMi/publish-unit-test-results-action@v2 @@ -133,24 +131,22 @@ jobs: name: Collect Artifacts needs: [test, spotless-check] runs-on: ubuntu-latest - # Corresponds to `when: always` if: always() steps: - # Checkout the repository to get access to analytics scripts etc. - - uses: actions/checkout@v4 - - - name: Set up JDK 11 and Maven + - uses: actions/checkout@v4 # Checkout needed for analytics scripts + + - name: Set up JDK 11 uses: actions/setup-java@v4 with: java-version: '11' distribution: 'temurin' - maven-version: '3.9.9' + cache: 'maven' - name: Download build artifact uses: actions/download-artifact@v4 with: - name: rumble-build - path: . + name: rumble-target + path: target - name: Download all surefire reports uses: actions/download-artifact@v4 @@ -162,7 +158,9 @@ jobs: - name: Collect and process artifacts run: | mkdir -p collected-artifacts/surefire-reports - # Copy all downloaded reports into the target directory for processing + # The download artifact action now places surefire reports in the correct directory + # so we just need to ensure the target/surefire-reports dir exists for the next step + mkdir -p target/surefire-reports cp all-surefire-reports/*.xml target/surefire-reports/ cp -r target/surefire-reports collected-artifacts/ @@ -182,11 +180,9 @@ jobs: name: Analytics and Plotting needs: collect runs-on: ubuntu-latest - # Corresponds to `when: always` if: always() steps: - # Checkout the repository to get the analytics scripts - - uses: actions/checkout@v4 + - uses: actions/checkout@v4 # Checkout needed for analytics scripts - name: Set up Python 3.12 uses: actions/setup-python@v5 From 7518e7bb07e76e58dacfb3bbce8cf7c82d7e20ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:11:47 +0200 Subject: [PATCH 04/17] checkout correct repo --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e991303..bc5d61f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,7 +22,7 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: - # If you truly need to build a different branch than the one triggering the workflow + repository: gfourny/rumble ref: ${{ env.TESTED_BRANCH }} # The 'maven-version' input is no longer valid and has been removed. From 02e0f034b4c16838465b8a628a473ed4c0677008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:12:59 +0200 Subject: [PATCH 05/17] Update main.yml --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bc5d61f..0bb2091 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -22,8 +22,9 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 with: - repository: gfourny/rumble + repository: RumbleDB/rumble ref: ${{ env.TESTED_BRANCH }} + path: rumble # The 'maven-version' input is no longer valid and has been removed. - name: Set up JDK 11 From 39c9995691de1e9fe27311a1f26eb2f470bf4189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:23:21 +0200 Subject: [PATCH 06/17] Update main.yml --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0bb2091..2e3e96f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,13 +36,14 @@ jobs: cache: 'maven' - name: Build with Maven + working-directory: ./rumble run: mvn clean compile assembly:single -quiet - name: Upload build artifact uses: actions/upload-artifact@v4 with: name: rumble-target - path: target + path: rumble/target test: name: Test - ${{ matrix.test_name }} From 8ca86a1177befb388a4e390b0d78ca609269696b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:26:31 +0200 Subject: [PATCH 07/17] jdk 17 --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2e3e96f..6ea7fcf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,10 +27,10 @@ jobs: path: rumble # The 'maven-version' input is no longer valid and has been removed. - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' # Cache Maven dependencies for faster builds cache: 'maven' From 6c32294fb05ad01fb975c15d143cb750e578a265 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:32:55 +0200 Subject: [PATCH 08/17] Update main.yml --- .github/workflows/main.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6ea7fcf..7e82403 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,7 +37,7 @@ jobs: - name: Build with Maven working-directory: ./rumble - run: mvn clean compile assembly:single -quiet + run: mvn clean compile assembly:single - name: Upload build artifact uses: actions/upload-artifact@v4 @@ -57,10 +57,10 @@ jobs: continue-on-error: true steps: - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' cache: 'maven' @@ -95,10 +95,10 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - name: Set up JDK 11 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' cache: 'maven' From fc55cd6f05f176b1b5bf278cd1955f2cc999bc99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:41:33 +0200 Subject: [PATCH 09/17] Update main.yml --- .github/workflows/main.yml | 75 +++++++++++++++++++------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7e82403..0a3ce2b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,44 +1,40 @@ -name: 'testing TODO-parser on TODO branch' +name: 'Test RumbleDB Parser' -# This workflow is triggered on pushes, pull requests, or can be run manually. on: push: - branches: [ main, master ] # Adjust to your default branch + branches: [ main, master ] pull_request: - branches: [ main, master ] # Adjust to your default branch + branches: [ main, master ] workflow_dispatch: -# Environment variables available to all jobs in the workflow env: TESTED_BRANCH: master TESTED_PARSER: jsoniq jobs: build: - name: Build + name: Build Rumble runs-on: ubuntu-latest steps: - # This is the correct way to check out your repository code - - name: Checkout Repository + - name: Checkout Rumble Repository uses: actions/checkout@v4 with: repository: RumbleDB/rumble ref: ${{ env.TESTED_BRANCH }} path: rumble - # The 'maven-version' input is no longer valid and has been removed. - name: Set up JDK 17 uses: actions/setup-java@v4 with: java-version: '17' distribution: 'temurin' - # Cache Maven dependencies for faster builds - cache: 'maven' - + # NOTE: Caching is removed here because the pom.xml is in a subdirectory ('rumble'), + # which the cache action cannot find at the root level. + - name: Build with Maven working-directory: ./rumble run: mvn clean compile assembly:single - + - name: Upload build artifact uses: actions/upload-artifact@v4 with: @@ -53,15 +49,19 @@ jobs: fail-fast: false matrix: test_name: [ 'app', 'array', 'fn1', 'fn2', 'map', 'math', 'misc', 'op', 'prod1', 'prod2', 'ser', 'xs' ] - + continue-on-error: true - steps: + # FIX: Added checkout step for the test-suite repository itself. + - name: Checkout Test Suite Repository + uses: actions/checkout@v4 + - name: Set up JDK 17 uses: actions/setup-java@v4 with: java-version: '17' distribution: 'temurin' + # Caching will now work because checkout places the pom.xml at the root. cache: 'maven' - name: Download build artifact @@ -73,9 +73,7 @@ jobs: - name: Run Maven Tests id: run-tests run: | - # Capitalize the first letter of the test_name to form the class name TEST_CLASS=$(echo ${{ matrix.test_name }} | awk '{print toupper(substr($0,1,1))substr($0,2)}')Test - if [ "${{ env.TESTED_PARSER }}" = "jsoniq" ]; then mvn -Dtest=${TEST_CLASS} test -quiet else @@ -95,20 +93,23 @@ jobs: needs: build runs-on: ubuntu-latest steps: + # FIX: Corrected logic to check out source code, not download an artifact. + - name: Checkout Rumble Repository + uses: actions/checkout@v4 + with: + repository: RumbleDB/rumble + ref: ${{ env.TESTED_BRANCH }} + path: rumble + - name: Set up JDK 17 uses: actions/setup-java@v4 with: java-version: '17' distribution: 'temurin' - cache: 'maven' - - - name: Download build artifact - uses: actions/download-artifact@v4 - with: - name: rumble-target - path: target + # NOTE: Caching is also removed here because pom.xml is in a subdirectory. - name: Run Spotless Check + working-directory: ./rumble run: mvn spotless:check publish-test-results: @@ -135,12 +136,13 @@ jobs: runs-on: ubuntu-latest if: always() steps: - - uses: actions/checkout@v4 # Checkout needed for analytics scripts - - - name: Set up JDK 11 + - name: Checkout Test Suite Repository + uses: actions/checkout@v4 + + - name: Set up JDK 17 uses: actions/setup-java@v4 with: - java-version: '11' + java-version: '17' distribution: 'temurin' cache: 'maven' @@ -149,7 +151,7 @@ jobs: with: name: rumble-target path: target - + - name: Download all surefire reports uses: actions/download-artifact@v4 with: @@ -159,14 +161,10 @@ jobs: - name: Collect and process artifacts run: | - mkdir -p collected-artifacts/surefire-reports - # The download artifact action now places surefire reports in the correct directory - # so we just need to ensure the target/surefire-reports dir exists for the next step + mkdir -p collected-artifacts mkdir -p target/surefire-reports cp all-surefire-reports/*.xml target/surefire-reports/ cp -r target/surefire-reports collected-artifacts/ - - # Run the collection/analytics Java code mvn compile mvn exec:java @@ -184,8 +182,9 @@ jobs: runs-on: ubuntu-latest if: always() steps: - - uses: actions/checkout@v4 # Checkout needed for analytics scripts - + - name: Checkout Test Suite Repository + uses: actions/checkout@v4 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: @@ -196,13 +195,13 @@ jobs: with: name: collected-data path: . - + - name: Install dependencies and run plotting script run: | pip install -r analytics/requirements.txt python analytics/plot.py echo "TO SEE THE PLOTS, CHECK THE ARTIFACTS OF THIS WORKFLOW RUN" - + - name: Upload plots uses: actions/upload-artifact@v4 with: From d93215c3853d970d9f67ae97dbd682b607984713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Thu, 25 Sep 2025 20:45:45 +0200 Subject: [PATCH 10/17] Update main.yml --- .github/workflows/main.yml | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0a3ce2b..20df60b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,8 +28,6 @@ jobs: with: java-version: '17' distribution: 'temurin' - # NOTE: Caching is removed here because the pom.xml is in a subdirectory ('rumble'), - # which the cache action cannot find at the root level. - name: Build with Maven working-directory: ./rumble @@ -52,7 +50,6 @@ jobs: continue-on-error: true steps: - # FIX: Added checkout step for the test-suite repository itself. - name: Checkout Test Suite Repository uses: actions/checkout@v4 @@ -61,14 +58,18 @@ jobs: with: java-version: '17' distribution: 'temurin' - # Caching will now work because checkout places the pom.xml at the root. cache: 'maven' - - name: Download build artifact + # --- START OF FIX --- + - name: Create directory structure for system-scoped dependency + run: mkdir -p rumble/target + + - name: Download build artifact into the correct path uses: actions/download-artifact@v4 with: name: rumble-target - path: target + path: rumble/target # This path now matches what the pom.xml expects + # --- END OF FIX --- - name: Run Maven Tests id: run-tests @@ -93,7 +94,6 @@ jobs: needs: build runs-on: ubuntu-latest steps: - # FIX: Corrected logic to check out source code, not download an artifact. - name: Checkout Rumble Repository uses: actions/checkout@v4 with: @@ -106,7 +106,6 @@ jobs: with: java-version: '17' distribution: 'temurin' - # NOTE: Caching is also removed here because pom.xml is in a subdirectory. - name: Run Spotless Check working-directory: ./rumble @@ -146,11 +145,16 @@ jobs: distribution: 'temurin' cache: 'maven' - - name: Download build artifact + # --- START OF FIX (Applied here as well) --- + - name: Create directory structure for system-scoped dependency + run: mkdir -p rumble/target + + - name: Download build artifact into the correct path uses: actions/download-artifact@v4 with: name: rumble-target - path: target + path: rumble/target # This path now matches what the pom.xml expects + # --- END OF FIX --- - name: Download all surefire reports uses: actions/download-artifact@v4 @@ -162,8 +166,9 @@ jobs: - name: Collect and process artifacts run: | mkdir -p collected-artifacts - mkdir -p target/surefire-reports - cp all-surefire-reports/*.xml target/surefire-reports/ + # This next line is important: Maven expects the surefire reports in target/, not the root. + mkdir -p target + mv all-surefire-reports target/surefire-reports cp -r target/surefire-reports collected-artifacts/ mvn compile mvn exec:java From a0fe5ab7bc7614ded10d1e5527326d9f2d43f9ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Tue, 30 Sep 2025 19:34:39 +0200 Subject: [PATCH 11/17] change to rumble 2.0 --- .github/workflows/main.yml | 14 +++----------- pom.xml | 12 +++++------- 2 files changed, 8 insertions(+), 18 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20df60b..88ed7fb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -60,7 +60,6 @@ jobs: distribution: 'temurin' cache: 'maven' - # --- START OF FIX --- - name: Create directory structure for system-scoped dependency run: mkdir -p rumble/target @@ -68,8 +67,7 @@ jobs: uses: actions/download-artifact@v4 with: name: rumble-target - path: rumble/target # This path now matches what the pom.xml expects - # --- END OF FIX --- + path: rumble/target - name: Run Maven Tests id: run-tests @@ -94,12 +92,8 @@ jobs: needs: build runs-on: ubuntu-latest steps: - - name: Checkout Rumble Repository + - name: Checkout Test Suite Repository uses: actions/checkout@v4 - with: - repository: RumbleDB/rumble - ref: ${{ env.TESTED_BRANCH }} - path: rumble - name: Set up JDK 17 uses: actions/setup-java@v4 @@ -145,7 +139,6 @@ jobs: distribution: 'temurin' cache: 'maven' - # --- START OF FIX (Applied here as well) --- - name: Create directory structure for system-scoped dependency run: mkdir -p rumble/target @@ -153,8 +146,7 @@ jobs: uses: actions/download-artifact@v4 with: name: rumble-target - path: rumble/target # This path now matches what the pom.xml expects - # --- END OF FIX --- + path: rumble/target - name: Download all surefire reports uses: actions/download-artifact@v4 diff --git a/pom.xml b/pom.xml index d387c28..2a37f03 100644 --- a/pom.xml +++ b/pom.xml @@ -69,26 +69,26 @@ com.github.rumbledb rumbledb - 1.23.0 + 2.0.0 system - ${project.basedir}/rumble/target/rumbledb-1.23.0-jar-with-dependencies.jar + ${project.basedir}/rumble/target/rumbledb-2.0.0-jar-with-dependencies.jar org.apache.spark spark-core_2.13 - 3.5.1 + 4.0.1 compile org.apache.spark spark-sql_2.13 - 3.5.1 + 4.0.1 compile org.apache.spark spark-mllib_2.13 - 3.5.1 + 4.0.1 compile @@ -115,6 +115,4 @@ test - - \ No newline at end of file From f88672dc9d10f1c2308bfdb0b86b647928f56d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Tue, 30 Sep 2025 19:43:24 +0200 Subject: [PATCH 12/17] adapt to newer sequenceOfItemsAPI --- .github/workflows/main.yml | 1 - src/test/java/iq/base/TestBase.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 88ed7fb..8828089 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -102,7 +102,6 @@ jobs: distribution: 'temurin' - name: Run Spotless Check - working-directory: ./rumble run: mvn spotless:check publish-test-results: diff --git a/src/test/java/iq/base/TestBase.java b/src/test/java/iq/base/TestBase.java index dab0537..00dee97 100644 --- a/src/test/java/iq/base/TestBase.java +++ b/src/test/java/iq/base/TestBase.java @@ -95,7 +95,7 @@ private List runQuery(String query, Rumble rumble, Environment environment } SequenceOfItems queryResult = rumble.runQuery(query); List resultAsList = new ArrayList<>(); - queryResult.populateListWithWarningOnlyIfCapReached(resultAsList); + queryResult.populateList(resultAsList); return resultAsList; } From e1c494cd31b9e2f16a5d1601215e8efed1484a0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marco=20Sch=C3=B6b?= Date: Tue, 30 Sep 2025 19:49:38 +0200 Subject: [PATCH 13/17] proper populateList API --- .github/workflows/main.yml | 2 +- src/test/java/iq/base/TestBase.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8828089..d837bdb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -124,7 +124,7 @@ jobs: collect: name: Collect Artifacts - needs: [test, spotless-check] + needs: test runs-on: ubuntu-latest if: always() steps: diff --git a/src/test/java/iq/base/TestBase.java b/src/test/java/iq/base/TestBase.java index 00dee97..3ccb18c 100644 --- a/src/test/java/iq/base/TestBase.java +++ b/src/test/java/iq/base/TestBase.java @@ -95,7 +95,7 @@ private List runQuery(String query, Rumble rumble, Environment environment } SequenceOfItems queryResult = rumble.runQuery(query); List resultAsList = new ArrayList<>(); - queryResult.populateList(resultAsList); + queryResult.populateList(resultAsList, 1000000000000); return resultAsList; } From 5b6dc8b427ed8d78c2398a4c511f6155c499b52d Mon Sep 17 00:00:00 2001 From: mschoeb Date: Tue, 30 Sep 2025 20:56:03 +0200 Subject: [PATCH 14/17] adapt to new rumble --- .github/workflows/main.yml | 7 +++---- pom.xml | 6 ++++++ src/test/java/iq/base/TestBase.java | 2 +- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d837bdb..d5a2b10 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,7 +46,7 @@ jobs: strategy: fail-fast: false matrix: - test_name: [ 'app', 'array', 'fn1', 'fn2', 'map', 'math', 'misc', 'op', 'prod1', 'prod2', 'ser', 'xs' ] + test_name: [ 'App', 'Array', 'Fn1', 'Fn2', 'Map', 'Math', 'Misc', 'Op', 'Prod1', 'Prod2', 'Ser', 'Xs' ] continue-on-error: true steps: @@ -72,11 +72,10 @@ jobs: - name: Run Maven Tests id: run-tests run: | - TEST_CLASS=$(echo ${{ matrix.test_name }} | awk '{print toupper(substr($0,1,1))substr($0,2)}')Test if [ "${{ env.TESTED_PARSER }}" = "jsoniq" ]; then - mvn -Dtest=${TEST_CLASS} test -quiet + mvn -Dtest=${{ matrix.test_name }}Test test -quiet else - mvn -Dtest=XQuery${TEST_CLASS} test -quiet + mvn -Dtest=XQuery${{ matrix.test_name }}Test test -quiet fi - name: Upload Surefire reports diff --git a/pom.xml b/pom.xml index 2a37f03..88796d6 100644 --- a/pom.xml +++ b/pom.xml @@ -114,5 +114,11 @@ 4.13.2 test + + org.apache.spark + spark-hive_2.13 + 4.0.1 + provided + \ No newline at end of file diff --git a/src/test/java/iq/base/TestBase.java b/src/test/java/iq/base/TestBase.java index 3ccb18c..0f50280 100644 --- a/src/test/java/iq/base/TestBase.java +++ b/src/test/java/iq/base/TestBase.java @@ -95,7 +95,7 @@ private List runQuery(String query, Rumble rumble, Environment environment } SequenceOfItems queryResult = rumble.runQuery(query); List resultAsList = new ArrayList<>(); - queryResult.populateList(resultAsList, 1000000000000); + queryResult.populateList(resultAsList, 1000000000); return resultAsList; } From 9d90e3e0d1b5eedd0152fb88d4c8d1f5fbbb27b8 Mon Sep 17 00:00:00 2001 From: mschoeb Date: Tue, 30 Sep 2025 21:38:50 +0200 Subject: [PATCH 15/17] try test summary --- .github/workflows/main.yml | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d5a2b10..fbcde8b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -86,6 +86,12 @@ jobs: path: target/surefire-reports/*.xml retention-days: 7 + - name: Test Summary + uses: test-summary/action@v2 + with: + paths: "target/surefire-reports/*.xml" + if: always() + spotless-check: name: Spotless Check needs: build @@ -103,24 +109,6 @@ jobs: - name: Run Spotless Check run: mvn spotless:check - publish-test-results: - name: Publish Test Results - needs: test - runs-on: ubuntu-latest - if: always() - steps: - - name: Download all surefire reports - uses: actions/download-artifact@v4 - with: - pattern: surefire-report-* - path: all-surefire-reports - merge-multiple: true - - - name: Publish Unit Test Results - uses: EnricoMi/publish-unit-test-results-action@v2 - with: - files: all-surefire-reports/**/*.xml - collect: name: Collect Artifacts needs: test @@ -156,7 +144,6 @@ jobs: - name: Collect and process artifacts run: | mkdir -p collected-artifacts - # This next line is important: Maven expects the surefire reports in target/, not the root. mkdir -p target mv all-surefire-reports target/surefire-reports cp -r target/surefire-reports collected-artifacts/ @@ -194,9 +181,9 @@ jobs: - name: Install dependencies and run plotting script run: | pip install -r analytics/requirements.txt - python analytics/plot.py - echo "TO SEE THE PLOTS, CHECK THE ARTIFACTS OF THIS WORKFLOW RUN" - + python analytics/plot.py > script_output.txt + cat script_output.txt >> $GITHUB_STEP_SUMMARY + - name: Upload plots uses: actions/upload-artifact@v4 with: From 154689a98fd0a7a054b6f7080ffae719b3cf1bdc Mon Sep 17 00:00:00 2001 From: mschoeb Date: Tue, 30 Sep 2025 21:55:18 +0200 Subject: [PATCH 16/17] try different test summary --- .github/workflows/main.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index fbcde8b..f537a0b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,6 +90,7 @@ jobs: uses: test-summary/action@v2 with: paths: "target/surefire-reports/*.xml" + show: "none" if: always() spotless-check: @@ -158,6 +159,21 @@ jobs: collected-artifacts analytics-results + - name: Test Summary + uses: test-summary/action@v2 + with: + paths: "target/surefire-reports/*.xml" + show: "none" + if: always() + + - name: Test Report + uses: dorny/test-reporter@v2 + if: always() + with: + name: Tests # Name of the check run which will be created + path: target/surefire-reports/*.xml" # Path to test results + reporter: java-junit # Format of test results + plot: name: Analytics and Plotting needs: collect From c365ed54914d9d47f33c0206b552a6ea74226bd2 Mon Sep 17 00:00:00 2001 From: mschoeb Date: Tue, 30 Sep 2025 22:05:06 +0200 Subject: [PATCH 17/17] fix second report --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index f537a0b..f421883 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -171,7 +171,7 @@ jobs: if: always() with: name: Tests # Name of the check run which will be created - path: target/surefire-reports/*.xml" # Path to test results + path: "target/surefire-reports/*.xml" # Path to test results reporter: java-junit # Format of test results plot: