chr-hertel · chr-hertel · Aug 17, 2026
@@ -1,9 +1,9 @@
 name: conformance-weekly
 
-# Runs the MCP conformance suite weekly against the latest
-# @modelcontextprotocol/conformance release. The on:pull_request pipeline
-# pins to whatever version is available at PR time; this schedule catches
-# upstream releases that add scenarios between PRs.
+# Runs the MCP conformance suite weekly against the newest published
+# @modelcontextprotocol/conformance. The on:pull_request pipeline pins an exact
+# version; this schedule catches upstream releases that add scenarios between
+# PRs, so the pin never silently goes stale.
 #
 # It also scores each run and publishes the client/server pass-rate as
 # shields.io endpoint JSON to the orphan `badges` branch (consumed by the
@@ -17,10 +17,26 @@ permissions:
   contents: write
   issues: write
 
+# The PR pipeline pins an exact conformance version so a PR only goes red for
+# reasons in the PR. That pin is only safe because this job tracks the moving
+# target instead — on both revisions, since each rides a different dist-tag:
+# the dated one is on `latest`, the draft one only on `alpha`.
 jobs:
   server:
-    name: conformance / server (latest)
+    name: conformance / server (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # Both legs hit the same URL: one fixture, built through
+          # Builder::build(), serves whichever era the request claims.
+          - spec-version: '2025-11-25'
+            dist-tag: latest
+            baseline: conformance-baseline.yml
+          - spec-version: '2026-07-28'
+            dist-tag: alpha
+            baseline: conformance-baseline-2026-07-28.yml
     steps:
       - uses: actions/checkout@v7
       - uses: actions/setup-node@v7
@@ -35,9 +51,17 @@ jobs:
           sleep 5
       - name: Run conformance tests
         working-directory: ./tests/Conformance
-        run: npx --yes @modelcontextprotocol/conformance@latest server --url http://localhost:8000/ --expected-failures conformance-baseline.yml --output-dir results
+        run: >-
+          npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} server
+          --url http://localhost:8000/
+          --suite all
+          --spec-version ${{ matrix.spec-version }}
+          --expected-failures ${{ matrix.baseline }}
+          --output-dir results
+      # The badge tracks the released revision, so only the dated entry feeds
+      # it; the draft entry runs purely to catch upstream drift.
       - name: Generate score badge
-        if: always()
+        if: always() && matrix.spec-version == '2025-11-25'
         run: php tests/Conformance/score.php server
       - name: Show docker logs on failure
         if: failure()
@@ -46,20 +70,30 @@ jobs:
         if: failure()
         uses: actions/upload-artifact@v7
         with:
-          name: conformance-server-results
+          name: conformance-server-results-${{ matrix.spec-version }}
           path: |
             tests/Conformance/logs
             tests/Conformance/results
       - name: Upload score badge
-        if: always()
+        if: always() && matrix.spec-version == '2025-11-25'
         uses: actions/upload-artifact@v7
         with:
           name: server-badge
           path: tests/Conformance/server-conformance.json
 
   client:
-    name: conformance / client (latest)
+    name: conformance / client (${{ matrix.spec-version }}, ${{ matrix.dist-tag }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - spec-version: '2025-11-25'
+            dist-tag: latest
+            baseline: conformance-baseline.yml
+          - spec-version: '2026-07-28'
+            dist-tag: alpha
+            baseline: conformance-baseline-2026-07-28.yml
     steps:
       - uses: actions/checkout@v7
       - uses: shivammathur/setup-php@v2
@@ -73,20 +107,26 @@ jobs:
       - run: mkdir -p tests/Conformance/logs
       - name: Run conformance tests
         working-directory: ./tests/Conformance
-        run: npx --yes @modelcontextprotocol/conformance@latest client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml --output-dir results
+        run: >-
+          npx --yes @modelcontextprotocol/conformance@${{ matrix.dist-tag }} client
+          --command "php ${{ github.workspace }}/tests/Conformance/client.php"
+          --suite all
+          --spec-version ${{ matrix.spec-version }}
+          --expected-failures ${{ matrix.baseline }}
+          --output-dir results
       - name: Generate score badge
-        if: always()
+        if: always() && matrix.spec-version == '2025-11-25'
         run: php tests/Conformance/score.php client
       - name: Upload conformance results
         if: failure()
         uses: actions/upload-artifact@v7
         with:
-          name: conformance-client-results
+          name: conformance-client-results-${{ matrix.spec-version }}
           path: |
             tests/Conformance/logs
             tests/Conformance/results
       - name: Upload score badge
-        if: always()
+        if: always() && matrix.spec-version == '2025-11-25'
         uses: actions/upload-artifact@v7
         with:
           name: client-badge

@@ -5,6 +5,12 @@ permissions:
   contents: read
   pull-requests: write
 
+env:
+  # Pinned so a PR only goes red for reasons in the PR. The 2026-07-28
+  # scenarios ship on the `alpha` dist-tag; `latest` (0.1.x) has none of them.
+  # conformance-weekly.yaml is what tracks upstream releases.
+  CONFORMANCE_PKG: "@modelcontextprotocol/conformance@0.2.0-alpha.11"
+
 jobs:
   unit:
     runs-on: ubuntu-latest
@@ -95,8 +101,24 @@ jobs:
         run: vendor/bin/phpunit --testsuite=inspector
 
   conformance-server:
-    name: conformance / server
+    name: conformance / server (${{ matrix.spec-version }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      # One entry per lifecycle, not per spec revision: `--spec-version` is
+      # cumulative across the dated revisions, so 2025-11-25 already covers
+      # 2025-06-18 and 2025-03-26. 2026-07-28 is not cumulative with them — it
+      # is the SEP-2575 stateless wire, served from its own endpoint with its
+      # own baseline, so it needs its own run.
+      matrix:
+        include:
+          - spec-version: '2025-11-25'
+            path: '/'
+            baseline: conformance-baseline.yml
+          - spec-version: '2026-07-28'
+            path: '/stateless'
+            baseline: conformance-baseline-2026-07-28.yml
+
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -118,7 +140,12 @@ jobs:
 
       - name: Run conformance tests
         working-directory: ./tests/Conformance
-        run: npx @modelcontextprotocol/conformance server --url http://localhost:8000/ --expected-failures conformance-baseline.yml
+        run: >-
+          npx --yes ${{ env.CONFORMANCE_PKG }} server
+          --url http://localhost:8000${{ matrix.path }}
+          --suite all
+          --spec-version ${{ matrix.spec-version }}
+          --expected-failures ${{ matrix.baseline }}
 
       - name: Show logs on failure
         if: failure()
@@ -146,8 +173,17 @@ jobs:
         run: docker compose -f tests/Conformance/Fixtures/docker-compose.yml down
 
   conformance-client:
-    name: conformance / client
+    name: conformance / client (${{ matrix.spec-version }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - spec-version: '2025-11-25'
+            baseline: conformance-baseline.yml
+          - spec-version: '2026-07-28'
+            baseline: conformance-baseline-2026-07-28.yml
+
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -171,7 +207,12 @@ jobs:
 
       - name: Run client conformance tests
         working-directory: ./tests/Conformance
-        run: npx @modelcontextprotocol/conformance client --command "php ${{ github.workspace }}/tests/Conformance/client.php" --suite all --expected-failures conformance-baseline.yml
+        run: >-
+          npx --yes ${{ env.CONFORMANCE_PKG }} client
+          --command "php ${{ github.workspace }}/tests/Conformance/client.php"
+          --suite all
+          --spec-version ${{ matrix.spec-version }}
+          --expected-failures ${{ matrix.baseline }}
 
       - name: Show logs on failure
         if: failure()

@@ -8,7 +8,7 @@ examples/**/cache
 examples/**/sessions
 tests/Conformance/client-conformance.json
 tests/Conformance/server-conformance.json
-tests/Conformance/results
+tests/Conformance/results*
 tests/Conformance/sessions
 tests/Conformance/logs/*.log