ci: migrate benchmarks to CodSpeed

.github/workflows/master_benchmarks.yml → .github/workflows/benchmarks.yml

@@ -1,8 +1,9 @@
-name: Master Benchmarks
+name: Benchmarks
 
 on:
   push:
     branches: master
+  pull_request:
   workflow_dispatch:
 
 concurrency:
@@ -14,9 +15,12 @@ env:
   php_version: "8.4"
 
 jobs:
-  run_benchmarks:
-    name: Run Master Benchmarks
+  benchmarks:
+    name: Run Benchmarks
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      id-token: write
     steps:
       - uses: actions/checkout@v7
       - name: Setup PHP
@@ -31,6 +35,10 @@ jobs:
           toolchain: stable
       - name: Cache cargo dependencies
         uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: |
+            benches
+            benches/ext
       - name: Cache LLVM and Clang
         id: cache-llvm
         uses: actions/cache@v6
@@ -46,29 +54,18 @@ jobs:
       - name: Configure Clang
         run: |
           echo "LIBCLANG_PATH=${{ runner.temp }}/llvm-${{ env.clang }}/lib" >> $GITHUB_ENV
-      - name: Install Valgrind
-        run: sudo apt-get update && sudo apt-get install -y valgrind
       - uses: taiki-e/install-action@cargo-binstall
-      - name: Install gungraun-runner
+      - name: Install cargo-codspeed
+        run: cargo binstall --no-confirm cargo-codspeed
+      - name: Build extension and benchmarks
+        working-directory: benches
         run: |
-          version=$(cargo metadata --manifest-path ./benches/Cargo.toml --format-version=1 |\
-            jq '.packages[] | select(.name == "gungraun").version' |\
-            tr -d '"'
-          )
-          cargo binstall --no-confirm --force gungraun-runner --version $version
-          echo "GUNGRAUN_RUNNER=$(which gungraun-runner)" >> $GITHUB_ENV
+          cargo build --manifest-path ext/Cargo.toml --release
+          cargo codspeed build
       - name: Run Benchmarks
-        run: cargo bench --manifest-path ./benches/Cargo.toml > benchmark_results.txt
-        env:
-          NO_COLOR: "1"
-      - uses: bencherdev/bencher@main
-      - name: Track Benchmarks with Bencher
-        run: |
-          bencher run \
-          --project ext-php-rs \
-          --token '${{ secrets.BENCHER_API_TOKEN }}' \
-          --branch master \
-          --testbed "$(php --version | head -n 1)" \
-          --err \
-          --adapter rust_gungraun \
-          --file "./benchmark_results.txt"
+        uses: CodSpeedHQ/action@v5
+        with:
+          mode: simulation
+          simulation-track-subprocess: true
+          working-directory: benches
+          run: cargo codspeed run

README.md

@@ -4,6 +4,7 @@
 [![docs.rs](https://img.shields.io/docsrs/ext-php-rs/latest)](https://docs.rs/ext-php-rs)
 [![Guide Workflow Status](https://img.shields.io/github/actions/workflow/status/extphprs/ext-php-rs/docs.yml?branch=master&label=guide)](https://ext-php.rs)
 ![CI Workflow Status](https://img.shields.io/github/actions/workflow/status/extphprs/ext-php-rs/master.yml?branch=master)
+[![CodSpeed Badge](https://img.shields.io/endpoint?url=https://app.codspeed.io//badge.json)](https://app.codspeed.io//extphprs/ext-php-rs?utm_source=badge)
 [![Coverage Status](https://coveralls.io/repos/github/extphprs/ext-php-rs/badge.svg?branch=master)](https://coveralls.io/github/extphprs/ext-php-rs?branch=master) [![Discord](https://img.shields.io/discord/115233111977099271)](https://discord.gg/dphp)
 
 Bindings and abstractions for the Zend API to build PHP extensions natively in

benches/.gitignore

@@ -1,3 +1,2 @@
 /target
 /ext/target
-callgrind.out.*

benches/Cargo.toml

@@ -5,7 +5,7 @@ edition = "2024"
 publish = false
 
 [dependencies]
-gungraun = "0.17.2"
+divan = { package = "codspeed-divan-compat", version = "5.0.1" }
 
 [features]
 default = ["enum", "runtime", "closure"]

benches/README.md

@@ -0,0 +1,30 @@
+# Benchmarks
+
+Benchmarks spawn `php -dextension=ext/target/release/libbenches.so <script> <count>`
+for each PHP script in `benches/` and measure the whole PHP process. Results are
+tracked on [CodSpeed](https://codspeed.io/extphprs/ext-php-rs) by the
+`Benchmarks` workflow in `simulation` mode (instruction count, php child tracked
+through `simulation-track-subprocess`).
+
+The harness is [divan](https://docs.rs/divan) through
+[`codspeed-divan-compat`](https://codspeed.io/docs/benchmarks/rust/divan), so
+plain `cargo bench` keeps working as a walltime run.
+
+## Running locally
+
+Always from a nix dev shell, which provides `php` and `cargo-codspeed`:
+
+```sh
+cd benches
+nix develop -c cargo bench
+nix develop -c cargo codspeed build
+nix develop -c cargo codspeed run
+```
+
+Without the CodSpeed runner, `cargo codspeed run` only prints `Checked: ...` for
+every benchmark, which is enough to validate the suite.
+
+The bench binary builds `ext/` (the PHP extension under test) with the features
+it was itself compiled with before running. That build is skipped when
+`CODSPEED_ENV` is set: under the CodSpeed runner every child process runs inside
+valgrind, so CI builds the extension in a separate step first.