CI: publish perf and coverage metrics as a sticky PR comment

.github/scripts/bench_metrics.sh

@@ -0,0 +1,66 @@
+#!/bin/sh
+# usage: bench_metrics.sh <putup> <config-dir> <source-dir> <build-dir>
+# Emits TSV; .github/scripts/compose_metrics.py renders it.
+set -eu
+PUTUP=$1
+CDIR=$2
+SDIR=$3
+BDIR=$4
+
+PERF=${PERF:-perf}
+REPEAT=${BENCH_REPEAT:-3}
+
+metric() {
+    awk -v ev="$1" -v div="$2" -v fmt="$3" '
+        index($0, ev) {
+            gsub(",", "", $1)
+            if ($1 ~ /^[0-9.]+$/) printf fmt, $1 / div
+            else printf "n/a"
+            exit
+        }' "$pf"
+}
+
+miss_rate() {
+    awk -v ev="$1" '
+        $0 ~ ev {
+            for (i = 1; i <= NF; i++)
+                if ($i == "rate:") { sub("%", "", $(i + 1)); print $(i + 1); exit }
+        }' "$cg"
+}
+
+run_row() {
+    name=$1
+    shift
+    tf=$(mktemp)
+    pf=$(mktemp)
+    /usr/bin/time -v "$@" >/dev/null 2>"$tf"
+    rss_mb=$(awk '/Maximum resident set size/{printf "%.1f", $NF/1024}' "$tf")
+    "$PERF" stat -r "$REPEAT" -e task-clock:u,page-faults:u,cycles:u,instructions:u -o "$pf" -- "$@" >/dev/null 2>&1
+
+    d1="n/a"
+    ll="n/a"
+    if command -v valgrind >/dev/null; then
+        cg=$(mktemp)
+        cgout=$(mktemp)
+        valgrind --tool=cachegrind --cache-sim=yes \
+            --cachegrind-out-file="$cgout" --log-file="$cg" "$@" >/dev/null 2>&1 || true
+        d1=$(miss_rate 'D1 +miss rate:')
+        ll=$(miss_rate 'LL miss rate:')
+        rm -f "$cg" "$cgout"
+    fi
+
+    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
+        "$name" \
+        "$(metric instructions:u 1e6 '%.0f')" \
+        "$(metric task-clock 1000 '%.2f')" \
+        "$(metric page-faults 1000 '%.1f')" \
+        "${d1:-n/a}" \
+        "${ll:-n/a}" \
+        "$(metric 'time elapsed' 1 '%.3f')" \
+        "$rss_mb"
+    rm -f "$tf" "$pf"
+}
+
+printf 'workload\tinstructions_m\tcpu_s\tfaults_k\td1_pct\tll_pct\twall_s\trss_mb\n'
+run_row "parse" "$PUTUP" parse -C "$CDIR" -S "$SDIR" -B "$BDIR"
+run_row "dry-run" "$PUTUP" -n -C "$CDIR" -S "$SDIR" -B "$BDIR" -j"$(nproc)"

.github/scripts/compose_metrics.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+import argparse
+import csv
+import sys
+
+PERF_NOTE = (
+    "Deterministic signals: page faults, peak RSS, and the cachegrind D1/LL miss"
+    " rates (simulated cache, exact across runs). CPU time is the stable compute"
+    " signal on shared runners; instructions read n/a on GitHub-hosted runners"
+    " (virtualized, no PMU)."
+)
+
+
+def read_tsv(path):
+    if not path:
+        return []
+    try:
+        with open(path, encoding="utf-8", newline="") as f:
+            return list(csv.DictReader(f, delimiter="\t"))
+    except OSError:
+        return []
+
+
+def num(value):
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return None
+
+
+def annotate(cur, base, kind, unit=""):
+    if cur is None:
+        return "n/a"
+    shown = f"{cur:g}{unit}"
+    if base is None:
+        return shown
+    if kind == "pct_change":
+        if base == 0:
+            return shown
+        change = (cur - base) / base * 100
+        if abs(change) < 0.05:
+            return shown
+        return f"{shown} ({change:+.1f}%)"
+    diff = cur - base
+    if abs(diff) < 0.05:
+        return shown
+    suffix = "pp" if kind == "pp" else unit.strip()
+    return f"{shown} ({diff:+.1f}{suffix})"
+
+
+def perf_section(cur_rows, base_rows):
+    base_by_name = {r.get("workload"): r for r in base_rows}
+    lines = [
+        "### Performance (gcc example, Linux)",
+        "",
+        "| Workload | Instructions | CPU time | Page faults | D1 miss | LL miss | Wall | Peak RSS |",
+        "|---|---|---|---|---|---|---|---|",
+    ]
+    for row in cur_rows:
+        base = base_by_name.get(row["workload"], {})
+        cells = [
+            row["workload"],
+            annotate(num(row["instructions_m"]), num(base.get("instructions_m")), "pct_change", " M"),
+            f'{num(row["cpu_s"]):g} s' if num(row["cpu_s"]) is not None else "n/a",
+            annotate(num(row["faults_k"]), num(base.get("faults_k")), "pct_change", " k"),
+            annotate(num(row["d1_pct"]), num(base.get("d1_pct")), "pp", "%"),
+            annotate(num(row["ll_pct"]), num(base.get("ll_pct")), "pp", "%"),
+            f'{num(row["wall_s"]):g} s' if num(row["wall_s"]) is not None else "n/a",
+            annotate(num(row["rss_mb"]), num(base.get("rss_mb")), "abs", " MB"),
+        ]
+        lines.append("| " + " | ".join(cells) + " |")
+    lines += ["", PERF_NOTE]
+    return lines
+
+
+def coverage_section(cur, base):
+    lines = [
+        "### Test coverage (lines)",
+        "",
+        "| Overall | Median file | Min file | Max file |",
+        "|---|---|---|---|",
+    ]
+    lines.append(
+        "| {} | {} | {:.1f}% `{}` | {:.1f}% `{}` |".format(
+            annotate(num(cur["overall"]), num(base.get("overall")), "pp", "%"),
+            annotate(num(cur["median"]), num(base.get("median")), "pp", "%"),
+            num(cur["min_pct"]),
+            cur["min_file"],
+            num(cur["max_pct"]),
+            cur["max_file"],
+        )
+    )
+    lines += [
+        "",
+        "{} files · {}/{} lines covered".format(
+            cur["files"], cur["covered"], cur["total"]
+        ),
+    ]
+    return lines
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--perf")
+    ap.add_argument("--coverage")
+    ap.add_argument("--baseline-perf")
+    ap.add_argument("--baseline-coverage")
+    ap.add_argument("--baseline-label")
+    args = ap.parse_args()
+
+    sections = []
+    baseline_used = False
+
+    perf = read_tsv(args.perf)
+    if perf:
+        base = read_tsv(args.baseline_perf)
+        baseline_used |= bool(base)
+        sections.append(perf_section(perf, base))
+
+    cov = read_tsv(args.coverage)
+    if cov:
+        base_rows = read_tsv(args.baseline_coverage)
+        base = base_rows[0] if base_rows else {}
+        baseline_used |= bool(base)
+        sections.append(coverage_section(cov[0], base))
+
+    if not sections:
+        print("no metrics inputs", file=sys.stderr)
+        return 1
+
+    out = []
+    for i, section in enumerate(sections):
+        if i:
+            out.append("")
+        out.extend(section)
+    if baseline_used and args.baseline_label:
+        out += ["", f"Deltas vs {args.baseline_label}."]
+    print("\n".join(out))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/coverage_stats.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+import json
+import statistics
+import sys
+
+
+def main() -> int:
+    with open(sys.argv[1], encoding="utf-8") as f:
+        summary = json.load(f)
+
+    files = [e for e in summary["files"] if e.get("line_total", 0) > 0]
+    if not files:
+        print("no coverage data", file=sys.stderr)
+        return 1
+
+    worst = min(files, key=lambda e: e["line_percent"])
+    best = max(files, key=lambda e: e["line_percent"])
+    median = statistics.median(e["line_percent"] for e in files)
+
+    print(
+        "overall\tmedian\tmin_pct\tmin_file\tmax_pct\tmax_file\tfiles\tcovered\ttotal"
+    )
+    print(
+        "\t".join(
+            str(v)
+            for v in (
+                summary["line_percent"],
+                median,
+                worst["line_percent"],
+                worst["filename"],
+                best["line_percent"],
+                best["filename"],
+                len(files),
+                summary["line_covered"],
+                summary["line_total"],
+            )
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/ci.yml

@@ -156,11 +156,20 @@ jobs:
         run: pipx install gcovr
       - name: Build instrumented variant and generate report
         run: make coverage PUTUP=./build/putup
+      - name: Coverage stats
+        run: |
+          python3 .github/scripts/coverage_stats.py build-coverage/report/summary.json > coverage-metrics.tsv
+          python3 .github/scripts/compose_metrics.py --coverage coverage-metrics.tsv >> "$GITHUB_STEP_SUMMARY"
       - name: Upload coverage report
         uses: actions/upload-artifact@v4
         with:
           name: coverage-report
           path: build-coverage/report/
+      - name: Upload coverage metrics
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-coverage-metrics
+          path: coverage-metrics.tsv
 
   build-gcc-bsp:
     needs: build-linux
@@ -194,3 +203,77 @@ jobs:
           echo 'int main() { return 0; }' > /tmp/test.c
           build-gcc/gcc/gcc/cc1 /tmp/test.c -quiet -o /tmp/test.s
           grep 'GCC.*15.2.0' /tmp/test.s
+      - name: Benchmark (perf)
+        run: |
+          sudo sysctl -w kernel.perf_event_paranoid=1
+          sudo apt-get install -y valgrind
+          sudo apt-get install -y linux-tools-$(uname -r) || sudo apt-get install -y linux-tools-generic
+          command -v perf >/dev/null || export PERF="$(ls /usr/lib/linux-tools/*/perf | sort -V | tail -1)"
+          .github/scripts/bench_metrics.sh build/putup examples/bsp source-root build-gcc > perf-metrics.tsv
+          python3 .github/scripts/compose_metrics.py --perf perf-metrics.tsv >> "$GITHUB_STEP_SUMMARY"
+      - name: Upload perf metrics
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-perf-metrics
+          path: perf-metrics.tsv
+
+  pr-metrics-comment:
+    if: github.event_name == 'pull_request'
+    needs: [coverage, build-gcc-bsp]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
+      actions: read
+    steps:
+      - uses: actions/checkout@v7
+      - name: Download perf metrics
+        uses: actions/download-artifact@v4
+        with:
+          name: pr-perf-metrics
+      - name: Download coverage metrics
+        uses: actions/download-artifact@v4
+        with:
+          name: pr-coverage-metrics
+      - name: Fetch baseline metrics from the last main run
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+        run: |
+          run=$(gh run list -R "$REPO" --workflow ci.yml --branch main --event push \
+            --status success --json databaseId,headSha --jq '.[0] // empty' -L 1)
+          if [ -n "$run" ]; then
+            gh run download "$(echo "$run" | jq -r .databaseId)" -R "$REPO" \
+              -n pr-perf-metrics -D baseline/perf || true
+            gh run download "$(echo "$run" | jq -r .databaseId)" -R "$REPO" \
+              -n pr-coverage-metrics -D baseline/cov || true
+            echo "BASE_SHA=$(echo "$run" | jq -r .headSha)" >> "$GITHUB_ENV"
+          fi
+      - name: Upsert sticky comment
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          PR: ${{ github.event.pull_request.number }}
+          SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          python3 .github/scripts/compose_metrics.py \
+            --perf perf-metrics.tsv --coverage coverage-metrics.tsv \
+            --baseline-perf baseline/perf/perf-metrics.tsv \
+            --baseline-coverage baseline/cov/coverage-metrics.tsv \
+            --baseline-label "main@${BASE_SHA:0:9}" > metrics.md
+          marker='<!-- pr-metrics -->'
+          {
+            echo "$marker"
+            echo "## PR metrics"
+            echo
+            cat metrics.md
+            echo
+            echo "_Updated for ${SHA}_"
+          } > body.md
+          cid=$(gh api "repos/$REPO/issues/$PR/comments" --paginate \
+            --jq "map(select(.body | startswith(\"$marker\"))) | .[0].id // empty")
+          if [ -n "$cid" ]; then
+            gh api -X PATCH "repos/$REPO/issues/comments/$cid" -F body=@body.md
+          else
+            gh api -X POST "repos/$REPO/issues/$PR/comments" -F body=@body.md
+          fi