Skip to content

Push master perf results to a history repo #23523

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
abulgit wants to merge 1 commit into
base: master
Choose a base branch
from
+193 −31
Open

Push master perf results to a history repo #23523

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 37 additions & 0 deletions .github/scripts/perf_publish.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/usr/bin/env bash
set -ueo pipefail

results="$1"
clone="$RUNNER_TEMP/perf-data"
key=~/.ssh/perf_data

mkdir -p ~/.ssh
printf '%s\n' "$PERF_DATA_KEY" > "$key"
chmod 600 "$key"
ssh-keyscan github.com >> ~/.ssh/known_hosts 2> /dev/null
export GIT_SSH_COMMAND="ssh -i $key -o IdentitiesOnly=yes"

git clone "git@github.com:$PERF_DATA_REPO.git" "$clone"
cd "$clone"
git config user.name dmd-perf-bot
git config user.email dmd-perf-bot@users.noreply.github.com

dest="data/${COMMITTED_AT:0:4}/${COMMITTED_AT:5:2}"
mkdir -p "$dest"
cp "$results" "$dest/$SHA.json"
git add "$dest/$SHA.json"

if git diff --cached --quiet; then
echo "$SHA already published"
exit 0
fi

git commit -m "$SHA"

for _ in 1 2 3; do
if git push; then
exit 0
fi
git pull --rebase
done
exit 1
90 changes: 67 additions & 23 deletions .github/workflows/perf.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ on:

concurrency:
group: perf-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
cancel-in-progress: ${{ github.event_name == 'pull_request' }}

permissions:
contents: read
Expand Down Expand Up @@ -42,12 +42,24 @@ jobs:
run: |
set -uexo pipefail
HEAD_SHA="${{ github.event.pull_request.head.sha || github.sha }}"
git fetch --no-tags origin master
BASE_SHA="$(git merge-base "$HEAD_SHA" origin/master)"
echo "head=$HEAD_SHA" >> "$GITHUB_OUTPUT"
echo "base=$BASE_SHA" >> "$GITHUB_OUTPUT"
echo "head=$HEAD_SHA" >> "$GITHUB_OUTPUT"
echo "branch=${GITHUB_BASE_REF:-master}" >> "$GITHUB_OUTPUT"

# A pushed commit is its own merge-base with master, so nothing
# to diff against there. Measure alone and record it as history.
if [ "${{ github.event_name }}" = pull_request ]; then
git fetch --no-tags origin master
echo "base=$(git merge-base "$HEAD_SHA" origin/master)" >> "$GITHUB_OUTPUT"
else
BEFORE="${{ github.event.before }}"
# All zeroes on the first push to a branch.
git rev-parse -q --verify "$BEFORE^{commit}" > /dev/null || BEFORE="$HEAD_SHA~"
echo "base=" >> "$GITHUB_OUTPUT"
echo "before=$BEFORE" >> "$GITHUB_OUTPUT"
echo "commits=$(git rev-list --count "$BEFORE..$HEAD_SHA")" >> "$GITHUB_OUTPUT"
echo "date=$(TZ=UTC git show -s --format=%cd --date=iso-strict-local "$HEAD_SHA")" >> "$GITHUB_OUTPUT"
fi

- name: Install prerequisites
run: |
sudo apt-get update
Expand All @@ -67,6 +79,7 @@ jobs:
source ~/dlang/*/activate
built=generated/$OS_NAME/release/$MODEL
profile="$RUNNER_TEMP/pgo/merged.data"
BASE_SHA="${{ steps.refs.outputs.base }}"
mkdir -p "$RUNNER_TEMP/pgo"

# Check out a ref and build a plain dmd + phobos
Expand All @@ -80,34 +93,56 @@ jobs:
( cd "$dir/../phobos" && make -j$N MODEL=$MODEL )
}

setup_ref "${{ steps.refs.outputs.base }}" base
setup_ref "${{ steps.refs.outputs.head }}" head
if [ -n "$BASE_SHA" ]; then
setup_ref "$BASE_SHA" base
train=base
else
train=head
fi

# dmd-pgo trains and rebuilds base with the merged profile, head just reuses it
( cd "$RUNNER_TEMP/base/dmd" && generated/build HOST_DMD=$DMD MODEL=$MODEL dmd-pgo -j$N --force )
cp "$RUNNER_TEMP/base/dmd/$built/dmd_profdata/merged.data" "$profile"
( cd "$RUNNER_TEMP/head/dmd" && generated/build HOST_DMD=$DMD MODEL=$MODEL \
ENABLE_RELEASE=1 ENABLE_LTO=1 \
DFLAGS="-fprofile-instr-use=$profile -wi" -j$N --force )
# dmd-pgo trains and rebuilds $train with the merged profile, the other ref just reuses it
( cd "$RUNNER_TEMP/$train/dmd" && generated/build HOST_DMD=$DMD MODEL=$MODEL dmd-pgo -j$N --force )
cp "$RUNNER_TEMP/$train/dmd/$built/dmd_profdata/merged.data" "$profile"
if [ -n "$BASE_SHA" ]; then
( cd "$RUNNER_TEMP/head/dmd" && generated/build HOST_DMD=$DMD MODEL=$MODEL \
ENABLE_RELEASE=1 ENABLE_LTO=1 \
DFLAGS="-fprofile-instr-use=$profile -wi" -j$N --force )
fi
deactivate

- name: Measure
run: |
set -uexo pipefail
source ~/dlang/*/activate
built=generated/$OS_NAME/release/$MODEL/dmd
cd tools/perfrunner
dub run -- \
--base-dmd "$RUNNER_TEMP/base/dmd/$built" \
--head-dmd "$RUNNER_TEMP/head/dmd/$built" \
--base-phobos "$RUNNER_TEMP/base/phobos" \
--head-phobos "$RUNNER_TEMP/head/phobos" \
--base-sha "${{ steps.refs.outputs.base }}" \
--head-sha "${{ steps.refs.outputs.head }}" \
--pr "${{ github.event.pull_request.number || 0 }}" \
--os ubuntu-24.04 \
--host-dmd "$HOST_DMD" \
BASE_SHA="${{ steps.refs.outputs.base }}"

args=(
--head-dmd "$RUNNER_TEMP/head/dmd/$built"
--head-phobos "$RUNNER_TEMP/head/phobos"
--head-sha "${{ steps.refs.outputs.head }}"
--os ubuntu-24.04
--host-dmd "$HOST_DMD"
--out "$GITHUB_WORKSPACE/results.json"
)
if [ -n "$BASE_SHA" ]; then
args+=(
--base-dmd "$RUNNER_TEMP/base/dmd/$built"
--base-phobos "$RUNNER_TEMP/base/phobos"
--base-sha "$BASE_SHA"
--pr "${{ github.event.pull_request.number || 0 }}"
)
else
args+=(
--before "${{ steps.refs.outputs.before }}"
--commits "${{ steps.refs.outputs.commits }}"
--committed-at "${{ steps.refs.outputs.date }}"
)
fi

cd tools/perfrunner
dub run -- "${args[@]}"

- name: Show results
run: cat "$GITHUB_WORKSPACE/results.json"
Expand All @@ -119,3 +154,12 @@ jobs:
with:
name: perf-results
path: results.json

- name: Publish to the history repo
if: github.event_name == 'push' && vars.PERF_DATA_REPO != ''
env:
PERF_DATA_REPO: ${{ vars.PERF_DATA_REPO }}
PERF_DATA_KEY: ${{ secrets.PERF_DATA_KEY }}
SHA: ${{ steps.refs.outputs.head }}
COMMITTED_AT: ${{ steps.refs.outputs.date }}
run: .github/scripts/perf_publish.sh "$GITHUB_WORKSPACE/results.json"
36 changes: 28 additions & 8 deletions tools/perfrunner/source/app.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import std.path : buildPath, dirName;
import std.stdio : stderr, writeln;

import metrics : measure, initials;
import report : MetricResult, render, Report;
import report : CommitRecord, MetricResult, render, renderCommit, Report;
import vibed : describeFlags;

enum workloads = buildPath(__FILE_FULL_PATH__.dirName.dirName, "workloads");
Expand All @@ -22,8 +22,9 @@ version (unittest) {} else
int main(string[] args)
{
string baseDmd, headDmd, basePhobos, headPhobos, baseSha, headSha, hostDmd, os;
string before, committedAt;
string outPath = "results.json";
long pr;
long pr, commits = 1;

auto help = getopt(args,
"base-dmd", "path to the base (merge-base) dmd binary", &baseDmd,
Expand All @@ -33,31 +34,50 @@ int main(string[] args)
"base-sha", "base commit sha (metadata)", &baseSha,
"head-sha", "head commit sha (metadata)", &headSha,
"pr", "pull request number (metadata)", &pr,
"before", "sha master pointed at before the push (metadata)", &before,
"commits", "commits contained in the push (metadata)", &commits,
"committed-at", "commit timestamp (metadata)", &committedAt,
"os", "runner OS label (metadata)", &os,
"host-dmd", "bootstrap dmd version (metadata)", &hostDmd,
"out", "where to write results.json", &outPath,
);

if (help.helpWanted)
{
writeln("usage: perfrunner --base-dmd <path> --head-dmd <path> "
~ "--base-phobos <dir> --head-phobos <dir> "
writeln("usage: perfrunner --head-dmd <path> --head-phobos <dir> "
~ "[--base-dmd <path> --base-phobos <dir>] "
~ "[--base-sha <sha> --head-sha <sha> --pr <n>] --out results.json");
writeln("without a base the single commit is measured for the history repo");
return 0;
}

if (baseDmd.length == 0 || headDmd.length == 0
|| basePhobos.length == 0 || headPhobos.length == 0)
if (headDmd.length == 0 || headPhobos.length == 0)
{
stderr.writeln("error: --base-dmd, --head-dmd, --base-phobos and --head-phobos are required");
stderr.writeln("error: --head-dmd and --head-phobos are required");
return 2;
}

immutable diff = baseDmd.length != 0;
if (diff && basePhobos.length == 0)
{
stderr.writeln("error: --base-dmd needs --base-phobos");
return 2;
}

auto tmp = buildPath(tempDir, "perfrunner");
mkdirRecurse(tmp);

// Resolved once so both refs compile vibe.d with the same flags.
auto vibedFlags = describeFlags(vibedDir, baseDmd);
auto vibedFlags = describeFlags(vibedDir, diff ? baseDmd : headDmd);

if (!diff)
{
auto m = measure(headDmd, workload, headPhobos, vibedRoot, vibedFlags, tmp, "head");
write(outPath, renderCommit(CommitRecord(headSha, committedAt, before, commits,
os, hostDmd, m.metrics, m.helloTrace, m.phobosTrace)));
writeln("wrote ", outPath);
return 0;
}

// measure base in a second thread while this one does head
auto baseTask = task!measure(baseDmd, workload, basePhobos, vibedRoot, vibedFlags, tmp, "base");
Expand Down
61 changes: 61 additions & 0 deletions tools/perfrunner/source/report.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,19 @@ struct Report
Trace phobosBase, phobosHead;
}

// One master commit measured on its own.
struct CommitRecord
{
string sha;
string committedAt;
string before;
long commits;
string os;
string hostDmd;
long[string] metrics;
Trace hello, phobos;
}

// Serialise a report to the initial schema
string render(Report rep)
{
Expand Down Expand Up @@ -61,6 +74,29 @@ string render(Report rep)
return root.toPrettyString();
}

// Serialise a single-commit measurement for the history repo
string renderCommit(CommitRecord rec)
{
JSONValue[string] metrics;
foreach (id, value; rec.metrics)
metrics[id] = JSONValue(value);

JSONValue root = [
"schema_version": JSONValue(3),
"commit": JSONValue(rec.sha),
"committed_at": JSONValue(rec.committedAt),
"push": JSONValue(["before": JSONValue(rec.before), "commits": JSONValue(rec.commits)]),
"runner": JSONValue(["os": JSONValue(rec.os), "host_dmd": JSONValue(rec.hostDmd)]),
"metrics": JSONValue(metrics),
"time_trace": JSONValue([
"hello": traceJson(rec.hello),
"phobos": traceJson(rec.phobos),
]),
];

return root.toPrettyString();
}

private JSONValue pair(long base, long head)
{
return JSONValue(["base": JSONValue(base), "head": JSONValue(head)]);
Expand All @@ -78,6 +114,18 @@ private JSONValue traceJson(Trace b, Trace h)
]);
}

private JSONValue traceJson(Trace t)
{
JSONValue[string] phases;
foreach (id; phaseIds)
phases[id] = JSONValue(t.phase(id));

return JSONValue([
"total_us": JSONValue(t.total),
"phases": JSONValue(phases),
]);
}

unittest
{
auto rep = Report("base1", "merge-base", "head1", 7, "ubuntu-latest", "2.112.0",
Expand All @@ -97,3 +145,16 @@ unittest
import std.math : isClose;
assert(isClose(m["delta_pct"].floating, 1.0));
}

unittest
{
auto rec = CommitRecord("head1", "2026-07-30T09:12:44Z", "before1", 3,
"ubuntu-latest", "ldc-1.42.0", ["compile_hello_debug_instr": 1000L]);

auto j = parseJSON(renderCommit(rec));
assert(j["schema_version"].integer == 3);
assert(j["commit"].str == "head1");
assert(j["push"]["commits"].integer == 3);
assert(j["metrics"]["compile_hello_debug_instr"].integer == 1000);
assert(j["time_trace"]["hello"]["total_us"].integer == 0);
}
Loading