Repository navigation
262 lines (228 loc) · 8.64 KB
/
Copy pathci.yml
File metadata and controls
262 lines (228 loc) · 8.64 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
name: ci
on:
push:
branches: [main]
pull_request:
workflow_dispatch:
# The release workflow calls this rather than restating the gate: a tag that
# would not have passed CI must not become a release. Tag pushes do not
# trigger the `push` filter above, so this runs once, not twice.
workflow_call:
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: true
env:
# Nothing in CI may reach a *billed* API. The template reasoner is the
# deterministic path, and a job that quietly started calling Claude would be
# non-reproducible and would burn credits on every push. The `semantic-evals`
# job overrides the embedder below: local weights are a cached download, not
# a metered call, which is exactly why the hosted lane cannot be tested here
# and the local one can.
OPS_RECALL_REASONER: template
OPS_RECALL_EMBEDDER: hashing
jobs:
test:
name: tests (py${{ matrix.python-version }})
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python-version: ["3.11", "3.12", "3.13"]
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
cache: pip
cache-dependency-path: pyproject.toml
- name: Install
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Tests
run: pytest
# The README claims the whole system runs offline. Asserting it here is
# cheap and catches the day someone adds a dependency that phones home.
- name: Demo runs end to end
run: ops-recall demo --plain
quality:
name: lint and types
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: pyproject.toml
- name: Install
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
- name: Ruff
run: ruff check --output-format=github .
# `ruff format` is deliberately not run: several tables and word lists in
# this codebase are hand-aligned for readability and the formatter would
# collapse them. Lint enforces correctness rules, not layout.
- name: Mypy
run: mypy
evals:
name: retrieval quality
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: pyproject.toml
- name: Install
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
# The floor is a merge gate, not a report: a change that makes retrieval
# worse should fail here rather than be noticed in a demo three weeks on.
- name: Score the labeled set
run: ops-recall eval --plain --fail-under 0.90
- name: Publish the scores to the run summary
if: always()
run: |
{
echo '### Retrieval quality'
echo '```'
ops-recall eval --plain || true
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
semantic-evals:
name: retrieval quality (local semantic lane)
runs-on: ubuntu-latest
# The hashing lane is a floor, not a model, so the `evals` job above scores
# a semantic lane that captures no semantics. This job runs the same suite
# against real weights. It needs no key and no credits -- which is the whole
# reason the local embedder exists: the hosted one can never run here.
env:
OPS_RECALL_EMBEDDER: fastembed
OPS_RECALL_FASTEMBED_CACHE_DIR: ${{ github.workspace }}/.fastembed
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: pyproject.toml
# ~67MB of ONNX weights. Keyed on the model name so a model change
# fetches once and then stops paying for the download on every push.
- name: Cache the model
uses: actions/cache@v4
with:
path: ${{ github.workspace }}/.fastembed
key: fastembed-bge-small-en-v1.5-v1
- name: Install
run: |
python -m pip install --upgrade pip
pip install -e ".[dev,fastembed]"
# Same floor as the hashing job. The two scoring identically is the
# finding, not a coincidence: see the README table.
- name: Score the labeled set
run: ops-recall eval --plain --fail-under 0.90
# The `slow` tests load these weights, so this is the one job that can
# run them. Chief among them: the paraphrase pair the hashing lane scores
# at exactly 0.0.
- name: Tests that need real weights
run: pytest -m slow
- name: Publish the scores to the run summary
if: always()
run: |
{
echo '### Retrieval quality (local semantic lane)'
echo '```'
ops-recall eval --plain || true
echo '```'
} >> "$GITHUB_STEP_SUMMARY"
telemetry:
name: live telemetry
runs-on: ubuntu-latest
# The catalog queries in the postgres provider are the kind of code that
# reads correctly and is not -- `SET` takes no bind parameters, blocking
# pids need a lateral join. Running them against a real server is the only
# thing that says so. Prometheus is covered by recorded responses instead:
# PromQL over HTTP has no equivalent way to be subtly wrong.
services:
postgres:
image: postgres:16
env:
POSTGRES_PASSWORD: probe
POSTGRES_DB: probe
ports: ["5432:5432"]
options: >-
--health-cmd pg_isready
--health-interval 5s
--health-timeout 5s
--health-retries 10
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: pyproject.toml
- name: Install
run: |
python -m pip install --upgrade pip
pip install -e ".[dev,postgres]"
# Without the DSN these skip, which is what happens on a contributor's
# laptop. Here it is set, so a skip would mean the driver is missing --
# and the fixture raises rather than skipping for that.
- name: Probe a real PostgreSQL
env:
OPS_RECALL_TEST_POSTGRES_DSN: postgresql://postgres:probe@127.0.0.1:5432/probe
run: pytest tests/test_postgres.py -q -rs
locked:
name: locked install
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
# The lock is generated on the declared floor, so this is the version
# it is guaranteed to describe. The matrix above covers the range by
# resolving from pyproject instead.
python-version: "3.11"
cache: pip
cache-dependency-path: requirements-dev.lock
- name: Install the exact locked set
run: |
python -m pip install --upgrade pip
pip install -r requirements-dev.lock
pip install --no-deps -e .
# A lock nothing installs from rots quietly. This is the job that notices
# the day it stops resolving, rather than a contributor six months out.
- name: Tests
run: pytest
- name: Retrieval quality
run: ops-recall eval --plain --fail-under 0.90
docker:
name: container
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Build
run: docker build -t ops-recall:ci .
# Building proves the image assembles; running proves the paths inside it
# resolve, which is the part that actually breaks.
- name: Demo inside the image
run: docker run --rm ops-recall:ci demo --plain
- name: Serve and answer a real alert
run: |
docker run -d --name ops-recall-ci -p 8000:8000 ops-recall:ci serve
for _ in $(seq 1 30); do
curl -sf http://127.0.0.1:8000/healthz && break
sleep 2
done
curl -sf -X POST http://127.0.0.1:8000/v1/alerts \
-H 'content-type: application/json' \
-d '{"title":"checkout-api 5xx ratio > 25% (ERR_5012 connection timeout)",
"description":"HikariPool-1 - Connection is not available.",
"service":"checkout-api","host":"db-prod-03","severity":"sev1"}' \
| python -c "import json,sys; r=json.load(sys.stdin); print(r['reconstruction']['headline'])"
docker rm -f ops-recall-ci