Repository navigation
259 lines (259 loc) · 12.2 KB
/
Copy pathci.yml
File metadata and controls
259 lines (259 loc) · 12.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
name: CI
on:
pull_request:
branches: [main]
# Every commit on main is tested too, including merges that skip the queue
# (an admin can bypass it).
push:
branches: [main]
# The merge queue tests each PR merged with main and with every PR queued
# ahead of it, before it lands, so that break shows up here instead of on
# main. A check the queue requires must run on this event, or the queue
# waits for it forever. Each queued commit is its own concurrency group.
merge_group:
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install ruff
- name: Run Ruff
# Use GitHub’s annotation format so lint errors show inline
run: ruff check --output-format=github .
- name: Test the optional Mac runner
run: python -m unittest discover -s test/ci -v
# Runs every test directory in CPU_TEST_PATHS (scripts/ci/select_cpu_tests.py)
# with no list of files to maintain: modules that need a GPU, weights or an
# optional package skip themselves; see CONTRIBUTING.md. test/ci checks that
# every other test module is excluded there with a reason. scripts/ci/check.sh
# runs the same paths for the Mac cpu-smoke check. The cpu-core job below
# reports the result as the required check "CPU Core".
cpu-tests:
name: ${{ matrix.name }}
runs-on: ubuntu-latest
timeout-minutes: 25
env:
# The CPU Core shards' pytest arguments; MSTAR_CI_SHARD picks each one's files.
CPU_CORE: >-
-p shard_plugin -n 4 --dist worksteal
--ignore=test/modular/test_tp_async_lockstep_sim.py
test/modular test/distributed test/chatterbox test/kokoro test/omnivoice test/sampling_test mstar/model/*/tests benchmark/worker_phases
strategy:
fail-fast: false
matrix:
include:
# Everything but the TP lockstep simulation, split by test file over
# two runners (MSTAR_CI_SHARD, scripts/ci/shard_plugin.py), one pytest
# worker per runner core (4); worksteal moves queued tests off a busy worker.
- name: CPU Core 1/2
id: cpu-tests-1
shard: 1/2
- name: CPU Core 2/2
id: cpu-tests-2
shard: 2/2
# Its three simulations take up to 3.5 minutes each on these runners,
# longer than either shard's slowest file, so they run beside them.
- name: CPU Core (TP lockstep sim)
id: cpu-tests-tp-sim
args: -n 3 test/modular/test_tp_async_lockstep_sim.py
# A new push to a PR cancels its older run. On main each merge commit gets
# its own group and is never cancelled, so every merge reports a result.
concurrency:
group: ${{ matrix.id }}-${{ github.event.pull_request.number || github.sha }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
steps:
- uses: actions/checkout@v4
- name: Install the pinned Rust toolchain (rust-toolchain.toml)
run: rustup toolchain install
- uses: Swatinem/rust-cache@v2
with:
workspaces: rust
# A queue run's branch is deleted at once, so a cache saved there is never read.
save-if: ${{ github.event_name != 'merge_group' }}
- uses: actions/setup-python@v5
with:
python-version: "3.12"
# uv installs into setup-python's interpreter (UV_SYSTEM_PYTHON below), with
# the same pins pip had, in a fraction of the time; its cache is keyed on them.
- uses: astral-sh/setup-uv@v10.3.3
with:
version: "0.13.5"
enable-cache: true
cache-dependency-glob: |
pyproject.toml
.github/workflows/ci.yml
- name: Install mstar with CPU PyTorch
env:
UV_SYSTEM_PYTHON: "1"
# Compile the installed packages' .pyc files at install, as pip does, so pytest's
# workers don't each compile them (mstar's own -e sources still compile on first import).
UV_COMPILE_BYTECODE: "1"
run: |
uv pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 \
--index-url https://download.pytorch.org/whl/cpu
# flex_attention imports torch._inductor, which needs Triton even on
# CPU; 3.6.0 is the version torch 2.10.0 pins.
uv pip install triton==3.6.0
uv pip install -e ".[dev,waypoint,metrics]"
# BAGEL's CPU graph/KV tests import these model modules without
# loading weights or requiring the CUDA-only BAGEL extras.
uv pip install einops regex transformers
# LingBot-World's request contract tests clean prompts and resample
# camera poses the way its reference does (no weights needed).
uv pip install ftfy scipy
# Model.load_audio decodes with libsndfile before torchcodec; the
# tests of that path skip without it.
uv pip install soundfile
# The eval runner (benchmark/asr_eval.py, imported by the WER and DROID
# dataset tests) needs aiohttp, which datasets 5.2 stopped pulling in.
uv pip install aiohttp
# The Wan VAE decode tests build tiny random AutoencoderKLWan models
# (no weights); 0.39 is the release the decoder was verified on.
uv pip install diffusers==0.39.0
# pytest is pinned with the rest of the test stack, so a release can't
# change fixture behaviour under every PR at once.
uv pip install pytest==9.2.0 pytest-xdist==3.8.0
# The AudioSeal watermark streaming tests build its generator with
# random weights (no download).
uv pip install "audioseal>=0.2,<0.3"
uv pip check
# The Qwen3-TTS codec tests load qwen-tts' speech-tokenizer modules.
# Its metadata pins transformers==4.57.3 and packages mstar never
# imports, so it goes in without them, after pip check.
uv pip install --no-deps "qwen-tts==0.1.1"
# The extension's build tool (next step).
uv pip install maturin
- name: Build and install the extension
env:
# maturin develop wants a virtualenv; setup-python's interpreter is
# the one the tests run on.
VIRTUAL_ENV: ${{ env.pythonLocation }}
run: maturin develop --release -m rust/Cargo.toml
- name: Run CPU tests
env:
MSTAR_CI_SHARD: ${{ matrix.shard }}
# Where -p finds shard_plugin; loaded even with plugin autoload off.
PYTHONPATH: scripts/ci
HF_HUB_OFFLINE: "1"
PYTEST_DISABLE_PLUGIN_AUTOLOAD: "1"
# One thread per worker: workers with 4 OpenMP threads each
# oversubscribe the 4 cores, and on a busy host the conv-heavy
# decode tests stall.
OMP_NUM_THREADS: "1"
MKL_NUM_THREADS: "1"
run: |
python -m pytest -q -ra --strict-markers --durations=20 -p xdist \
--junitxml=${{ matrix.id }}.xml ${{ matrix.args || env.CPU_CORE }}
- name: Upload CPU test report
if: always()
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.id }}-results
path: ${{ matrix.id }}.xml
retention-days: 7
# The required check "CPU Core": it passes only when every cpu-tests job
# passed (both shards and the TP lockstep sim), and fails when one failed,
# timed out or was cancelled. It runs on every event the workflow runs on.
cpu-core:
name: CPU Core
needs: cpu-tests
if: always()
runs-on: ubuntu-latest
steps:
- name: Every CPU Core job passed
run: test "${{ needs.cpu-tests.result }}" = success
# Dynamo backend contract: bridge body translations plus a registration
# smoke against the pinned ai-dynamo wheel (in-memory discovery backend —
# no GPU, no etcd). Fails here when a wheel bump moves the contract.
dynamo-smoke:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Install mstar with the dynamo extra (cpu torch)
run: |
python -m pip install --upgrade pip
pip install torch==2.10.0 torchvision==0.25.0 torchaudio==2.10.0 \
--index-url https://download.pytorch.org/whl/cpu
pip install -e ".[dynamo]" pytest==9.2.0
- name: Bridge + registration tests
env:
HF_HUB_OFFLINE: "1"
run: pytest test/modular/test_dynamo_bridges.py test/modular/test_dynamo_registration_smoke.py -v
# Vendored Rust transport (rust/): cargo tests, then pytest test/rust
# against a freshly built extension.
rust-transport:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Install the pinned Rust toolchain (rust-toolchain.toml)
run: rustup toolchain install
- uses: Swatinem/rust-cache@v2
with:
save-if: ${{ github.event_name != 'merge_group' }}
workspaces: |
rust
rust/server
- uses: actions/setup-python@v5
with:
python-version: "3.12"
# pip's download cache, keyed on this file, which holds the pins below
cache: pip
cache-dependency-path: .github/workflows/ci.yml
- name: Install libzmq and ffmpeg
# ffmpeg: the video parity tests mux a generated sound track, which skip without it
run: sudo apt-get update && sudo apt-get install -y libzmq3-dev ffmpeg
- name: Cargo tests (transport + arena semantics)
working-directory: rust
# The bindings without extension-module (see rust/Cargo.toml): the test
# binary is an executable, so it links libpython rather than resolving
# CPython symbols at import time. setup-python's interpreter is
# --enable-shared, so point the loader at its libdir for the run.
env:
LD_LIBRARY_PATH: ${{ env.pythonLocation }}/lib
run: cargo test --release --no-default-features --features python
- run: cargo test --release --manifest-path rust/server/Cargo.toml
- name: Build the Rust frontend (mstar-server)
working-directory: rust/server
run: cargo build --release
- name: Build and install the extension
working-directory: rust
run: |
python -m pip install --upgrade pip maturin
maturin build --release
pip install target/wheels/*.whl
- name: Interop tests (transport + arena + frontend)
run: |
# pyyaml: the runtime tests reach mstar.model.base, which imports it;
# prometheus-client: the frontend's /metrics test renders a real registry;
# fastapi, uvicorn, python-multipart and httpx: the Rust/Python chat
# parity cases run the Python app through fastapi's TestClient;
# soundfile (its wheel bundles libsndfile): the voice-clip checks and
# the compressed-speech byte parity, which skip without it;
# websockets and onnxruntime: the WebSocket parity cases and realtime's
# turn detection; tokenizers: the realtime suite's stand-in tokenizer;
# silero-vad only for its bundled ONNX file, so without its torchaudio
pip install pytest==9.2.0 msgpack numpy pyyaml prometheus-client \
fastapi uvicorn python-multipart httpx soundfile websockets onnxruntime tokenizers
pip install --no-deps silero-vad
# cpu torch + triton: the arena test imports the tensor-transport
# stack, which reaches mstar.utils.sampling (triton at module level).
# torchaudio: load_speech_file resamples with it, which the
# transcription parity cases reach on their 44.1 kHz upload. Pinned
# as CPU Core pins them.
pip install torch==2.10.0 torchaudio==2.10.0 \
--index-url https://download.pytorch.org/whl/cpu
pip install triton==3.6.0
# the package itself, without its remaining (heavy) deps
pip install --no-deps -e .
# the row store needs the columnar tensor store; CPU Core skips most of it
pytest test/rust/ test/modular/test_row_output_store.py -v