Repository navigation
Expand file tree
/
Copy pathJustfile
More file actions
259 lines (226 loc) · 10 KB
/
Copy pathJustfile
File metadata and controls
259 lines (226 loc) · 10 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
set dotenv-load
set dotenv-required
set shell := ["bash", "-c"]
USER_NAME := env_var_or_default("USER_NAME", `whoami`)
NAMESPACE := USER_NAME + "-llm-d-wide-ep"
HF_TOKEN := "$HF_TOKEN"
GH_TOKEN := "$GH_TOKEN"
QUAY_REPO := env_var("QUAY_REPO")
QUAY_ROBOT := env_var("QUAY_ROBOT")
QUAY_PASSWORD := "$QUAY_PASSWORD"
QUAY_USERNAME := QUAY_REPO + "+" + QUAY_ROBOT
QUAY_REGISTRY := "quay.io/" + QUAY_REPO
MODEL := "deepseek-ai/DeepSeek-R1-0528"
KN := "kubectl -n " + NAMESPACE
EXAMPLE_DIR := "llm-d/guides/wide-ep-lws"
@print-gpus:
kubectl get pods -A -o json | jq -r ' \
.items \
| map(select(.status.phase=="Running")) \
| map({ \
ns: .metadata.namespace, \
pod: .metadata.name, \
node: .spec.nodeName, \
gpus: ([ .spec.containers[]? \
| ( .resources.limits."nvidia.com/gpu" \
// .resources.requests."nvidia.com/gpu" \
// "0" ) | tonumber ] | add) \
}) \
| map(select(.gpus>0 and .node != null)) \
| sort_by(.node, .ns, .pod) \
| group_by(.node) \
| .[] as $grp \
| "== Node: \($grp[0].node) ==", \
"NAMESPACE\tPOD\tGPUs", \
( $grp[] | "\(.ns)\t\(.pod)\t\(.gpus)" ), \
"" \
' | column -t -s $'\t' \
| awk 'NR==1{print; next} /^== /{print ""; print; next} {print}'
# Print table of nodes on Coreweave. Quiet recipe since the command is messy
@cks-nodes:
kubectl get nodes -o=custom-columns="NAME:metadata.name,IP:status.addresses[?(@.type=='InternalIP')].address,TYPE:metadata.labels['node\.coreweave\.cloud\/type'],LINK:metadata.labels['ethernet\.coreweave\.cloud/speed'],READY:status.conditions[?(@.type=='Ready')].status,CORDON:spec.unschedulable,TAINT:spec.taints[?(@.key=='qos.coreweave.cloud/interruptable')].effect,RELIABILITY:metadata.labels['node\.coreweave\.cloud\/reliability'],LG:metadata.labels['ib\.coreweave\.cloud\/leafgroup'],VERSION:metadata.labels['node\.coreweave\.cloud\/version'],IB:metadata.labels['ib\.coreweave\.cloud\/speed'],STATE:metadata.labels['node\.coreweave\.cloud\/state'],RESERVED:metadata.labels['node\.coreweave\.cloud\/reserved']"
create-secrets:
kubectl create secret generic llm-d-hf-token \
--from-literal=HF_TOKEN={{HF_TOKEN}} \
-n {{NAMESPACE}} \
--dry-run=client -o yaml \
| kubectl apply -n {{NAMESPACE}} -f - \
&& kubectl create secret generic gh-token-secret \
--from-literal=GH_TOKEN={{GH_TOKEN}} \
-n {{NAMESPACE}} \
--dry-run=client -o yaml \
| kubectl apply -n {{NAMESPACE}} -f - \
&& just create-registry-auth \
&& just patch-image-pull-secrets
create-registry-auth:
#!/usr/bin/env bash
echo "Creating registry auth secret..."
mkdir -p .tmp
AUTH_STRING=$(echo -n "{{QUAY_USERNAME}}:{{QUAY_PASSWORD}}" | base64)
cat > .tmp/auth.json << EOF
{
"auths": {
"quay.io": {
"auth": "${AUTH_STRING}"
}
}
}
EOF
kubectl create secret generic registry-auth \
--from-file=.dockerconfigjson=.tmp/auth.json \
--type=kubernetes.io/dockerconfigjson \
-n {{NAMESPACE}} \
--dry-run=client -o yaml \
| kubectl apply -n {{NAMESPACE}} -f -
rm -rf .tmp/auth.json
echo "Registry auth secret created!"
patch-image-pull-secrets:
#!/usr/bin/env bash
echo "Patching manifests with imagePullSecrets..."
# Add imagePullSecrets to decode.yaml if not present
if ! grep -q "imagePullSecrets:" {{EXAMPLE_DIR}}/manifests/modelserver/base/decode.yaml; then
sed -i.bak '/serviceAccountName: deepseek-r1/a\
imagePullSecrets:\
- name: registry-auth' {{EXAMPLE_DIR}}/manifests/modelserver/base/decode.yaml
rm -f {{EXAMPLE_DIR}}/manifests/modelserver/base/decode.yaml.bak
fi
# Add imagePullSecrets to prefill.yaml if not present
if ! grep -q "imagePullSecrets:" {{EXAMPLE_DIR}}/manifests/modelserver/base/prefill.yaml; then
sed -i.bak '/serviceAccountName: deepseek-r1/a\
imagePullSecrets:\
- name: registry-auth' {{EXAMPLE_DIR}}/manifests/modelserver/base/prefill.yaml
rm -f {{EXAMPLE_DIR}}/manifests/modelserver/base/prefill.yaml.bak
fi
echo "Manifests patched with imagePullSecrets!"
start-bench:
{{KN}} apply -f benchmark-interactive-pod.yaml
stop-bench:
{{KN}} delete pod benchmark-interactive --ignore-not-found
interact-bench:
mkdir -p ./.tmp \
&& echo "MODEL := \"{{MODEL}}\"" > .tmp/Justfile.remote.tmp \
&& sed -e 's#__BASE_URL__#\"http://infra-wide-ep-inference-gateway-istio.{{NAMESPACE}}.svc.cluster.local\"#g' \
-e 's#__ENDPOINT__#\"/v1/chat/completions\"#g' \
Justfile.remote >> .tmp/Justfile.remote.tmp \
&& kubectl cp .tmp/Justfile.remote.tmp {{NAMESPACE}}/benchmark-interactive:/app/Justfile \
&& kubectl cp ./run.sh {{NAMESPACE}}/benchmark-interactive:/app/run.sh \
&& {{KN}} exec -it benchmark-interactive -- /bin/bash
run-bench name in_tokens='128' out_tokens='2048' num_prompts='16384' concurrency_levels='8192 16384 32768':
mkdir -p ./.tmp \
&& echo $(date +%m%d%H%M) > .tmp/TIMESTAMP \
&& echo "{{name}}" > .tmp/NAME \
&& echo "MODEL := \"{{MODEL}}\"" > .tmp/Justfile.remote \
&& sed -e 's#__BASE_URL__#\"http://infra-wide-ep-inference-gateway-istio.{{NAMESPACE}}.svc.cluster.local\"#g' \
-e 's#__ENDPOINT__#\"/v1/chat/completions\"#g' \
Justfile.remote >> .tmp/Justfile.remote \
&& kubectl cp .tmp/TIMESTAMP {{NAMESPACE}}/benchmark-interactive:/app/TIMESTAMP \
&& kubectl cp .tmp/NAME {{NAMESPACE}}/benchmark-interactive:/app/NAME \
&& kubectl cp .tmp/Justfile.remote {{NAMESPACE}}/benchmark-interactive:/app/Justfile \
&& kubectl cp ./run.sh {{NAMESPACE}}/benchmark-interactive:/app/run.sh \
&& kubectl cp {{EXAMPLE_DIR}}/inferencepool.values.yaml {{NAMESPACE}}/benchmark-interactive:/app/values.yaml \
&& {{KN}} exec benchmark-interactive -- \
env INPUT_TOKENS={{in_tokens}} \
OUTPUT_TOKENS={{out_tokens}} \
NUM_PROMPTS={{num_prompts}} \
CONCURRENCY_LEVELS="{{concurrency_levels}}" \
bash /app/run.sh
cp-results:
kubectl cp benchmark-interactive:/app/results/$(cat ./.tmp/TIMESTAMP) \
results/$(cat ./.tmp/TIMESTAMP)
set-namespace:
kubectl config set-context --current --namespace={{NAMESPACE}}
get-pods:
{{KN}} get pods
describe name='':
scripts/describe-pod.sh {{NAMESPACE}} "{{name}}"
stern name='':
scripts/stern-pod.sh {{NAMESPACE}} "{{name}}"
status:
watch -n 2 {{KN}} get pods
start:
kubectl apply -k {{EXAMPLE_DIR}}/manifests/modelserver/coreweave -n {{NAMESPACE}} \
&& helm install deepseek-r1 \
-n {{NAMESPACE}} \
-f {{EXAMPLE_DIR}}/inferencepool.values.yaml \
--set "provider.name=istio" \
--set "inferenceExtension.monitoring.prometheus.enable=true" \
oci://us-central1-docker.pkg.dev/k8s-staging-images/gateway-api-inference-extension/charts/inferencepool --version v1.0.1 \
&& kubectl apply -k {{EXAMPLE_DIR}}/manifests/gateway/istio -n {{NAMESPACE}}
wait-ready timeout='600s':
#!/usr/bin/env bash
echo "⏳ Waiting for pods to be ready (timeout: {{timeout}})..."
kubectl wait --for=condition=ready pod \
-l llm-d.ai/role=prefill \
-n {{NAMESPACE}} \
--timeout={{timeout}} && \
kubectl wait --for=condition=ready pod \
-l llm-d.ai/role=decode \
-n {{NAMESPACE}} \
--timeout={{timeout}}
if [ $? -eq 0 ]; then
echo "✅ All model server pods are ready!"
{{KN}} get pods
# Send notification with sound (cross-platform)
if [[ "$OSTYPE" == "darwin"* ]]; then
osascript -e 'display notification "All model server pods are ready!" with title "llm-d-utils" sound name "Glass"' 2>/dev/null
afplay /System/Library/Sounds/Glass.aiff 2>/dev/null &
elif command -v notify-send &> /dev/null; then
notify-send "llm-d-utils" "All model server pods are ready!" 2>/dev/null &
fi
else
echo "⚠️ Timeout waiting for pods"
if [[ "$OSTYPE" == "darwin"* ]]; then
osascript -e 'display notification "Timeout waiting for pods" with title "llm-d-utils" sound name "Basso"' 2>/dev/null
afplay /System/Library/Sounds/Basso.aiff 2>/dev/null &
elif command -v notify-send &> /dev/null; then
notify-send "llm-d-utils" "Timeout waiting for pods" -u critical 2>/dev/null &
fi
fi
stop:
helm uninstall deepseek-r1 -n {{NAMESPACE}} \
&& kubectl delete -k {{EXAMPLE_DIR}}/manifests/modelserver/coreweave -n {{NAMESPACE}} \
&& kubectl delete -k {{EXAMPLE_DIR}}/manifests/gateway/istio -n {{NAMESPACE}}
restart:
just stop && just start
restart-bench:
just stop-bench && just start-bench
print-results DIR STR:
grep "{{STR}}" {{DIR}}/*.log \
| awk -F'[/_]' '{print $3, $0}' \
| sort -n \
| cut -d' ' -f2-
print-throughput DIR:
just print-results {{DIR}} "Output token throughput"
print-tpot DIR:
just print-results {{DIR}} "Median TPOT"
start-build-pod:
{{KN}} delete pod buildah-build --ignore-not-found
{{KN}} apply -f buildah-build-pod.yaml
{{KN}} wait --for=condition=ready pod/buildah-build --timeout=120s
build-image vllm_commit tag='custom' use_sccache='false':
{{KN}} exec buildah-build -- bash -c ' \
dnf install -y git make && \
rm -rf /tmp/llm-d && \
git clone https://github.com/llm-d/llm-d.git /tmp/llm-d && \
cd /tmp/llm-d && \
sed -i "s|ARG VLLM_COMMIT_SHA=.*|ARG VLLM_COMMIT_SHA=\"{{vllm_commit}}\"|" docker/Dockerfile.cuda && \
if [ "{{use_sccache}}" = "false" ]; then \
sed -i "s|CC=\"sccache gcc\"|CC=\"gcc\"|g" docker/Dockerfile.cuda && \
sed -i "s|CXX=\"sccache g++\"|CXX=\"g++\"|g" docker/Dockerfile.cuda; \
fi && \
buildah build \
--build-arg USE_SCCACHE={{use_sccache}} \
-t {{QUAY_REGISTRY}}/llm-d-cuda-dev:{{tag}} \
-f docker/Dockerfile.cuda . && \
buildah push {{QUAY_REGISTRY}}/llm-d-cuda-dev:{{tag}}'
stop-build-pod:
{{KN}} delete pod buildah-build --ignore-not-found
update-image tag:
#!/usr/bin/env bash
IMAGE="{{QUAY_REGISTRY}}/llm-d-cuda-dev:{{tag}}"
echo "Updating manifests to use image: $IMAGE"
sed -i.bak "s|image: .*llm-d-cuda-dev.*|image: $IMAGE|" {{EXAMPLE_DIR}}/manifests/modelserver/base/decode.yaml
sed -i.bak "s|image: .*llm-d-cuda-dev.*|image: $IMAGE|" {{EXAMPLE_DIR}}/manifests/modelserver/base/prefill.yaml
rm -f {{EXAMPLE_DIR}}/manifests/modelserver/base/*.bak
echo "Updated decode.yaml and prefill.yaml"
echo "Run 'just restart' to deploy with the new image"