Skip to content

Commit e53fe60

Browse files
authored
bug: workshop maps build from the map VPK inside the addon VPK Steam downloads them as (#859)
1 parent e2d7b2e commit e53fe60

6 files changed

Lines changed: 321 additions & 49 deletions

File tree

‎docs/3d-replay-map-meshes.md‎

Lines changed: 21 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -217,19 +217,32 @@ For each `name=id`:
217217
the workshop copy, the assets come from the install's);
218218
- `_night` variants are skipped, as for the install (consumers fold them
219219
onto the base map);
220-
- otherwise `<dir>/<id>/<name>.vpk`, or the item's only map VPK, is built,
221-
published under `name` (the maps-table name the consumers look up, which
222-
for a map added by hand is its numeric id) and read under the name inside
223-
the VPK (its file name);
220+
- otherwise `<dir>/<id>/<name>.vpk`, or the item's only VPK, is opened.
221+
Steam downloads an item as an addon VPK named after its id (`<id>.vpk`, or
222+
`<id>_dir.vpk` with `<id>_000.vpk`… beside it), and the map's own VPK is an
223+
entry inside it, `maps/<map>.vpk`, next to its skybox's
224+
(`maps/de_rooftop_marius_skybox.vpk`). The entry named `name`, else the only
225+
one that is not a skybox, is unpacked to TMPDIR for the build (an item that
226+
is itself a map VPK is read where it is). The map is published under `name`
227+
(the maps-table name the consumers look up, which for a map added by hand is
228+
its numeric id) and read under the name inside the item;
224229
- an item this node never downloaded is skipped and keeps whatever entry
225230
the base manifest has: it is not dropped. Play the map once on a server on
226231
that node, or run the build on a node that has it.
227232

228-
`source.vpk_sha256` is then the workshop VPK's hash, so a workshop update
229-
rebuilds the map at the next run.
233+
`source.vpk_sha256` is then the hash of the item's VPK (`<id>_dir.vpk`, whose
234+
directory holds every entry's CRC), so a workshop update rebuilds the map at
235+
the next run, and an unchanged item is never unpacked.
230236

231-
The lower-level scripts take a single VPK too:
232-
`node scripts/extract-map-meshes.mjs --vpk <dir>/<id>/<map>.vpk`, and the same
237+
Prophunt items (Prophunt Inferno, Mirage and Office) are overlays: their map is
238+
a 12-triangle box plus prop templates, and the world players walk on is the
239+
stock map's. They build, but what they publish is that box.
240+
241+
The lower-level scripts take a single map VPK too, unpacked first from a
242+
workshop item:
243+
`Source2Viewer-CLI -i <dir>/<id>/<id>_dir.vpk --vpk_filepath maps/<map>.vpk -o <out> -d`
244+
(with `<out>` an existing folder), then
245+
`node scripts/extract-map-meshes.mjs --vpk <out>/maps/<map>.vpk`, and the same
233246
for `extract-map-callouts.mjs` and `extract-map-blast.mjs`.
234247

235248
### Prep

‎scripts/build-map-assets.mjs‎

Lines changed: 77 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,8 @@
3535
// game server's workshop downloads: --workshop <dir>, by default
3636
// <install>/game/bin/linuxsteamrt64/steamapps/workshop/content/730, holding
3737
// one <id>/ folder per item. A map is published under its maps-table name and
38-
// read from its VPK under the name inside it. A copy in the install wins over
38+
// read under the name inside the item, from the map VPK the item's addon VPK
39+
// carries (unpacked to TMPDIR for the build). A copy in the install wins over
3940
// a workshop one; `_night` variants are skipped (consumers fold them onto the
4041
// base map); an item this node never downloaded keeps whatever entry the base
4142
// manifest has.
@@ -92,12 +93,14 @@ import { createHash } from "node:crypto";
9293
import {
9394
existsSync,
9495
mkdirSync,
96+
mkdtempSync,
9597
readdirSync,
9698
readFileSync,
9799
realpathSync,
98100
rmSync,
99101
writeFileSync,
100102
} from "node:fs";
103+
import { tmpdir } from "node:os";
101104
import { basename, dirname, join } from "node:path";
102105
import { fileURLToPath } from "node:url";
103106
import { gzipSync } from "node:zlib";
@@ -127,7 +130,7 @@ import {
127130
retryMaps,
128131
sameContent,
129132
} from "./lib-manifest.mjs";
130-
import { resolveCli } from "./lib-s2v.mjs";
133+
import { decompile, listVpk, missingOutput, resolveCli } from "./lib-s2v.mjs";
131134
import { loadPrep, makePrep, prepFileName } from "./replay-stage/build-prep.mjs";
132135

133136
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
@@ -238,9 +241,10 @@ function workshopList(text) {
238241
}
239242

240243
/**
241-
* The VPK a workshop item was downloaded as, and the map's name inside it:
242-
* `<root>/<id>/<name>.vpk` when there is one, else the item's only map VPK.
243-
* null when the item is not there, or holds several VPKs none named `name`.
244+
* The VPK a workshop item was downloaded as: `<root>/<id>/<name>.vpk` when
245+
* there is one, else the item's only VPK (`<id>.vpk`, or `<id>_dir.vpk` with
246+
* its chunks beside it). null when the item is not there, or holds several
247+
* VPKs none named `name`.
244248
*/
245249
export function workshopVpk(root, id, name) {
246250
const dir = join(root, id);
@@ -251,7 +255,45 @@ export function workshopVpk(root, id, name) {
251255
const pick =
252256
vpks.find((f) => f === `${name}.vpk` || f === `${name}_dir.vpk`) ??
253257
(vpks.length === 1 ? vpks[0] : null);
254-
return pick ? { vpk: join(dir, pick), inner: pick.replace(/(_dir)?\.vpk$/, "") } : null;
258+
return pick ? { vpk: join(dir, pick) } : null;
259+
}
260+
261+
/**
262+
* The map VPK of a workshop item and the map's name inside it. Steam
263+
* downloads an item as an addon VPK that carries the map's own VPK as
264+
* maps/<map>.vpk, beside its skybox's, so that one is unpacked into `dir`;
265+
* an item that is itself a map VPK is read where it is. The map is the one
266+
* named `name`, else the only one that is not a skybox.
267+
*/
268+
export function workshopMap(cli, vpk, name, dir) {
269+
const maps = [];
270+
for (const path of listVpk(cli, vpk, "maps/")) {
271+
const nested = /^maps\/([^/]+)\.vpk$/.exec(path)?.[1];
272+
const own = /^maps\/([^/]+)\/world_physics\.vmdl_c$/.exec(path)?.[1];
273+
if (nested || own) {
274+
maps.push({ inner: nested ?? own, nested: Boolean(nested) });
275+
}
276+
}
277+
const playable = maps.filter((m) => !m.inner.includes("skybox"));
278+
const pick =
279+
maps.find((m) => m.inner === name) ?? (playable.length === 1 ? playable[0] : null);
280+
if (!pick) {
281+
throw new Error(
282+
playable.length
283+
? `${basename(vpk)} holds ${playable.map((m) => m.inner).join(", ")}, none of them ${name}`
284+
: `${basename(vpk)} holds no map`,
285+
);
286+
}
287+
if (!pick.nested) {
288+
return { vpk, inner: pick.inner };
289+
}
290+
const file = `maps/${pick.inner}.vpk`;
291+
const run = decompile(cli, vpk, file, dir);
292+
const unpacked = join(dir, file);
293+
if (!existsSync(unpacked)) {
294+
throw missingOutput(file, run);
295+
}
296+
return { vpk: unpacked, inner: pick.inner };
255297
}
256298

257299
/** Writes `body` raw and gzipped; returns the manifest's `{ file, sha256 }`. */
@@ -376,7 +418,7 @@ async function prepPatch(map, entry, { store, prep, assets, rawDir, filesDir })
376418
};
377419
}
378420

379-
async function buildMap(map, { from, rawDir, filesDir, cli, pak, prep, assets }) {
421+
async function buildMap(map, { from, scratch, rawDir, filesDir, cli, pak, prep, assets }) {
380422
const start = performance.now();
381423
for (const asset of assets) {
382424
rmSync(join(filesDir, asset.file(map)), { force: true });
@@ -389,9 +431,13 @@ async function buildMap(map, { from, rawDir, filesDir, cli, pak, prep, assets })
389431
viewFailed: false,
390432
};
391433

434+
let input = from;
392435
let meshes;
393436
try {
394-
meshes = await extractMap(map, { vpk: from.vpk, inner: from.inner, outDir: rawDir, cli });
437+
if (from.workshop) {
438+
input = workshopMap(cli, from.vpk, map, scratch);
439+
}
440+
meshes = await extractMap(map, { vpk: input.vpk, inner: input.inner, outDir: rawDir, cli });
395441
if (!meshes.tri) {
396442
throw new Error(`collision mesh over the cap (${meshes.note})`);
397443
}
@@ -404,7 +450,7 @@ async function buildMap(map, { from, rawDir, filesDir, cli, pak, prep, assets })
404450

405451
let callouts;
406452
try {
407-
callouts = extractCallouts(map, { vpk: from.vpk, inner: from.inner, pak, cli });
453+
callouts = extractCallouts(map, { vpk: input.vpk, inner: input.inner, pak, cli });
408454
} catch (error) {
409455
callouts = { failed: error.message };
410456
result.calloutsFailed = true;
@@ -441,8 +487,8 @@ async function buildMap(map, { from, rawDir, filesDir, cli, pak, prep, assets })
441487
let blasted = false;
442488
try {
443489
const blast = extractBlast(map, {
444-
vpk: from.vpk,
445-
inner: from.inner,
490+
vpk: input.vpk,
491+
inner: input.inner,
446492
cli,
447493
targets: callouts.bombTargets ?? [],
448494
callouts: callouts.callouts ?? [],
@@ -491,7 +537,7 @@ async function buildMap(map, { from, rawDir, filesDir, cli, pak, prep, assets })
491537
console.log(
492538
`${map.padEnd(18)} ${describe(meshes)}; callouts ${calloutNote}; ` +
493539
`prep ${prepNote}; blast ${blastNote}` +
494-
`${from.workshop ? `; workshop ${from.workshop} (${basename(from.vpk)})` : ""}\n` +
540+
`${from.workshop ? `; workshop ${from.workshop} (${basename(input.vpk)})` : ""}\n` +
495541
`${"".padEnd(18)} ${sizes.join(", ")} ` +
496542
`[${seconds(performance.now() - start)}: ${stages}]`,
497543
);
@@ -828,11 +874,25 @@ async function build(args) {
828874
results.push({ map, unchanged: true });
829875
continue;
830876
}
831-
const result = await buildMap(map, { from, rawDir, filesDir, cli, pak, prep, assets });
832-
results.push({
833-
...result,
834-
source: { ...source, ...(result.files.prep ? { prep: prep.PREP_VERSION } : {}) },
835-
});
877+
const scratch = mkdtempSync(join(tmpdir(), `map-${map}-`));
878+
try {
879+
const result = await buildMap(map, {
880+
from,
881+
scratch,
882+
rawDir,
883+
filesDir,
884+
cli,
885+
pak,
886+
prep,
887+
assets,
888+
});
889+
results.push({
890+
...result,
891+
source: { ...source, ...(result.files.prep ? { prep: prep.PREP_VERSION } : {}) },
892+
});
893+
} finally {
894+
rmSync(scratch, { recursive: true, force: true });
895+
}
836896
}
837897
const attempted = results.filter((r) => !r.unchanged && !r.patch);
838898
const fresh = attempted.filter((r) => !r.triFailed).length;

‎scripts/lib-s2v.mjs‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
// `.cache/s2v/` unless CLI names one already on disk -- which is how these run
55
// in-cluster, where the inventory plugin's model extraction has already put one
66
// at /cs2-models/.work/cs2-model-extract/cli/Source2Viewer-CLI.
7-
import { execFileSync } from "node:child_process";
7+
import { execFileSync, spawnSync } from "node:child_process";
88
import {
99
closeSync,
1010
existsSync,
@@ -17,7 +17,7 @@ import {
1717
rmSync,
1818
} from "node:fs";
1919
import { tmpdir } from "node:os";
20-
import { dirname, join, resolve } from "node:path";
20+
import { basename, dirname, join, resolve } from "node:path";
2121
import { fileURLToPath } from "node:url";
2222

2323
const cliDir = join(dirname(fileURLToPath(import.meta.url)), "..", ".cache", "s2v");
@@ -149,6 +149,38 @@ export function missingOutput(what, run) {
149149
);
150150
}
151151

152+
/**
153+
* The paths in `vpk` under `prefix`, a folder ending in a slash. A VPK the CLI
154+
* cannot read still exits 0, with the exception on stderr, so a listing that
155+
* comes back empty beside one is that failure, not an empty folder.
156+
*/
157+
export function listVpk(cli, vpk, prefix) {
158+
const scratch = mkdtempSync(join(tmpdir(), "s2v-"));
159+
try {
160+
const run = spawnSync(
161+
cli.includes("/") ? resolve(cli) : cli,
162+
["-i", resolve(vpk), "--vpk_list", "--vpk_filepath", prefix],
163+
{ cwd: scratch, encoding: "utf8", maxBuffer: 64 * 1024 * 1024 },
164+
);
165+
const paths = (run.stdout ?? "")
166+
.split(/\r?\n/)
167+
.filter((line) => / CRC:\S+ size:\d+$/.test(line))
168+
.map((line) => line.split(" CRC:")[0]);
169+
const stderr = tailLines(run.stderr ?? "", STDERR_LINES);
170+
const failure =
171+
run.error?.message ?? (run.status !== 0 ? `exit ${run.status ?? run.signal}` : null);
172+
if (failure || (!paths.length && stderr.length)) {
173+
throw new Error(
174+
`Source2Viewer-CLI could not list ${basename(vpk)}` +
175+
`${failure ? ` (${failure})` : ""}${stderr.length ? `: ${stderr.join(" | ")}` : ""}`,
176+
);
177+
}
178+
return paths;
179+
} finally {
180+
rmSync(scratch, { recursive: true, force: true });
181+
}
182+
}
183+
152184
/**
153185
* Decompile one VPK entry, or a whole folder if the path ends in a slash.
154186
* `extra` carries export flags such as --gltf_export_format. Returns the tail

‎tests/helpers/fakeMapTools.ts‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,23 +9,48 @@ import { vi } from "vitest";
99
// `{ stdout, stderr, repeat }`, printed before exiting (stdout `repeat`
1010
// times), with stderr also appended to ./exceptions.txt the way the real CLI
1111
// logs what it swallows. FAKE_S2V_VERSION is what --version reports; every
12-
// export is appended to FAKE_S2V_LOG, and the directory it ran in to
13-
// FAKE_S2V_CWD_LOG, when set.
12+
// export is appended to FAKE_S2V_LOG, its input VPK's name and the export to
13+
// FAKE_S2V_INPUT_LOG, and the directory it ran in to FAKE_S2V_CWD_LOG, when
14+
// set. FAKE_S2V_LIST maps an input VPK's name to the paths --vpk_list prints;
15+
// a listing is logged to FAKE_S2V_INPUT_LOG as `<vpk> --vpk_list <prefix>` and
16+
// prints FAKE_S2V_OUTPUT["--vpk_list"] too.
1417
const FAKE_CLI = `
1518
import { appendFileSync, copyFileSync, mkdirSync, writeSync } from "node:fs";
16-
import { dirname, join } from "node:path";
19+
import { basename, dirname, join } from "node:path";
1720
const args = process.argv.slice(2);
1821
if (args.includes("--version")) {
1922
console.log("Version: " + (process.env.FAKE_S2V_VERSION ?? "fake-1"));
2023
console.log("OS: whatever the machine is");
2124
process.exit(0);
2225
}
2326
const at = (flag) => args[args.indexOf(flag) + 1];
27+
if (args.includes("--vpk_list")) {
28+
const prefix = args.includes("--vpk_filepath") ? at("--vpk_filepath") : "";
29+
if (process.env.FAKE_S2V_INPUT_LOG) {
30+
appendFileSync(
31+
process.env.FAKE_S2V_INPUT_LOG,
32+
basename(at("-i")) + " --vpk_list " + prefix + "\\n",
33+
);
34+
}
35+
const listing = JSON.parse(process.env.FAKE_S2V_OUTPUT ?? "{}")["--vpk_list"] ?? {};
36+
if (listing.stderr) {
37+
writeSync(2, listing.stderr);
38+
appendFileSync("exceptions.txt", listing.stderr);
39+
}
40+
const listed = JSON.parse(process.env.FAKE_S2V_LIST ?? "{}")[basename(at("-i"))] ?? [];
41+
for (const path of listed.filter((p) => p.startsWith(prefix))) {
42+
console.log(path + " CRC:0000000000 size:1");
43+
}
44+
process.exit(0);
45+
}
2446
const filepath = at("--vpk_filepath");
2547
const format = args.includes("--gltf_export_format") ? at("--gltf_export_format") : null;
2648
if (process.env.FAKE_S2V_LOG) {
2749
appendFileSync(process.env.FAKE_S2V_LOG, filepath + "\\n");
2850
}
51+
if (process.env.FAKE_S2V_INPUT_LOG) {
52+
appendFileSync(process.env.FAKE_S2V_INPUT_LOG, basename(at("-i")) + " " + filepath + "\\n");
53+
}
2954
if (process.env.FAKE_S2V_CWD_LOG) {
3055
appendFileSync(process.env.FAKE_S2V_CWD_LOG, process.cwd() + "\\n");
3156
}

0 commit comments

Comments
 (0)