From c9cf6432e41dc5600a793337bac4bb37c6b0b58e Mon Sep 17 00:00:00 2001 From: yang you <119789235+REDrighthand05@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:52:20 +0800 Subject: [PATCH] Speed up Base depth and preserve Windows reconstruction preview on resize --- .github/workflows/desktop-runtime.yml | 34 ++++ app.py | 32 ++- app/echo_scene/pipeline.py | 191 +++++++++++------- app/web_app.py | 53 +++-- docs/WINDOWS_RECONSTRUCTION_PERFORMANCE.md | 72 +++++++ echo_scene/depth.py | 101 ++++++++-- echo_scene/depth_batch.py | 216 +++++++++++++-------- echo_scene/pipeline.py | 183 +++++++++++------ echo_scene/voxel_export.py | 58 +++++- installer/version.inc | 4 +- tests/test_depth_dispatch.py | 99 ++++++++++ tests/test_native_window.py | 76 ++++++++ tests/test_scene_export_stream.py | 39 ++++ tests/test_window_resize.py | 46 +++++ web_app.py | 132 ++++++++++--- web_ui/app.js | 66 +++++-- 16 files changed, 1092 insertions(+), 310 deletions(-) create mode 100644 .github/workflows/desktop-runtime.yml create mode 100644 docs/WINDOWS_RECONSTRUCTION_PERFORMANCE.md create mode 100644 tests/test_depth_dispatch.py create mode 100644 tests/test_native_window.py create mode 100644 tests/test_scene_export_stream.py create mode 100644 tests/test_window_resize.py diff --git a/.github/workflows/desktop-runtime.yml b/.github/workflows/desktop-runtime.yml new file mode 100644 index 00000000..4b48d909 --- /dev/null +++ b/.github/workflows/desktop-runtime.yml @@ -0,0 +1,34 @@ +name: Desktop runtime regression +on: + pull_request: + branches: [main] + paths: + - 'web_ui/**' + - 'web_app.py' + - 'app.py' + - 'echo_scene/depth.py' + - 'echo_scene/depth_batch.py' + - 'echo_scene/pipeline.py' + - 'tests/test_scene_export_stream.py' + - 'tests/test_window_resize.py' + - 'tests/test_native_window.py' + - 'tests/test_depth_dispatch.py' + - '.github/workflows/desktop-runtime.yml' +permissions: + contents: read +jobs: + windows: + runs-on: windows-latest + timeout-minutes: 12 + env: + PYTHONUTF8: "1" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install numpy pillow onnxruntime-directml playwright pywebview + - run: python tests/test_depth_dispatch.py + - run: python tests/test_scene_export_stream.py + - run: python tests/test_window_resize.py + - run: python tests/test_native_window.py diff --git a/app.py b/app.py index c8f7aa1a..37ff2315 100644 --- a/app.py +++ b/app.py @@ -93,6 +93,7 @@ def run_pipeline(video: Path, jobs_root: Path, log, job_sink=None) -> Path: """Run every stage in order. log(str) receives human progress lines; job_sink(job_dir), when given, is called once the job folder exists (the live preview watches it from there).""" + started = time.perf_counter() from echo_scene.pipeline import stage_depth, stage_extract, stage_present, stage_select from echo_scene.pipeline import stage_clean, stage_mesh, stage_poses, stage_preview, stage_replay, stage_scale, stage_voxel from echo_scene.fuse import stage_fuse @@ -144,11 +145,40 @@ def run_pipeline(video: Path, jobs_root: Path, log, job_sink=None) -> Path: ("正在生成过程回放", lambda: stage_preview(job)), ("正在生成路线回放", lambda: stage_replay(job)), ] + timings = [] + cpu_stages = {"select", "fuse", "clean", "scale", "mesh", + "voxel", "present", "preview", "replay"} for label, fn in steps: log(label + "…") + tick = time.perf_counter() info = fn() - if isinstance(info, dict) and info.get("ok") is False: + entry = {"stage": info.get("stage"), "seconds": round(time.perf_counter() - tick, 4)} + for key in ("device", "depth_runtime", "frame_count", "registered_images", "depth_frames"): + if key in info: + entry[key] = info[key] + if entry["stage"] in cpu_stages: + entry.setdefault("device", "CPU") + if isinstance(info.get("depth_runtime"), dict): + runtime = info["depth_runtime"] + observed = runtime["profiled_model_node_providers"] + configured = runtime["session_providers_at_end"] + providers = ", ".join(observed) + if not providers and configured == ["CPUExecutionProvider"]: + providers = "CPU" + log(f"深度推理:{providers or '执行节点未验证'},CUDA 执行证据:{runtime['cuda_execution']}。") + if info.get("scene_html") and entry["stage"] == "present": + entry["scene_written_at_seconds"] = round(time.perf_counter() - started, 4) + timings.append(entry) + if info.get("ok") is False: raise RuntimeError(f"{label}失败:{info.get('reason', '未知原因')}") + total = round(time.perf_counter() - started, 4) + try: + (job / "reconstruction-timing.json").write_text( + json.dumps({"total_seconds": total, "stages": timings}, ensure_ascii=False, indent=2), + encoding="utf-8") + except OSError as exc: + log(f"无法保存重建计时:{exc}") + log(f"重建耗时 {total:.2f} 秒。") log("完成。") return job diff --git a/app/echo_scene/pipeline.py b/app/echo_scene/pipeline.py index 3d4a288c..ae008fa9 100644 --- a/app/echo_scene/pipeline.py +++ b/app/echo_scene/pipeline.py @@ -31,10 +31,14 @@ import base64 import json import math +import os +import re import shutil import subprocess import sys +import tempfile from pathlib import Path +from time import perf_counter import numpy as np @@ -509,6 +513,7 @@ def stage_extract(video: Path, job_dir: Path) -> dict: thumbs=thumbs) info = { "stage": "extract", + "device": "CPU (software decode, resize and JPEG)", "video": str(video), "duration_seconds": round(duration, 2), "frame_count": len(frames), @@ -540,6 +545,7 @@ def stage_extract(video: Path, job_dir: Path) -> dict: if not accel: raise globals()["_HWACCEL_CACHE"] = [] + accel = [] _run([_tool_path("ffmpeg"), "-y", "-v", "error", "-i", video, *tail]) frames = sorted(frames_dir.glob("frame_*.jpg")) @@ -549,6 +555,8 @@ def stage_extract(video: Path, job_dir: Path) -> dict: "stage": "extract", "video": str(video), "duration_seconds": round(duration, 2), + "device": (f"FFmpeg {accel[-1]} decode requested; CPU frame output" + if accel else "CPU (software decode and JPEG)"), "frame_interval_seconds": round(interval, 3), "frame_count": len(frames), "frames_dir": str(frames_dir), @@ -1005,6 +1013,8 @@ def stage_poses(job_dir: Path) -> dict: info = { "stage": "poses", "ok": True, + "device": ("COLMAP automatic device selection; check COLMAP SIFT/BA log" + if getattr(pycolmap, "has_cuda", False) else "CPU (COLMAP)"), "registered_images": rec.num_reg_images(), "relocalized_images": len(relocalized), "match_pairs": match_pairs, @@ -1030,7 +1040,10 @@ def stage_select(job_dir: Path, max_frames: int = 120, min_sharpness: float = 20 def stage_depth(job_dir: Path) -> dict: - from .depth import estimate_depth, save_depth_preview + from .depth import (_GPU_PROVIDERS, _get_session, _provider_chain, + end_provider_profile, estimate_depth, save_depth_preview) + from .depth_batch import (_pixel_batch_dimension, iter_depth_estimates, + iter_depth_estimates_parallel, parallel_workers) # The Base model gives visibly smoother, more continuous surfaces # than Small (verified on the room job); fall back to Small when the @@ -1050,56 +1063,90 @@ def stage_depth(job_dir: Path) -> dict: preview_dir = job_dir / "depth_preview" depth_dir.mkdir(exist_ok=True) preview_dir.mkdir(exist_ok=True) - # Batched inference (DS kernel, merge-gated in - # tests/test_depth_batch.py), streamed chunk by chunk so peak - # memory stays bounded no matter how many frames registered: - # the pre-streaming batch call held every depth map in RAM - # (~6.4GB at 773 full-res frames) and was OOM-killed with zero - # output. Falls back to per-frame on any batch-level failure - # (from wherever streaming stopped), per frame on a batch miss. - existing = [n for n in names if (frames_dir / n).exists()] - done = [] + existing = [name for name in names if (frames_dir / name).exists()] + paths = [frames_dir / name for name in existing] + runtime = {"requested_providers": _provider_chain(), "cpu_write_seconds": 0.0} + done = set() + profile_all = os.environ.get("ECHO_DEPTH_PROFILE") == "1" + profile_prefix = None + if paths and (profile_all or _GPU_PROVIDERS.intersection(runtime["requested_providers"])): + profile_prefix = (job_dir / "ort-depth").resolve() + tick = perf_counter() + session = _get_session(model, profile_prefix=profile_prefix) + runtime["session_init_seconds"] = perf_counter() - tick + initial_providers = session.get_providers() + runtime["session_providers_at_start"] = initial_providers + runtime["model_batch_dimension"] = _pixel_batch_dimension(session) + has_gpu = bool(_GPU_PROVIDERS.intersection(initial_providers)) + workers = 0 if profile_all else parallel_workers(model, session, len(paths)) + runtime["cpu_worker_sessions"] = workers + # Keep the existing maximum batch on dynamic models. Smaller cards + # can select one frame, and a rejected dynamic batch downgrades once. + max_batch = os.environ.get("ECHO_DEPTH_BATCH_SIZE", "4") + batch_size = int(max_batch) if max_batch in {"1", "2", "3", "4"} else 4 + runtime["requested_batch_size"] = batch_size + runtime["cpu_prefetch"] = has_gpu and os.environ.get("ECHO_DEPTH_PREFETCH") != "0" + runtime["cpu_fallback_frames"] = 0 + providers_seen = {} + profile_open = profile_prefix is not None + cpu_session = None def _save_depth(name, depth): + tick = perf_counter() stem = Path(name).stem np.save(depth_dir / f"{stem}.npy", depth) save_depth_preview(depth, preview_dir / f"{stem}.png") - done.append(name) - - streamed = True + runtime["cpu_write_seconds"] += perf_counter() - tick + done.add(name) + + def _retry(path): + nonlocal cpu_session + if not has_gpu: + return estimate_depth(path, model, session=session) + if cpu_session is None: + cpu_session = _get_session(model, providers=["CPUExecutionProvider"]) + runtime["cpu_fallback_frames"] += 1 + return estimate_depth(path, model, session=cpu_session) + + estimates = (iter_depth_estimates_parallel(paths, model, workers) if workers + else iter_depth_estimates(paths, session, batch_size=batch_size, + prefetch=runtime["cpu_prefetch"], timings=runtime)) try: - from .depth import _get_session - from .depth_batch import (iter_depth_estimates, - iter_depth_estimates_parallel, - parallel_workers) - - session = _get_session(model) - # Thread-parallel depth when it pays (CPU session, enough - # frames, enough cores): one 1-thread session per worker - # thread, ~1.3x on the real 40-frame check, outputs - # bit-identical to serial single-frame. Any failure here -- - # including one mid-stream -- falls through to the serial - # per-frame loop below, which re-runs only the frames not - # yet saved. - workers = parallel_workers(model, session, len(existing)) - if workers: - estimates = iter_depth_estimates_parallel( - [frames_dir / n for n in existing], model, workers) - else: - estimates = iter_depth_estimates( - [frames_dir / n for n in existing], session, batch_size=4) - for i, depth in estimates: - if depth is None: - depth = estimate_depth(frames_dir / existing[i], model) - _save_depth(existing[i], depth) - except Exception: - streamed = False - if not streamed: - have = set(done) - for name in existing: - if name not in have: - _save_depth(name, estimate_depth(frames_dir / name, model)) - info = {"stage": "depth", "ok": True, "depth_frames": len(done), "model": model.name} + try: + for index, depth in estimates: + if profile_open and not profile_all and depth is not None: + providers_seen = end_provider_profile(session) + profile_open = False + if depth is None: + depth = _retry(paths[index]) + _save_depth(existing[index], depth) + except Exception as exc: + # Retry only unfinished frames. Saving and error reporting + # still fail the job if a frame cannot be produced. + estimates.close() + runtime["stream_error"] = str(exc) + for path in paths: + if path.name not in done: + _save_depth(path.name, _retry(path)) + finally: + estimates.close() + if profile_open: + providers_seen = end_provider_profile(session, keep_file=profile_all) + + runtime["session_providers_at_end"] = session.get_providers() + runtime["profiled_model_node_providers"] = providers_seen + runtime["provider_profile_scope"] = ("full inference session" if profile_all else + "first successful group" if profile_prefix else "off") + if providers_seen.get("CUDAExecutionProvider"): + runtime["cuda_execution"] = "observed" + elif ("CUDAExecutionProvider" not in initial_providers + or (profile_all and providers_seen)): + runtime["cuda_execution"] = "not_used" + else: + runtime["cuda_execution"] = "unverified" + info = {"stage": "depth", "ok": True, "depth_frames": len(done), "model": model.name, + "device": "ONNX inference; CPU preprocessing, resize and file output", + "depth_runtime": runtime} (job_dir / "job.json").write_text(json.dumps(info, ensure_ascii=False, indent=2)) return info @@ -1206,6 +1253,13 @@ def stage_scale(job_dir: Path) -> dict: return info +def _write_embedded_buffer(output, path): + # Keep full 3-byte groups across reads so padding only appears at EOF. + with path.open("rb") as source: + while block := source.read(3 * 256 * 1024): + output.write(base64.b64encode(block).decode("ascii")) + + def stage_present(job_dir: Path) -> dict: """Write the standalone scene page into the job folder. @@ -1244,34 +1298,35 @@ def stage_present(job_dir: Path) -> dict: "reason": "viewer template not found (looked in app/ and install root)"} (job_dir / "job.json").write_text(json.dumps(info, ensure_ascii=False, indent=2)) return info - html = template - points_b64 = base64.b64encode(points_bin.read_bytes()).decode() - html = html.replace( - "\n\n' + script) + cursor = 0 + for match in re.finditer(r"fetch\('(mesh_vertices\.bin|mesh_indices\.bin)'\)", after): + buffer_path = job_dir / match[1] + if buffer_path.is_file(): + output.write(after[cursor:match.start()] + "fetch('data:application/octet-stream;base64,") + _write_embedded_buffer(output, buffer_path) + output.write("')") + cursor = match.end() + output.write(after[cursor:]) + temp_path.replace(scene_path) + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) files = {"scene_html": str(scene_path)} for key, name in (("voxel_glb", "voxel_model.glb"), ("voxel_ply", "voxel_model.ply"), diff --git a/app/web_app.py b/app/web_app.py index e70d7b87..085ce527 100644 --- a/app/web_app.py +++ b/app/web_app.py @@ -282,6 +282,7 @@ def __init__(self): # maximise, so restore can put it back where it was self._inbox_sig = None self._cli_cursor = cli_events.load_cursor() + self._preview_lock = threading.Lock() self._preview_seq = 0 self._preview_data = None self._preview_stop = threading.Event() @@ -564,7 +565,7 @@ def get_snapshot(self): "cliEvents": self._drain_cli_events(recent_only=True), } - def poll(self): + def poll(self, known_preview_seq=None, include_preview=True): out = { "status": self.status, "frac": self.frac, @@ -573,7 +574,6 @@ def poll(self): "hasJob": self.job is not None, "depthReady": self.depth_out is not None, "logs": self._pending_log[:], - "previewSeq": self._preview_seq, "update": self.update, } self._pending_log = [] @@ -594,12 +594,11 @@ def poll(self): if isig != self._inbox_sig: self._inbox_sig = isig out["inbox"] = inbox - # v0.6.29: the preview frame used to ride every poll as a - # full base64 PNG, twice a second for the whole depth job -- - # while the page never reads it (it watches previewSeq and - # fetches the frame through get_preview_frame). That is the - # main jank during a depth job; nothing is sent here any - # more (get_preview_frame still serves the latest frame). + frame = self.get_preview_frame() + out["previewSeq"] = frame["seq"] + if (include_preview and known_preview_seq is not None + and known_preview_seq != frame["seq"] and frame["data"]): + out["preview"] = frame["data"] if self._manual_check_result is not None: out["manualCheck"] = dict(self._manual_check_result) if self._update_phase is not None: @@ -625,7 +624,14 @@ def poll(self): def get_preview_frame(self): """The latest preview frame (data URL) + its sequence.""" - return {"seq": self._preview_seq, "data": self._preview_data} + with self._preview_lock: + return {"seq": self._preview_seq, "data": self._preview_data} + + def _publish_preview_png(self, data): + url = "data:image/png;base64," + base64.b64encode(data).decode("ascii") + with self._preview_lock: + self._preview_data = url + self._preview_seq += 1 def get_toolbox(self): """The toolbox page's payload: every registered echo_super @@ -784,10 +790,7 @@ def _depth_stills_loop(self, stills_dir, stop): # (seen is only updated on publish). if _png_complete(data): seen = (newest.name, size) - self._preview_data = ( - "data:image/png;base64," - + base64.b64encode(data).decode()) - self._preview_seq += 1 + self._publish_preview_png(data) except Exception: pass stop.wait(0.8) @@ -1123,10 +1126,7 @@ def _preview_loop(self, job, stop): frame = Image.fromarray(img) buf = io.BytesIO() frame.save(buf, format="PNG") - self._preview_data = ( - "data:image/png;base64," - + base64.b64encode(buf.getvalue()).decode()) - self._preview_seq += 1 + self._publish_preview_png(buf.getvalue()) last_render, last_signature = now, signature stop.wait(0.7) except Exception: @@ -3288,17 +3288,16 @@ def main(): pass bridge = Bridge() - try: # say once, in the log, where depth compute will run - import onnxruntime as _ort + def report_depth_runtime(): + try: + from echo_scene.depth import depth_runtime_inventory - _provs = _ort.get_available_providers() - if ("DmlExecutionProvider" in _provs - or "CUDAExecutionProvider" in _provs): - bridge._log("显卡加速可用,深度计算会走显卡。") - else: - bridge._log("深度计算使用 CPU(未检测到显卡运行时)。") - except Exception: - pass + for line in depth_runtime_inventory(): + bridge._log(line) + except ImportError: + bridge._log("深度运行时无法导入。") + + threading.Thread(target=report_depth_runtime, daemon=True).start() # The page ships in the loose payload: resolve it next to this # file first (that is the copy hot updates replace), and only # fall back to the frozen resource path in a source tree. diff --git a/docs/WINDOWS_RECONSTRUCTION_PERFORMANCE.md b/docs/WINDOWS_RECONSTRUCTION_PERFORMANCE.md new file mode 100644 index 00000000..b4c841c3 --- /dev/null +++ b/docs/WINDOWS_RECONSTRUCTION_PERFORMANCE.md @@ -0,0 +1,72 @@ +# EchoScene reconstruction performance + +This change follows #74 and reuses the desktop window and stream-export work in #194. The baseline is [the v0.6.81 payload in #195](https://github.com/REDrighthand05/echo4d/pull/195), commit `1ba6bae3ac54918fe0e3b03e0d1a5c8687cac098`. To run that snapshot from source, copy its four partial hot-update files under `app/` over the matching files at repository root. The candidate already has those changes in the source layout. + +## Same-video comparison + +Input is the RGB stream of [TUM Freiburg1 desk](https://cvg.cit.tum.de/data/datasets/rgbd-dataset/download), originally 640×480. From the sorted RGB filenames, take Python slice `[45:139:3]`, copy those 32 images as numbered PNGs starting at 000 and encode with `ffmpeg -framerate 10 -i %03d.png -c:v libx264 -threads 2 -crf 18 -pix_fmt yuv420p tum-desk-32.mp4`. MP4 SHA-256 from this run is `6eb4aa7b106b83bb0872287ef393eaa96d2719536f12bc0bbf5de40ed1e481be`. The archive SHA-256 is `e983d6830916e66dc4a46a71368046b149b283de87769690e7aa4e0b9483530c`. Neither the depth stream nor the TUM trajectory was used. + +Linux, AMD EPYC 9V74 with 9 visible logical CPUs, Python 3.12.14, ONNX Runtime 1.32.0 and PyCOLMAP 4.2.1 CPU. Both runs used the shipped Base ONNX model (SHA-256 `dab40f67d5ead5e0ed52acfae41a717a708b02bc57f6cb761c5de828f26a7289`), `OPENBLAS_NUM_THREADS=2`, `OMP_NUM_THREADS=4`, `ECHO_DEPTH_PROVIDER=CPUExecutionProvider` and default scheduling. Each timed run starts in a fresh process and job directory. The desktop calls the same `app.run_pipeline` function. This test omits the live window and its preview thread. + +| Stage | v0.6.81 (s) | Candidate (s) | Device in this run | +| --- | ---: | ---: | --- | +| Decode and extract | 0.73 | 0.73 | CPU (two FFmpeg passes, JPEG output) | +| Sharpness selection | 0.13 | 0.15 | CPU | +| Camera trajectory | 12.56 | 12.74 | CPU (SIFT, matching, mapping and bundle adjustment) | +| Depth stage | 82.55 | 44.00 | CPU (Pillow/NumPy and ONNX Runtime) | +| Fusion | 1.11 | 1.20 | CPU (NumPy and disk spill) | +| Cleanup | 2.19 | 2.29 | CPU | +| Scale | 1.61 | 1.87 | CPU | +| Mesh | 6.47 | 8.51 | CPU (Open3D Poisson) | +| Voxel | 0.11 | 0.12 | CPU | +| Standalone scene export | 0.02 | 0.03 | CPU and disk | +| Preview and replay export | 0.28 | 0.28 | CPU and disk | +| End to end | **108.13** | **72.20** | **33.2% less time on this input** | + +The baseline used two single-thread CPU sessions for Base when at least eight frames were registered. A separate baseline run with `ECHO_DEPTH_WORKERS=0` took 70.36 s (38.73 s in depth), identifying scheduling as the main measured gain. The candidate defaults Base on CPU to one ORT session; Small keeps its existing scheduling, and an explicit worker override still applies. CPU input preparation took 0.39 s cumulatively in the baseline and 0.17 s in the candidate. The candidate spent 41.83 s inside ORT calls and 0.67 s writing depths and previews. The ORT call includes any input transfer, synchronization and output transfer on a GPU runtime. CPU preparation can overlap the current GPU group and file writes with one group of lookahead. A fixed single-frame graph is never fed a larger batch. Dynamic batches can drop to single frames after rejection without preprocessing those frames again. + +Both runs selected, registered, inferred and fused all 32 frames, and all 32 depth arrays were bit-for-bit equal. Independently estimated trajectories differed by 0.032% of the baseline path's bounding-box diagonal in aligned camera-center RMS, with 0.042° median orientation difference. Each exported HTML embedded the original point and mesh buffers byte-for-byte. The two runs exported 79,207 / 80,990 points and 249,999 triangles each. The point count varies with the independent COLMAP solves. + +## Windows and NVIDIA reproduction + +Keep the same MP4, model SHA-256 and Python dependencies for both worktrees. Put the Base model at `models/depth_anything_v2_base.onnx`, with FFmpeg and FFprobe on PATH. Open PowerShell at each worktree root: + +```powershell +$env:PYTHONUTF8 = '1' +$env:ECHO_DEPTH_PROVIDER = 'DmlExecutionProvider' +python -c "import sys; from pathlib import Path; import app; app.run_pipeline(Path(sys.argv[1]),Path(sys.argv[2]),print)" C:/data/tum-desk-32.mp4 C:/echo-ab/run-a +``` + +Use distinct output directories and repeat at least three times after a warm-up. To record baseline per-stage timings, use the same command with the timing wrapper below saved outside both checkouts. For the candidate, `reconstruction-timing.json` also records total time, time up to scene write, per-stage execution details and valid frame counts. + +```python +import functools +import sys +from pathlib import Path +from time import perf_counter + +sys.path.insert(0, str(Path.cwd())) +import app +from echo_scene import fuse, pipeline + +def timed(fn): + @functools.wraps(fn) + def run(*args, **kwargs): + start = perf_counter() + result = fn(*args, **kwargs) + print(result["stage"], round(perf_counter() - start, 3), flush=True) + return result + return run + +for module in (pipeline, fuse): + for name, fn in vars(module).copy().items(): + if name.startswith("stage_"): + setattr(module, name, timed(fn)) +start = perf_counter() +job = app.run_pipeline(Path(sys.argv[1]), Path(sys.argv[2]), print) +print("total", round(perf_counter() - start, 3), job) +``` + +The released installer uses DirectML. For a CUDA run, use a separate Python environment with `onnxruntime-gpu` and its compatible NVIDIA CUDA/cuDNN libraries in place of `onnxruntime-directml`, then set `$env:ECHO_DEPTH_PROVIDER = 'CUDAExecutionProvider'`. `nvidia-smi` device discovery, runtime provider availability and actual operator execution are reported separately. Default reporting samples initial inference when a GPU provider is requested. Set `$env:ECHO_DEPTH_PROFILE = '1'` for the full operator trace, in a separate diagnostic run because profiling changes timing. A CUDA model-node event confirms CUDA ran. The trace excludes memory-copy events from this conclusion. Use `ECHO_DEPTH_BATCH_SIZE=1` on low-VRAM systems and `ECHO_DEPTH_PREFETCH=0` to compare preparation overlap. Hardware decode in the legacy extraction route is logged as requested, while the normal dense decode is software. COLMAP may select a device when its wheel supports CUDA, and its SIFT and bundle-adjustment logs are authoritative for that stage. + +For the window check, leave a scene reconstructing in the desktop app, orbit the preview, maximize, minimize and restore. Check document identity, task progress, view and preview sequence. The Windows regression exercises the native WebView2 window and the Edge WebGL resize path. The Linux CPU run cannot measure Windows/CUDA throughput or interactive frame latency while a consumer GPU is busy. diff --git a/echo_scene/depth.py b/echo_scene/depth.py index 5010b8d4..20476594 100644 --- a/echo_scene/depth.py +++ b/echo_scene/depth.py @@ -1,8 +1,12 @@ """Monocular depth estimation with an ONNX Depth Anything V2 model.""" +import json import os +import shutil +import subprocess +from collections import Counter from pathlib import Path -from typing import Dict, Union +from typing import Union import numpy as np import onnxruntime as ort @@ -15,7 +19,7 @@ _MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32).reshape(1, 3, 1, 1) _STD = np.array([0.229, 0.224, 0.225], dtype=np.float32).reshape(1, 3, 1, 1) -_SESSIONS: Dict[str, ort.InferenceSession] = {} +_SESSIONS: dict[tuple[str, tuple[str, ...]], ort.InferenceSession] = {} # The active execution provider, recorded on first session creation # ("CPUExecutionProvider" until a GPU runtime is installed). @@ -25,11 +29,9 @@ def _provider_chain() -> list: """Best-first providers the installed onnxruntime actually has. - CUDA first, then DirectML (any DX12 GPU on Windows, no toolkit - needed), then CPU. The bundled runtime is CPU-only today, so this - resolves to CPU there and lights up by itself the day the GPU - runtime ships in the installer. ECHO_DEPTH_PROVIDER pins a choice - (benchmarking / debugging).""" + CUDA first when installed, then DirectML, then CPU. + ECHO_DEPTH_PROVIDER selects an available provider for comparison runs. + """ available = list(ort.get_available_providers()) forced = os.environ.get("ECHO_DEPTH_PROVIDER") if forced and forced in available: @@ -40,22 +42,90 @@ def _provider_chain() -> list: return chain -def _get_session(model_path: Union[str, os.PathLike]) -> ort.InferenceSession: +_GPU_PROVIDERS = {"CUDAExecutionProvider", "DmlExecutionProvider"} + + +def depth_runtime_inventory(): + tool = shutil.which("nvidia-smi") + names = [] + if tool: + try: + result = subprocess.run( + [tool, "--query-gpu=name", "--format=csv,noheader"], + capture_output=True, text=True, errors="replace", timeout=3, + check=True, creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + names = [line.strip() for line in result.stdout.splitlines() if line.strip()] + except (OSError, subprocess.SubprocessError): + pass + cards = ", ".join(names) if names else "未由 nvidia-smi 确认" + providers = ", ".join(ort.get_available_providers()) + return [f"NVIDIA 显卡检测:{cards}。", f"已安装的深度运行时 provider:{providers}。"] + + +def _get_session(model_path: Union[str, os.PathLike], *, providers=None, + profile_prefix=None) -> ort.InferenceSession: global ACTIVE_PROVIDER - key = os.fspath(model_path) - session = _SESSIONS.get(key) + model = os.fspath(model_path) + chain = _provider_chain() if providers is None else providers + key = (model, tuple(chain)) + session = _SESSIONS.get(key) if profile_prefix is None else None if session is None: - if not os.path.isfile(key): - raise FileNotFoundError(f"ONNX model file not found: {key}") - session = ort.InferenceSession(key, providers=_provider_chain()) + if not os.path.isfile(model): + raise FileNotFoundError(f"ONNX model file not found: {model}") + options = ort.SessionOptions() + if "DmlExecutionProvider" in chain: + options.enable_mem_pattern = False + options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + if profile_prefix is not None: + options.enable_profiling = True + options.profile_file_prefix = os.fspath(profile_prefix) + try: + session = ort.InferenceSession(model, options, providers=chain) + except (RuntimeError, ValueError): + if not _GPU_PROVIDERS.intersection(chain): + raise + return _get_session(model, providers=["CPUExecutionProvider"], + profile_prefix=profile_prefix) ACTIVE_PROVIDER = session.get_providers()[0] - _SESSIONS[key] = session + if profile_prefix is None: + _SESSIONS[key] = session return session +def end_provider_profile(session, *, keep_file=False): + """Read operator execution events, excluding data transfers.""" + path = None + try: + filename = session.end_profiling() + if not filename: + return {} + path = Path(filename) + events = json.loads(path.read_text(encoding="utf-8")) + counts = Counter() + for event in events: + args = event.get("args", {}) + if (event.get("cat") == "Node" + and event.get("name", "").endswith("_kernel_time") + and args.get("op_name") + and not str(args["op_name"]).lower().startswith("memcpy") + and args.get("provider")): + counts[args["provider"]] += 1 + return dict(counts) + except (OSError, ValueError, RuntimeError): + return {} + finally: + if path is not None and not keep_file: + try: + path.unlink(missing_ok=True) + except OSError: + pass + + def estimate_depth( image_path: Union[str, os.PathLike], model_path: Union[str, os.PathLike], + *, session=None, ) -> np.ndarray: """Estimate relative inverse depth for an image using an ONNX model. @@ -79,7 +149,8 @@ def estimate_depth( tensor = (tensor - _MEAN) / _STD tensor = np.ascontiguousarray(tensor, dtype=np.float32) - session = _get_session(model_path) + if session is None: + session = _get_session(model_path) outputs = session.run([_OUTPUT_NAME], {_INPUT_NAME: tensor}) depth = np.asarray(outputs[0], dtype=np.float32) diff --git a/echo_scene/depth_batch.py b/echo_scene/depth_batch.py index 9d959c39..8bb7cf5a 100644 --- a/echo_scene/depth_batch.py +++ b/echo_scene/depth_batch.py @@ -3,24 +3,28 @@ import os import threading from concurrent.futures import ThreadPoolExecutor +from time import perf_counter import numpy as np import onnxruntime as ort from PIL import Image +_MEAN = np.array([0.485, 0.456, 0.406], dtype=np.float32).reshape(3, 1, 1) +_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32).reshape(3, 1, 1) + def _preprocess_image(path): with Image.open(path) as img: img = img.convert("RGB") width, height = img.size resized = img.resize((518, 518), Image.BILINEAR) - pixels = np.asarray(resized, dtype=np.float32) / 255.0 + pixels = np.asarray(resized) - tensor = pixels.transpose(2, 0, 1) - mean = np.array([0.485, 0.456, 0.406], dtype=np.float32).reshape(3, 1, 1) - std = np.array([0.229, 0.224, 0.225], dtype=np.float32).reshape(3, 1, 1) - tensor = (tensor - mean) / std - return np.ascontiguousarray(tensor, dtype=np.float32), width, height + tensor = np.array(pixels.transpose(2, 0, 1), dtype=np.float32, order="C") + tensor /= 255.0 + tensor -= _MEAN + tensor /= _STD + return tensor, width, height def _postprocess_frame(raw_frame, width, height): @@ -45,17 +49,23 @@ def _extract_frame_output(output, index, batch_len): raise ValueError("unexpected batch output shape") -def _infer_single(path, session): +def _infer_prepared(tensor, width, height, session, timings=None): try: - tensor, width, height = _preprocess_image(path) - input_array = np.ascontiguousarray(tensor[np.newaxis, :, :, :], dtype=np.float32) - outputs = session.run(["predicted_depth"], {"pixel_values": input_array}) + outputs = _run_ort(session, tensor[None], timings) frame = _extract_frame_output(outputs[0], 0, 1) return _postprocess_frame(frame, width, height) except Exception: return None +def _infer_single(path, session): + try: + prepared = _preprocess_image(path) + except Exception: + return None + return _infer_prepared(*prepared, session) + + def make_session(model_path, providers=None): if providers is not None: return ort.InferenceSession(str(model_path), providers=providers) @@ -106,7 +116,10 @@ def parallel_workers(model_path, session, n_frames): big = os.path.getsize(model_path) >= _LARGE_MODEL_BYTES except Exception: big = False - return 2 if big else min(4, cores) + # Base's replica and activation workspace make the default two + # one-thread sessions slower on the desktop reconstruction A/B. + # An explicit override can still opt into multiple CPU sessions. + return 0 if big else min(4, cores) def _worker_session(model_path): @@ -153,83 +166,116 @@ def _one(path): yield start + offset, depth +def _pixel_batch_dimension(session): + for model_input in session.get_inputs(): + if model_input.name == "pixel_values" and model_input.shape: + return model_input.shape[0] + return None + + +def _prepare_group(paths): + prepared = [] + for path in paths: + try: + prepared.append(_preprocess_image(path)) + except Exception: + prepared.append(None) + return prepared + + +def _run_ort(session, batch, timings): + started = perf_counter() + try: + return session.run(["predicted_depth"], {"pixel_values": batch}) + finally: + if timings is not None: + timings["ort_run_calls"] = timings.get("ort_run_calls", 0) + 1 + timings["ort_run_seconds"] = timings.get("ort_run_seconds", 0) + perf_counter() - started + + +def _infer_group(prepared, session, dimension, batch_failed, timings): + valid = [(i, item) for i, item in enumerate(prepared) if item is not None] + fixed = isinstance(dimension, int) and dimension > 1 + if valid and (fixed or (not batch_failed and len(valid) == len(prepared) > 1)): + tensors = [item[0] for _, item in valid] + if fixed: + tensors.extend([tensors[-1]] * (dimension - len(tensors))) + try: + batch = np.stack(tensors) + outputs = _run_ort(session, batch, timings) + raw = [_extract_frame_output(outputs[0], i, len(batch)) for i in range(len(valid))] + except Exception: + if fixed: + raise + # Retry with the already prepared inputs, and avoid hitting the + # same batch error (for example GPU memory pressure) next time. + batch_failed = True + else: + results = [None] * len(prepared) + for frame, (index, (_, width, height)) in zip(raw, valid): + try: + results[index] = _postprocess_frame(frame, width, height) + except Exception: + pass + return results, batch_failed + results = [_infer_prepared(*item, session, timings) if item is not None else None + for item in prepared] + return results, batch_failed + + def iter_depth_estimates(image_paths, session, batch_size: int = 4, - chunk_size: int = 16): - """Yield (index, depth_or_None) streaming over bounded chunks. - - estimate_depth_batch holds every result until it returns; at full - frame resolution one depth map is ~8MB, so a job with several - hundred registered frames kept ~6.4GB resident and was OOM-killed - before writing anything (room2 recheck, 2026-11-04). This walks - the list in chunks and yields each frame as its chunk finishes, - so the caller can save it and let it go. chunk_size is rounded - down to a whole number of batches, so the batch groupings -- and - therefore every result -- are exactly the ones a single - estimate_depth_batch call over the whole list would produce. + chunk_size: int = 16, *, prefetch: bool = False, timings=None): + """Stream ordered depth maps with at most one prepared group ahead. + + chunk_size is retained for callers of the previous chunked iterator. + A group now yields as soon as its own depths are ready, with no change + to frame grouping. Prefetch only overlaps CPU preparation with the + caller and ORT; it never runs two inferences or uploads at once. """ paths = list(image_paths) - if batch_size > 1: - chunk_size = max(batch_size, - chunk_size // batch_size * batch_size) - for start in range(0, len(paths), chunk_size): - chunk = paths[start:start + chunk_size] - depths = estimate_depth_batch(chunk, session, - batch_size=batch_size) - for offset, depth in enumerate(depths): - yield start + offset, depth - del depths + if not paths: + return + dimension = _pixel_batch_dimension(session) + size = max(1, batch_size) + if dimension == 1: + size = 1 + elif isinstance(dimension, int) and dimension > 1: + size = dimension + groups = iter(range(0, len(paths), size)) + start = next(groups) + executor = ThreadPoolExecutor(max_workers=1) if prefetch else None + batch_failed = False + pending = None + try: + prep_start = perf_counter() + prepared = _prepare_group(paths[start:start + size]) + if timings is not None: + timings["cpu_input_wait_seconds"] = perf_counter() - prep_start + while True: + next_start = next(groups, None) + if executor is not None and next_start is not None: + pending = executor.submit(_prepare_group, paths[next_start:next_start + size]) + infer_started = perf_counter() + depths, batch_failed = _infer_group(prepared, session, dimension, batch_failed, timings) + if timings is not None: + elapsed = perf_counter() - infer_started + timings["group_inference_and_postprocess_seconds"] = ( + timings.get("group_inference_and_postprocess_seconds", 0.0) + elapsed) + timings["dynamic_batch_fell_back"] = batch_failed + for offset, depth in enumerate(depths): + yield start + offset, depth + if next_start is None: + return + start = next_start + prep_start = perf_counter() + prepared = (pending.result() if pending is not None + else _prepare_group(paths[start:start + size])) + if timings is not None: + timings["cpu_input_wait_seconds"] += perf_counter() - prep_start + finally: + if executor is not None: + executor.shutdown(wait=True, cancel_futures=True) def estimate_depth_batch(image_paths, session, batch_size: int = 4) -> list: - paths = list(image_paths) - if not paths: - return [] - - if batch_size <= 1: - return [_infer_single(path, session) for path in paths] - - results = [None] * len(paths) - - for start in range(0, len(paths), batch_size): - chunk_paths = paths[start:start + batch_size] - chunk_tensors = [] - chunk_meta = [] - chunk_indices = [] - preprocess_ok = True - - for offset, path in enumerate(chunk_paths): - try: - tensor, width, height = _preprocess_image(path) - except Exception: - preprocess_ok = False - break - chunk_tensors.append(tensor) - chunk_meta.append((width, height)) - chunk_indices.append(start + offset) - - if not preprocess_ok: - for offset, path in enumerate(chunk_paths): - results[start + offset] = _infer_single(path, session) - continue - - try: - batch = np.ascontiguousarray(np.stack(chunk_tensors, axis=0), dtype=np.float32) - outputs = session.run(["predicted_depth"], {"pixel_values": batch}) - batch_output = outputs[0] - raw_frames = [ - _extract_frame_output(batch_output, i, len(chunk_tensors)) - for i in range(len(chunk_tensors)) - ] - except Exception: - for offset, path in enumerate(chunk_paths): - results[start + offset] = _infer_single(path, session) - continue - - for i, (width, height) in enumerate(chunk_meta): - idx = chunk_indices[i] - try: - results[idx] = _postprocess_frame(raw_frames[i], width, height) - except Exception: - results[idx] = None - - return results + return [depth for _, depth in iter_depth_estimates(image_paths, session, batch_size)] diff --git a/echo_scene/pipeline.py b/echo_scene/pipeline.py index 643e1311..ae008fa9 100644 --- a/echo_scene/pipeline.py +++ b/echo_scene/pipeline.py @@ -31,10 +31,14 @@ import base64 import json import math +import os +import re import shutil import subprocess import sys +import tempfile from pathlib import Path +from time import perf_counter import numpy as np @@ -509,6 +513,7 @@ def stage_extract(video: Path, job_dir: Path) -> dict: thumbs=thumbs) info = { "stage": "extract", + "device": "CPU (software decode, resize and JPEG)", "video": str(video), "duration_seconds": round(duration, 2), "frame_count": len(frames), @@ -540,6 +545,7 @@ def stage_extract(video: Path, job_dir: Path) -> dict: if not accel: raise globals()["_HWACCEL_CACHE"] = [] + accel = [] _run([_tool_path("ffmpeg"), "-y", "-v", "error", "-i", video, *tail]) frames = sorted(frames_dir.glob("frame_*.jpg")) @@ -549,6 +555,8 @@ def stage_extract(video: Path, job_dir: Path) -> dict: "stage": "extract", "video": str(video), "duration_seconds": round(duration, 2), + "device": (f"FFmpeg {accel[-1]} decode requested; CPU frame output" + if accel else "CPU (software decode and JPEG)"), "frame_interval_seconds": round(interval, 3), "frame_count": len(frames), "frames_dir": str(frames_dir), @@ -1005,6 +1013,8 @@ def stage_poses(job_dir: Path) -> dict: info = { "stage": "poses", "ok": True, + "device": ("COLMAP automatic device selection; check COLMAP SIFT/BA log" + if getattr(pycolmap, "has_cuda", False) else "CPU (COLMAP)"), "registered_images": rec.num_reg_images(), "relocalized_images": len(relocalized), "match_pairs": match_pairs, @@ -1030,7 +1040,10 @@ def stage_select(job_dir: Path, max_frames: int = 120, min_sharpness: float = 20 def stage_depth(job_dir: Path) -> dict: - from .depth import estimate_depth, save_depth_preview + from .depth import (_GPU_PROVIDERS, _get_session, _provider_chain, + end_provider_profile, estimate_depth, save_depth_preview) + from .depth_batch import (_pixel_batch_dimension, iter_depth_estimates, + iter_depth_estimates_parallel, parallel_workers) # The Base model gives visibly smoother, more continuous surfaces # than Small (verified on the room job); fall back to Small when the @@ -1050,56 +1063,90 @@ def stage_depth(job_dir: Path) -> dict: preview_dir = job_dir / "depth_preview" depth_dir.mkdir(exist_ok=True) preview_dir.mkdir(exist_ok=True) - # Batched inference (DS kernel, merge-gated in - # tests/test_depth_batch.py), streamed chunk by chunk so peak - # memory stays bounded no matter how many frames registered: - # the pre-streaming batch call held every depth map in RAM - # (~6.4GB at 773 full-res frames) and was OOM-killed with zero - # output. Falls back to per-frame on any batch-level failure - # (from wherever streaming stopped), per frame on a batch miss. - existing = [n for n in names if (frames_dir / n).exists()] - done = [] + existing = [name for name in names if (frames_dir / name).exists()] + paths = [frames_dir / name for name in existing] + runtime = {"requested_providers": _provider_chain(), "cpu_write_seconds": 0.0} + done = set() + profile_all = os.environ.get("ECHO_DEPTH_PROFILE") == "1" + profile_prefix = None + if paths and (profile_all or _GPU_PROVIDERS.intersection(runtime["requested_providers"])): + profile_prefix = (job_dir / "ort-depth").resolve() + tick = perf_counter() + session = _get_session(model, profile_prefix=profile_prefix) + runtime["session_init_seconds"] = perf_counter() - tick + initial_providers = session.get_providers() + runtime["session_providers_at_start"] = initial_providers + runtime["model_batch_dimension"] = _pixel_batch_dimension(session) + has_gpu = bool(_GPU_PROVIDERS.intersection(initial_providers)) + workers = 0 if profile_all else parallel_workers(model, session, len(paths)) + runtime["cpu_worker_sessions"] = workers + # Keep the existing maximum batch on dynamic models. Smaller cards + # can select one frame, and a rejected dynamic batch downgrades once. + max_batch = os.environ.get("ECHO_DEPTH_BATCH_SIZE", "4") + batch_size = int(max_batch) if max_batch in {"1", "2", "3", "4"} else 4 + runtime["requested_batch_size"] = batch_size + runtime["cpu_prefetch"] = has_gpu and os.environ.get("ECHO_DEPTH_PREFETCH") != "0" + runtime["cpu_fallback_frames"] = 0 + providers_seen = {} + profile_open = profile_prefix is not None + cpu_session = None def _save_depth(name, depth): + tick = perf_counter() stem = Path(name).stem np.save(depth_dir / f"{stem}.npy", depth) save_depth_preview(depth, preview_dir / f"{stem}.png") - done.append(name) - - streamed = True + runtime["cpu_write_seconds"] += perf_counter() - tick + done.add(name) + + def _retry(path): + nonlocal cpu_session + if not has_gpu: + return estimate_depth(path, model, session=session) + if cpu_session is None: + cpu_session = _get_session(model, providers=["CPUExecutionProvider"]) + runtime["cpu_fallback_frames"] += 1 + return estimate_depth(path, model, session=cpu_session) + + estimates = (iter_depth_estimates_parallel(paths, model, workers) if workers + else iter_depth_estimates(paths, session, batch_size=batch_size, + prefetch=runtime["cpu_prefetch"], timings=runtime)) try: - from .depth import _get_session - from .depth_batch import (iter_depth_estimates, - iter_depth_estimates_parallel, - parallel_workers) - - session = _get_session(model) - # Thread-parallel depth when it pays (CPU session, enough - # frames, enough cores): one 1-thread session per worker - # thread, ~1.3x on the real 40-frame check, outputs - # bit-identical to serial single-frame. Any failure here -- - # including one mid-stream -- falls through to the serial - # per-frame loop below, which re-runs only the frames not - # yet saved. - workers = parallel_workers(model, session, len(existing)) - if workers: - estimates = iter_depth_estimates_parallel( - [frames_dir / n for n in existing], model, workers) - else: - estimates = iter_depth_estimates( - [frames_dir / n for n in existing], session, batch_size=4) - for i, depth in estimates: - if depth is None: - depth = estimate_depth(frames_dir / existing[i], model) - _save_depth(existing[i], depth) - except Exception: - streamed = False - if not streamed: - have = set(done) - for name in existing: - if name not in have: - _save_depth(name, estimate_depth(frames_dir / name, model)) - info = {"stage": "depth", "ok": True, "depth_frames": len(done), "model": model.name} + try: + for index, depth in estimates: + if profile_open and not profile_all and depth is not None: + providers_seen = end_provider_profile(session) + profile_open = False + if depth is None: + depth = _retry(paths[index]) + _save_depth(existing[index], depth) + except Exception as exc: + # Retry only unfinished frames. Saving and error reporting + # still fail the job if a frame cannot be produced. + estimates.close() + runtime["stream_error"] = str(exc) + for path in paths: + if path.name not in done: + _save_depth(path.name, _retry(path)) + finally: + estimates.close() + if profile_open: + providers_seen = end_provider_profile(session, keep_file=profile_all) + + runtime["session_providers_at_end"] = session.get_providers() + runtime["profiled_model_node_providers"] = providers_seen + runtime["provider_profile_scope"] = ("full inference session" if profile_all else + "first successful group" if profile_prefix else "off") + if providers_seen.get("CUDAExecutionProvider"): + runtime["cuda_execution"] = "observed" + elif ("CUDAExecutionProvider" not in initial_providers + or (profile_all and providers_seen)): + runtime["cuda_execution"] = "not_used" + else: + runtime["cuda_execution"] = "unverified" + info = {"stage": "depth", "ok": True, "depth_frames": len(done), "model": model.name, + "device": "ONNX inference; CPU preprocessing, resize and file output", + "depth_runtime": runtime} (job_dir / "job.json").write_text(json.dumps(info, ensure_ascii=False, indent=2)) return info @@ -1206,6 +1253,13 @@ def stage_scale(job_dir: Path) -> dict: return info +def _write_embedded_buffer(output, path): + # Keep full 3-byte groups across reads so padding only appears at EOF. + with path.open("rb") as source: + while block := source.read(3 * 256 * 1024): + output.write(base64.b64encode(block).decode("ascii")) + + def stage_present(job_dir: Path) -> dict: """Write the standalone scene page into the job folder. @@ -1244,26 +1298,35 @@ def stage_present(job_dir: Path) -> dict: "reason": "viewer template not found (looked in app/ and install root)"} (job_dir / "job.json").write_text(json.dumps(info, ensure_ascii=False, indent=2)) return info - html = template - points_b64 = base64.b64encode(points_bin.read_bytes()).decode() - html = html.replace( - "\n\n' + script) + cursor = 0 + for match in re.finditer(r"fetch\('(mesh_vertices\.bin|mesh_indices\.bin)'\)", after): + buffer_path = job_dir / match[1] + if buffer_path.is_file(): + output.write(after[cursor:match.start()] + "fetch('data:application/octet-stream;base64,") + _write_embedded_buffer(output, buffer_path) + output.write("')") + cursor = match.end() + output.write(after[cursor:]) + temp_path.replace(scene_path) + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) files = {"scene_html": str(scene_path)} for key, name in (("voxel_glb", "voxel_model.glb"), ("voxel_ply", "voxel_model.ply"), diff --git a/echo_scene/voxel_export.py b/echo_scene/voxel_export.py index 4940b87b..0f7af57f 100644 --- a/echo_scene/voxel_export.py +++ b/echo_scene/voxel_export.py @@ -590,6 +590,57 @@ def _sample_vertex_colors(verts, best_view, best_u, best_v, return colors +_DEDUP_CHUNK = 1_000_000 + + +def _chunked_unique_packed(key): + """(uniq, first, inv) identical to np.unique(key, + return_index=True, return_inverse=True), but no single unique + call ever sees more than _DEDUP_CHUNK keys. + + Measured (v0.6.73, exp/chunkuniq73.py, real 120-frame task, + 6,788,892 corners, 1,313,755 uniques): the single np.unique + over the packed keys is the function's peak -- argsort perm + + sorted copy + cumsum + inv, ~360 MB in a forked child vs + ~270 MB for this form, wall time unchanged (~0.79 s both). + Per chunk: unique (small transient), then ONE unique over the + concatenation of the chunk unique sets (~1.5M keys at 120 + frames, not 6.8M); first is the min over chunks via + np.minimum.at on searchsorted ranks, inv is rank[chunk_inv] + per chunk. The chunk inverses are kept until consumed (54 MB + at 120 frames, the same order as the final inv). + """ + n = len(key) + if n == 0: + empty = np.empty(0, dtype=np.int64) + return empty, empty.copy(), empty.copy() + uniq_parts = [] + first_parts = [] + inv_parts = [] + for start in range(0, n, _DEDUP_CHUNK): + stop = min(start + _DEDUP_CHUNK, n) + uq, first_c, inv_c = np.unique( + key[start:stop], return_index=True, return_inverse=True) + uniq_parts.append(uq) + first_parts.append(first_c.astype(np.int64) + start) + inv_parts.append(inv_c) + if len(uniq_parts) == 1: + uniq = uniq_parts[0] + else: + uniq = np.unique(np.concatenate(uniq_parts)) + first = np.full(len(uniq), np.iinfo(np.int64).max, dtype=np.int64) + inv = np.empty(n, dtype=np.int64) + for uq, first_c, inv_c, start in zip( + uniq_parts, first_parts, inv_parts, + range(0, n, _DEDUP_CHUNK)): + stop = min(start + _DEDUP_CHUNK, n) + rank = np.searchsorted(uniq, uq) + np.minimum.at(first, rank, first_c) + inv[start:stop] = rank[inv_c] + del rank + return uniq, first, inv + + def write_textured_glb(path, verts, normals, tris, uvs, png_bytes, extras=None): """Minimal self-contained textured GLB (glTF 2.0 binary). @@ -635,8 +686,11 @@ def write_textured_glb(path, verts, normals, tris, uvs, png_bytes, (_uq.astype(np.int64) << 20) | _vq.astype(np.int64)) del _uq, _vq - _uniq_key, first, inv = np.unique( - _key, return_index=True, return_inverse=True) + # (v0.6.73) Chunked unique: identical (uniq, first, inv), + # peak ~90 MB lower at 120 frames, wall time unchanged -- + # see _chunked_unique_packed above. The single np.unique + # was the function's peak (v0.6.72 REPORT section 1). + _uniq_key, first, inv = _chunked_unique_packed(_key) _uniq_vi = _uniq_key >> 40 pos = np.ascontiguousarray( np.asarray(verts, dtype=np.float32)[_uniq_vi]) diff --git a/installer/version.inc b/installer/version.inc index 67c7c996..bfcf064f 100644 --- a/installer/version.inc +++ b/installer/version.inc @@ -3,5 +3,5 @@ ; workflow override it from the release tag vX.Y.Z via /DAppVersion). ; EchoScene.iss includes this file only when AppVersion is not already ; defined on the ISCC command line. -#define AppVersion "0.6.71" -#define AppVersionInfo "0.6.71.0" +#define AppVersion "0.6.81" +#define AppVersionInfo "0.6.81.0" diff --git a/tests/test_depth_dispatch.py b/tests/test_depth_dispatch.py new file mode 100644 index 00000000..3d917fe5 --- /dev/null +++ b/tests/test_depth_dispatch.py @@ -0,0 +1,99 @@ +"""Check batch dispatch, without downloading models or requiring a GPU.""" +import tempfile +import unittest +from pathlib import Path +import sys +from types import SimpleNamespace +from unittest.mock import patch + +from PIL import Image +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from echo_scene import depth_batch + + +class Session: + def __init__(self, dimension, reject_batches=False): + self.dimension = dimension + self.reject_batches = reject_batches + self.calls = [] + + def get_inputs(self): + return [SimpleNamespace(name="pixel_values", shape=[self.dimension, 3, 518, 518])] + + def run(self, outputs, inputs): + tensor = inputs["pixel_values"] + self.calls.append(len(tensor)) + if self.reject_batches and len(tensor) > 1: + raise ValueError("batch unsupported") + return [tensor[:, 0]] + + +class DepthDispatch(unittest.TestCase): + def test_fixed_dynamic_and_fallback(self): + with tempfile.TemporaryDirectory() as temp: + paths = [] + for i in range(5): + path = Path(temp) / f"{i}.png" + Image.new("RGB", (12 + i, 20), (40 * i, 0, 0)).save(path) + paths.append(path) + fixed = Session(1) + golden = depth_batch.estimate_depth_batch(paths, fixed) + self.assertEqual(fixed.calls, [1] * 5) + for reject, expected in [(False, [4, 1]), (True, [4, 1, 1, 1, 1, 1])]: + session = Session("N", reject) + with patch.object(depth_batch, "_preprocess_image", + wraps=depth_batch._preprocess_image) as prep: + result = depth_batch.estimate_depth_batch(paths, session) + self.assertEqual(session.calls, expected) + self.assertEqual(prep.call_count, len(paths)) + for wanted, got in zip(golden, result): + np.testing.assert_array_equal(wanted, got) + mixed = depth_batch.estimate_depth_batch([paths[0], Path(temp)/"missing", paths[2]], Session("N")) + self.assertIsNone(mixed[1]) + np.testing.assert_array_equal(golden[2], mixed[2]) + + def test_fixed_groups_prefetch_and_dynamic_downgrade(self): + with tempfile.TemporaryDirectory() as temp: + paths = [Path(temp) / f"{i}.png" for i in range(9)] + for i, path in enumerate(paths): + Image.new("RGB", (13 + i, 20), (13 * i, 0, 0)).save(path) + golden = depth_batch.estimate_depth_batch(paths, Session(1)) + for session, expected in [ + (Session(1), [1] * 9), + (Session(2), [2] * 5), + (Session("N", True), [4] + [1] * 9)]: + timings = {} + result = list(depth_batch.iter_depth_estimates( + paths, session, prefetch=True, timings=timings)) + self.assertEqual(session.calls, expected) + self.assertEqual(timings["ort_run_calls"], len(expected)) + for index, output in result: + np.testing.assert_array_equal(output, golden[index]) + + def test_profile_distinguishes_cuda_transfers_from_model_execution(self): + import json + from echo_scene.depth import end_provider_profile + + def event(op, provider): + return {"cat": "Node", "name": op + "_kernel_time", + "args": {"op_name": op, "provider": provider}} + + with tempfile.TemporaryDirectory() as temp: + trace = Path(temp) / "trace.json" + data = [event("MemcpyFromHost", "CUDAExecutionProvider"), + event("Conv", "CPUExecutionProvider")] + trace.write_text(json.dumps(data), encoding="utf-8") + session = SimpleNamespace(end_profiling=lambda: str(trace)) + self.assertEqual(end_provider_profile(session, keep_file=True), + {"CPUExecutionProvider": 1}) + data.append(event("MatMul", "CUDAExecutionProvider")) + trace.write_text(json.dumps(data), encoding="utf-8") + self.assertEqual(end_provider_profile(session), + {"CPUExecutionProvider": 1, "CUDAExecutionProvider": 1}) + self.assertFalse(trace.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_native_window.py b/tests/test_native_window.py new file mode 100644 index 00000000..45f89edd --- /dev/null +++ b/tests/test_native_window.py @@ -0,0 +1,76 @@ +"""Exercise the desktop caption handlers on a real Windows WebView2 window.""" +import ctypes +from ctypes import wintypes +from pathlib import Path +import sys +import threading +import time +import uuid + +import webview + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from web_app import Bridge, _polish_and_hook + + +def wait_until(predicate, operation): + deadline = time.monotonic() + 10 + while not predicate(): + if time.monotonic() >= deadline: + raise AssertionError(f"timed out during {operation}") + time.sleep(0.05) + + +def main(): + user32 = ctypes.windll.user32 + user32.FindWindowW.argtypes = [wintypes.LPCWSTR, wintypes.LPCWSTR] + user32.FindWindowW.restype = wintypes.HWND + for check in (user32.IsZoomed, user32.IsIconic): + check.argtypes = [wintypes.HWND] + check.restype = wintypes.BOOL + title = "Echo window regression " + uuid.uuid4().hex + url = (Path(__file__).resolve().parents[1] / "web_ui/index.html").as_uri() + window = webview.create_window(title, url, width=800, height=600, frameless=True, easy_drag=False) + ready = threading.Event() + window.events.loaded += ready.set + window.events.shown += lambda: _polish_and_hook(window) + bridge = Bridge() + bridge._window = window + failures = [] + + def exercise(): + try: + assert ready.wait(30), "page did not load" + hwnd = user32.FindWindowW(None, title) + assert hwnd, "native window not found" + bridge.busy, bridge.busy_kind, bridge.frac = True, "scene", 0.5 + bridge._publish_preview_png(b"test") + preview = bridge.get_preview_frame() + assert not bridge.poll(preview["seq"]).get("preview") + assert not bridge.poll(-1, False).get("preview") + assert bridge.poll(-1)["preview"] == preview["data"] + window.evaluate_js("window.testDocument = document; true") + for _ in range(10): + bridge.win_toggle_max() + wait_until(bridge._is_maximized, "maximize") + bridge.win_toggle_max() + wait_until(lambda: not bridge._is_maximized(), "restore size") + bridge.win_minimize() + wait_until(lambda: user32.IsIconic(hwnd), "minimize") + window.restore() + wait_until(lambda: not user32.IsIconic(hwnd), "restore from minimize") + assert window.evaluate_js("window.testDocument === document"), "page reloaded" + assert bridge.busy and bridge.frac == 0.5 + assert bridge.get_preview_frame() == preview + except Exception as exc: + failures.append(exc) + finally: + window.destroy() + + webview.start(exercise, gui="edgechromium") + if failures: + raise failures[0] + + +if __name__ == "__main__": + main() diff --git a/tests/test_scene_export_stream.py b/tests/test_scene_export_stream.py new file mode 100644 index 00000000..204d167b --- /dev/null +++ b/tests/test_scene_export_stream.py @@ -0,0 +1,39 @@ +"""Check the standalone scene payload and atomic replacement on write failure.""" +import base64 +import tempfile +from pathlib import Path +import sys +import unittest +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from echo_scene import pipeline + + +class SceneExport(unittest.TestCase): + def test_buffers_and_failed_reexport(self): + template = (Path(pipeline.__file__).resolve().parents[1] / "viewer.html").read_text(encoding="utf-8") + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + points = bytes(range(253)) * 4001 + (root / "points.bin").write_bytes(points) + expected = template.replace("\n