Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 34 additions & 0 deletions .github/workflows/desktop-runtime.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: Desktop runtime regression
on:
pull_request:
branches: [product/echo-scene]
paths:
- 'web_ui/**'
- 'web_app.py'
- 'echo_scene/depth_batch.py'
- 'echo_scene/pipeline.py'
- 'tests/test_scene_export_stream.py'
- 'tests/test_web_bridge.py'
- 'tests/test_window_resize.py'
- 'tests/test_native_window.py'
- 'tests/test_depth_dispatch.py'
- '.github/workflows/desktop-runtime.yml'
permissions:
contents: read
jobs:
windows:
runs-on: windows-latest
timeout-minutes: 12
env:
PYTHONUTF8: "1"
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- run: python -m pip install numpy pillow onnxruntime-directml playwright pywebview
- run: python tests/test_web_bridge.py
- run: python tests/test_depth_dispatch.py
- run: python tests/test_scene_export_stream.py
- run: python tests/test_window_resize.py
- run: python tests/test_native_window.py
43 changes: 31 additions & 12 deletions echo_scene/depth_batch.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
"""Batch depth estimation utilities for a single-frame ONNX depth model."""

import numpy as np
import onnxruntime as ort
from PIL import Image


Expand Down Expand Up @@ -41,9 +40,8 @@ def _extract_frame_output(output, index, batch_len):
raise ValueError("unexpected batch output shape")


def _infer_single(path, session):
def _infer_prepared(tensor, width, height, session):
try:
tensor, width, height = _preprocess_image(path)
input_array = np.ascontiguousarray(tensor[np.newaxis, :, :, :], dtype=np.float32)
outputs = session.run(["predicted_depth"], {"pixel_values": input_array})
frame = _extract_frame_output(outputs[0], 0, 1)
Expand All @@ -52,12 +50,29 @@ def _infer_single(path, session):
return None


def _infer_single(path, session):
try:
tensor, width, height = _preprocess_image(path)
except Exception:
return None
return _infer_prepared(tensor, width, height, session)


def make_session(model_path, providers=None):
import onnxruntime as ort

if providers is not None:
return ort.InferenceSession(str(model_path), providers=providers)
return ort.InferenceSession(str(model_path))


def _pixel_batch_dimension(session):
for model_input in session.get_inputs():
if model_input.name == "pixel_values" and model_input.shape:
return model_input.shape[0]
return None


def iter_depth_estimates(image_paths, session, batch_size: int = 4,
chunk_size: int = 16):
"""Yield (index, depth_or_None) streaming over bounded chunks.
Expand All @@ -73,6 +88,10 @@ def iter_depth_estimates(image_paths, session, batch_size: int = 4,
estimate_depth_batch call over the whole list would produce.
"""
paths = list(image_paths)
if paths:
print(f"Depth runtime: providers={session.get_providers()}, "
f"model batch dimension={_pixel_batch_dimension(session)}, "
f"requested batch={batch_size}", flush=True)
if batch_size > 1:
chunk_size = max(batch_size,
chunk_size // batch_size * batch_size)
Expand All @@ -90,7 +109,9 @@ def estimate_depth_batch(image_paths, session, batch_size: int = 4) -> list:
if not paths:
return []

if batch_size <= 1:
# ONNX models exported with batch dimension 1 cannot accept a stack.
# A symbolic/unknown dimension keeps the existing batched path.
if batch_size <= 1 or _pixel_batch_dimension(session) == 1:
return [_infer_single(path, session) for path in paths]

results = [None] * len(paths)
Expand All @@ -100,21 +121,19 @@ def estimate_depth_batch(image_paths, session, batch_size: int = 4) -> list:
chunk_tensors = []
chunk_meta = []
chunk_indices = []
preprocess_ok = True

for offset, path in enumerate(chunk_paths):
try:
tensor, width, height = _preprocess_image(path)
except Exception:
preprocess_ok = False
break
continue
chunk_tensors.append(tensor)
chunk_meta.append((width, height))
chunk_indices.append(start + offset)

if not preprocess_ok:
for offset, path in enumerate(chunk_paths):
results[start + offset] = _infer_single(path, session)
if len(chunk_tensors) != len(chunk_paths):
for idx, tensor, (width, height) in zip(chunk_indices, chunk_tensors, chunk_meta):
results[idx] = _infer_prepared(tensor, width, height, session)
continue

try:
Expand All @@ -126,8 +145,8 @@ def estimate_depth_batch(image_paths, session, batch_size: int = 4) -> list:
for i in range(len(chunk_tensors))
]
except Exception:
for offset, path in enumerate(chunk_paths):
results[start + offset] = _infer_single(path, session)
for idx, tensor, (width, height) in zip(chunk_indices, chunk_tensors, chunk_meta):
results[idx] = _infer_prepared(tensor, width, height, session)
continue

for i, (width, height) in enumerate(chunk_meta):
Expand Down
48 changes: 34 additions & 14 deletions echo_scene/pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,11 @@
import base64
import json
import math
import re
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path

import numpy as np
Expand Down Expand Up @@ -1150,6 +1152,13 @@ def stage_scale(job_dir: Path) -> dict:
return info


def _write_embedded_buffer(output, path):
# Every non-final block must encode without Base64 padding.
with path.open("rb") as source:
while block := source.read(3 * 256 * 1024):
output.write(base64.b64encode(block).decode("ascii"))


def stage_present(job_dir: Path) -> dict:
"""Write the standalone scene page into the job folder.

Expand All @@ -1168,25 +1177,36 @@ def stage_present(job_dir: Path) -> dict:
return info
template = Path(__file__).resolve().parent.parent / "viewer.html"
html = template.read_text(encoding="utf-8")
points_b64 = base64.b64encode(points_bin.read_bytes()).decode()
html = html.replace(
"<script>\n",
f'<script>window.POINTS_B64 = "{points_b64}";</script>\n<script>\n',
1,
)
if "window.POINTS_B64" not in html:
before, script, after = html.partition("<script>\n")
if not script:
info = {"stage": "present", "ok": False,
"reason": "viewer template seam not found"}
(job_dir / "job.json").write_text(json.dumps(info, ensure_ascii=False, indent=2))
return info
for name in ("mesh_vertices.bin", "mesh_indices.bin"):
buf = job_dir / name
if buf.exists():
uri = ("data:application/octet-stream;base64,"
+ base64.b64encode(buf.read_bytes()).decode())
html = html.replace(f"fetch('{name}')", f"fetch('{uri}')")

scene_path = job_dir / "scene.html"
scene_path.write_text(html, encoding="utf-8")
# Publish only a complete HTML file, including when re-exporting a job.
temp_path = None
try:
with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8",
dir=job_dir, delete=False) as output:
temp_path = Path(output.name)
output.write(before + '<script>window.POINTS_B64 = "')
_write_embedded_buffer(output, points_bin)
output.write('";</script>\n' + script)
cursor = 0
for match in re.finditer(r"fetch\('(mesh_vertices\.bin|mesh_indices\.bin)'\)", after):
buf = job_dir / match[1]
if buf.exists():
output.write(after[cursor:match.start()] + "fetch('data:application/octet-stream;base64,")
_write_embedded_buffer(output, buf)
output.write("')")
cursor = match.end()
output.write(after[cursor:])
temp_path.replace(scene_path)
finally:
if temp_path is not None:
temp_path.unlink(missing_ok=True)
files = {"scene_html": str(scene_path)}
for key, name in (("voxel_glb", "voxel_model.glb"),
("voxel_ply", "voxel_model.ply"),
Expand Down
59 changes: 59 additions & 0 deletions tests/test_depth_dispatch.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
"""Check batch dispatch, without downloading models or requiring a GPU."""
import tempfile
import unittest
from pathlib import Path
import sys
from types import SimpleNamespace
from unittest.mock import patch

from PIL import Image
import numpy as np

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from echo_scene import depth_batch


class Session:
def __init__(self, dimension, reject_batches=False):
self.dimension = dimension
self.reject_batches = reject_batches
self.calls = []

def get_inputs(self):
return [SimpleNamespace(name="pixel_values", shape=[self.dimension, 3, 518, 518])]

def run(self, outputs, inputs):
tensor = inputs["pixel_values"]
self.calls.append(len(tensor))
if self.reject_batches and len(tensor) > 1:
raise ValueError("batch unsupported")
return [tensor[:, 0]]


class DepthDispatch(unittest.TestCase):
def test_fixed_dynamic_and_fallback(self):
with tempfile.TemporaryDirectory() as temp:
paths = []
for i in range(5):
path = Path(temp) / f"{i}.png"
Image.new("RGB", (12 + i, 20), (40 * i, 0, 0)).save(path)
paths.append(path)
fixed = Session(1)
golden = depth_batch.estimate_depth_batch(paths, fixed)
self.assertEqual(fixed.calls, [1] * 5)
for reject, expected in [(False, [4, 1]), (True, [4, 1, 1, 1, 1, 1])]:
session = Session("N", reject)
with patch.object(depth_batch, "_preprocess_image",
wraps=depth_batch._preprocess_image) as prep:
result = depth_batch.estimate_depth_batch(paths, session)
self.assertEqual(session.calls, expected)
self.assertEqual(prep.call_count, len(paths))
for wanted, got in zip(golden, result):
np.testing.assert_array_equal(wanted, got)
mixed = depth_batch.estimate_depth_batch([paths[0], Path(temp)/"missing", paths[2]], Session("N"))
self.assertIsNone(mixed[1])
np.testing.assert_array_equal(golden[2], mixed[2])


if __name__ == "__main__":
unittest.main()
67 changes: 67 additions & 0 deletions tests/test_native_window.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
"""Exercise the desktop caption handlers on a real Windows WebView2 window."""
import ctypes
from ctypes import wintypes
from pathlib import Path
import sys
import threading
import time
import uuid

import webview

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from web_app import Bridge


def wait_until(predicate, operation):
deadline = time.monotonic() + 10
while not predicate():
if time.monotonic() >= deadline:
raise AssertionError(f"timed out during {operation}")
time.sleep(0.05)


def main():
user32 = ctypes.windll.user32
user32.FindWindowW.argtypes = [wintypes.LPCWSTR, wintypes.LPCWSTR]
user32.FindWindowW.restype = wintypes.HWND
for check in (user32.IsZoomed, user32.IsIconic):
check.argtypes = [wintypes.HWND]
check.restype = wintypes.BOOL
title = "Echo window regression " + uuid.uuid4().hex
url = (Path(__file__).resolve().parents[1] / "web_ui/index.html").as_uri()
window = webview.create_window(title, url, width=800, height=600, frameless=True)
ready = threading.Event()
window.events.loaded += ready.set
bridge = Bridge()
bridge._window = window
failures = []

def exercise():
try:
assert ready.wait(30), "page did not load"
hwnd = user32.FindWindowW(None, title)
assert hwnd, "native window not found"
window.evaluate_js("window.testDocument = document; true")
for _ in range(10):
bridge.win_toggle_max()
wait_until(lambda: user32.IsZoomed(hwnd), "maximize")
bridge.win_toggle_max()
wait_until(lambda: not user32.IsZoomed(hwnd), "restore size")
bridge.win_minimize()
wait_until(lambda: user32.IsIconic(hwnd), "minimize")
window.restore()
wait_until(lambda: not user32.IsIconic(hwnd), "restore from minimize")
assert window.evaluate_js("window.testDocument === document"), "page reloaded"
except Exception as exc:
failures.append(exc)
finally:
window.destroy()

webview.start(exercise, gui="edgechromium")
if failures:
raise failures[0]


if __name__ == "__main__":
main()
39 changes: 39 additions & 0 deletions tests/test_scene_export_stream.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
"""Check the standalone scene payload and atomic replacement on write failure."""
import base64
import tempfile
from pathlib import Path
import sys
import unittest
from unittest.mock import patch

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from echo_scene import pipeline


class SceneExport(unittest.TestCase):
def test_buffers_and_failed_reexport(self):
template = (Path(pipeline.__file__).resolve().parents[1] / "viewer.html").read_text(encoding="utf-8")
with tempfile.TemporaryDirectory() as folder:
root = Path(folder)
points = bytes(range(253)) * 4001
(root / "points.bin").write_bytes(points)
expected = template.replace("<script>\n", '<script>window.POINTS_B64 = "'
+ base64.b64encode(points).decode()
+ '";</script>\n<script>\n', 1)
for name in [None, "mesh_vertices.bin", "mesh_indices.bin"]:
if name:
(root / name).write_bytes(b"xyz1")
expected = expected.replace(f"fetch('{name}')",
"fetch('data:application/octet-stream;base64,eHl6MQ==')")
self.assertTrue(pipeline.stage_present(root)["ok"])
self.assertEqual((root / "scene.html").read_text(encoding="utf-8"), expected)
names = set(root.iterdir())
with patch.object(pipeline, "_write_embedded_buffer", side_effect=OSError("disk full")):
with self.assertRaises(OSError):
pipeline.stage_present(root)
self.assertEqual(set(root.iterdir()), names)
self.assertEqual((root / "scene.html").read_text(encoding="utf-8"), expected)


if __name__ == "__main__":
unittest.main()
Loading
Loading