|
| 1 | +#!/usr/bin/env bash |
| 2 | +# Real-app round trip for `wally openwork` on macOS: launches the installed |
| 3 | +# OpenWork on wally's private profile, asks its engine which models it offers, |
| 4 | +# sends one prompt per provider, quits the app and checks nothing is left |
| 5 | +# running. Opt-in: needs WALLY_E2E_OPENWORK=1, OpenWork.app, and a model |
| 6 | +# (WALLY_E2E_OPENWORK_MODEL, a local id or a hosted one on a signed-in |
| 7 | +# account). WALLY_E2E_OPENWORK_HOSTED names a hosted model to prompt as well. |
| 8 | +# |
| 9 | +# WALLY_E2E_OPENWORK=1 WALLY_E2E_OPENWORK_MODEL=qwen3-4b-instruct-2507 \ |
| 10 | +# bash scripts/test/e2e-openwork.sh ./build/wally |
| 11 | +set -euo pipefail |
| 12 | + |
| 13 | +WALLY="${1:?usage: e2e-openwork.sh <wally binary>}" |
| 14 | +if [[ "${WALLY_E2E_OPENWORK:-0}" != "1" ]]; then |
| 15 | + echo " skip openwork (WALLY_E2E_OPENWORK is not 1)" |
| 16 | + exit 0 |
| 17 | +fi |
| 18 | +if [[ "$(uname -s)" != "Darwin" ]]; then |
| 19 | + echo " skip openwork (macOS only)" |
| 20 | + exit 0 |
| 21 | +fi |
| 22 | +MODEL="${WALLY_E2E_OPENWORK_MODEL:?set WALLY_E2E_OPENWORK_MODEL}" |
| 23 | +HOSTED="${WALLY_E2E_OPENWORK_HOSTED:-}" |
| 24 | +STATE="${XDG_STATE_HOME:-${HOME}/.local/state}/runanywhere/openwork" |
| 25 | +ENGINES="${STATE}/config/engine-instances.json" |
| 26 | +LOG="$(mktemp -t wally-openwork-e2e)" |
| 27 | + |
| 28 | +if pgrep -f "OpenWork.app/Contents/MacOS/OpenWork" >/dev/null; then |
| 29 | + echo " FAIL openwork: quit every OpenWork window first, so the instance this run starts can be told apart" |
| 30 | + exit 1 |
| 31 | +fi |
| 32 | + |
| 33 | +# Only a folder this run creates counts against it. |
| 34 | +HOME_WORKSPACE_EXISTED=0 |
| 35 | +[[ -e "${HOME}/OpenWork Chat" ]] && HOME_WORKSPACE_EXISTED=1 |
| 36 | + |
| 37 | +rm -f "${ENGINES}" |
| 38 | +"${WALLY}" openwork -m "${MODEL}" >"${LOG}" 2>&1 & |
| 39 | +WALLY_PID=$! |
| 40 | +cleanup() { |
| 41 | + osascript -e 'tell application "OpenWork" to quit' >/dev/null 2>&1 || true |
| 42 | + for _ in $(seq 1 30); do |
| 43 | + kill -0 "${WALLY_PID}" 2>/dev/null || return 0 |
| 44 | + sleep 1 |
| 45 | + done |
| 46 | + echo " FAIL openwork: wally was still running 30 s after the app was asked to quit" |
| 47 | + kill "${WALLY_PID}" 2>/dev/null || true |
| 48 | + return 1 |
| 49 | +} |
| 50 | +trap cleanup EXIT |
| 51 | + |
| 52 | +for _ in $(seq 1 120); do |
| 53 | + [[ -s "${ENGINES}" ]] && break |
| 54 | + sleep 1 |
| 55 | +done |
| 56 | +if [[ ! -s "${ENGINES}" ]]; then |
| 57 | + echo " FAIL openwork: the engine never started" |
| 58 | + cat "${LOG}" |
| 59 | + exit 1 |
| 60 | +fi |
| 61 | + |
| 62 | +python3 - "${ENGINES}" "${STATE}/workspace" "${MODEL}" "${HOSTED}" <<'PY' |
| 63 | +import json, sys, time, urllib.parse, urllib.request |
| 64 | +
|
| 65 | +engines, workspace, model, hosted = sys.argv[1:5] |
| 66 | +# The file is written before the engine is registered in it. |
| 67 | +for _ in range(120): |
| 68 | + entries = json.load(open(engines)).get("entries") or [] |
| 69 | + if entries: |
| 70 | + break |
| 71 | + time.sleep(1) |
| 72 | +assert entries, "the engine never registered" |
| 73 | +entry = entries[0] |
| 74 | +base, auth = entry["url"], entry["authProbe"] |
| 75 | +query = "?directory=" + urllib.parse.quote(workspace) |
| 76 | +
|
| 77 | +def call(path, body=None, timeout=30): |
| 78 | + data = None if body is None else json.dumps(body).encode() |
| 79 | + request = urllib.request.Request(base + path + query, data=data, |
| 80 | + headers={"Authorization": auth, "Content-Type": "application/json"}) |
| 81 | + with urllib.request.urlopen(request, timeout=timeout) as response: |
| 82 | + return json.load(response) |
| 83 | +
|
| 84 | +for _ in range(60): |
| 85 | + try: |
| 86 | + providers = call("/config/providers") |
| 87 | + break |
| 88 | + except OSError: |
| 89 | + time.sleep(1) |
| 90 | +by_id = {p["id"]: p for p in providers["providers"]} |
| 91 | +connected = call("/provider")["connected"] |
| 92 | +selected = call("/config").get("model", "") |
| 93 | +ours = [pid for pid in ("runanywhere", "runanywhere-local") if pid in by_id] |
| 94 | +assert ours, f"no RunAnywhere provider in {sorted(by_id)}" |
| 95 | +for pid in ours: |
| 96 | + assert pid in connected, f"{pid} is not connected: {connected}" |
| 97 | + print(f" ok openwork lists {len(by_id[pid]['models'])} models under {pid}") |
| 98 | +assert selected.endswith("/" + model), f"selected {selected!r}, launched {model!r}" |
| 99 | +print(f" ok openwork selects {selected}") |
| 100 | +
|
| 101 | +targets = [(selected.split("/", 1)[0], model)] |
| 102 | +if hosted: |
| 103 | + targets.append(("runanywhere", hosted)) |
| 104 | +for provider, model_id in targets: |
| 105 | + session = call("/session", {})["id"] |
| 106 | + reply = call(f"/session/{session}/message", { |
| 107 | + "model": {"providerID": provider, "modelID": model_id}, |
| 108 | + "parts": [{"type": "text", "text": "Reply with exactly the word pong and nothing else."}], |
| 109 | + }, timeout=300) |
| 110 | + error = reply.get("info", {}).get("error") |
| 111 | + text = " ".join(p.get("text", "") for p in reply.get("parts", []) if p.get("type") == "text") |
| 112 | + assert not error and text.strip().strip(".").lower() == "pong", f"{provider}/{model_id}: error={error} text={text!r}" |
| 113 | + print(f" ok openwork answered on {provider}/{model_id}: {text.strip()[:40]!r}") |
| 114 | +PY |
| 115 | + |
| 116 | +trap - EXIT |
| 117 | +cleanup || exit 1 |
| 118 | +# The engine is stopped by the app on its way out, which takes a few seconds. |
| 119 | +for _ in $(seq 1 20); do |
| 120 | + pgrep -f "OpenWork.app/Contents/Resources/sidecars/opencode" >/dev/null || break |
| 121 | + sleep 1 |
| 122 | +done |
| 123 | +if pgrep -f "OpenWork.app/Contents/Resources/sidecars/opencode" >/dev/null; then |
| 124 | + echo " FAIL openwork: its engine is still running after the app quit" |
| 125 | + exit 1 |
| 126 | +fi |
| 127 | +if [[ "${HOME_WORKSPACE_EXISTED}" == 0 && -e "${HOME}/OpenWork Chat" ]]; then |
| 128 | + echo " FAIL openwork: the default workspace landed in the real home" |
| 129 | + exit 1 |
| 130 | +fi |
| 131 | +echo " ok openwork quit cleanly" |
0 commit comments