From ef31cac802393b39107eaa3bda35f1252d107f6b Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:09:26 -0700 Subject: [PATCH 01/24] Replace the always-on runtime-backup service with the opt-in github-sync skill GitHub sync is now opt-in: the runtime-backup supervisord service, bootstrap's runtime/ worktree init (mindsbackup/$MNGR_AGENT_ID), the always-installed core.hooksPath post-commit auto-push, and all GH_TOKEN usage are removed from the default template. The new github-sync skill enables sync on request: it creates a dedicated PRIVATE GitHub repo through latchkey (no token in the container), points origin at it, wires plain git push through the latchkey gateway for every checkout (per-VPS secondary gateway as offline fallback), installs the post-commit hook so agent and worker commits auto-push, and adds a [program:github-sync] service that syncs runtime/ to the stable runtime-sync orphan branch. libs/runtime_backup is renamed and extended into libs/github_sync. The service re-verifies repo visibility and halts all pushes when the repo is public or unverifiable. Workspaces recreated from a synced repo self-heal once permissions are re-granted. Co-Authored-By: Claude Fable 5 --- .../scripts/scaffold_flask_lib.py | 2 +- .agents/skills/edit-services/SKILL.md | 2 +- .agents/skills/github-sync/SKILL.md | 184 +++++++++ .agents/skills/publish-inspiration/SKILL.md | 2 +- .gitignore | 4 +- .mngr/settings.toml | 37 +- CLAUDE.md | 12 +- Dockerfile | 2 +- README.md | 2 +- .../imbue/system_interface/server.py | 4 +- changelog/mngr-github-backup-skill.md | 11 + libs/bootstrap/README.md | 19 +- libs/bootstrap/src/bootstrap/manager.py | 268 +------------ libs/bootstrap/src/bootstrap/manager_test.py | 39 +- libs/browser/README.md | 2 +- libs/browser/src/browser/manifest.py | 2 +- libs/browser/src/browser/session.py | 4 +- libs/github_sync/README.md | 73 ++++ libs/github_sync/pyproject.toml | 20 + .../src/github_sync}/__init__.py | 0 libs/github_sync/src/github_sync/cli.py | 95 +++++ libs/github_sync/src/github_sync/config.py | 105 ++++++ .../src/github_sync/config_test.py | 103 +++++ libs/github_sync/src/github_sync/conftest.py | 44 +++ libs/github_sync/src/github_sync/runner.py | 353 ++++++++++++++++++ .../src/github_sync/runner_test.py | 233 ++++++++++++ libs/github_sync/src/github_sync/testing.py | 52 +++ .../github_sync/src/github_sync/visibility.py | 105 ++++++ .../src/github_sync/visibility_test.py | 74 ++++ libs/github_sync/src/github_sync/wiring.py | 134 +++++++ .../src/github_sync/wiring_test.py | 98 +++++ libs/github_sync/src/github_sync/worktree.py | 258 +++++++++++++ .../src/github_sync/worktree_test.py | 127 +++++++ .../test_github_sync_ratchets.py} | 0 libs/host_backup/README.md | 11 +- libs/host_backup/src/host_backup/config.py | 2 +- libs/runtime_backup/README.md | 51 --- libs/runtime_backup/pyproject.toml | 19 - .../src/runtime_backup/runner.py | 197 ---------- .../src/runtime_backup/runner_test.py | 126 ------- pyproject.toml | 6 +- scripts/claude_open_tickets_reminder.sh | 2 +- scripts/claude_open_tickets_stop_nudge.sh | 2 +- scripts/git_hooks/post-commit | 59 ++- supervisord.conf | 26 +- uv.lock | 32 +- 46 files changed, 2216 insertions(+), 787 deletions(-) create mode 100644 .agents/skills/github-sync/SKILL.md create mode 100644 changelog/mngr-github-backup-skill.md create mode 100644 libs/github_sync/README.md create mode 100644 libs/github_sync/pyproject.toml rename libs/{runtime_backup/src/runtime_backup => github_sync/src/github_sync}/__init__.py (100%) create mode 100644 libs/github_sync/src/github_sync/cli.py create mode 100644 libs/github_sync/src/github_sync/config.py create mode 100644 libs/github_sync/src/github_sync/config_test.py create mode 100644 libs/github_sync/src/github_sync/conftest.py create mode 100644 libs/github_sync/src/github_sync/runner.py create mode 100644 libs/github_sync/src/github_sync/runner_test.py create mode 100644 libs/github_sync/src/github_sync/testing.py create mode 100644 libs/github_sync/src/github_sync/visibility.py create mode 100644 libs/github_sync/src/github_sync/visibility_test.py create mode 100644 libs/github_sync/src/github_sync/wiring.py create mode 100644 libs/github_sync/src/github_sync/wiring_test.py create mode 100644 libs/github_sync/src/github_sync/worktree.py create mode 100644 libs/github_sync/src/github_sync/worktree_test.py rename libs/{runtime_backup/test_runtime_backup_ratchets.py => github_sync/test_github_sync_ratchets.py} (100%) delete mode 100644 libs/runtime_backup/README.md delete mode 100644 libs/runtime_backup/pyproject.toml delete mode 100644 libs/runtime_backup/src/runtime_backup/runner.py delete mode 100644 libs/runtime_backup/src/runtime_backup/runner_test.py diff --git a/.agents/skills/build-web-service/scripts/scaffold_flask_lib.py b/.agents/skills/build-web-service/scripts/scaffold_flask_lib.py index ed9fb078c..b6990998a 100644 --- a/.agents/skills/build-web-service/scripts/scaffold_flask_lib.py +++ b/.agents/skills/build-web-service/scripts/scaffold_flask_lib.py @@ -43,7 +43,7 @@ "cloudflare-tunnel", "app-watcher", "bootstrap", - "runtime-backup", + "github-sync", "host-backup", "terminal", "deferred-install", diff --git a/.agents/skills/edit-services/SKILL.md b/.agents/skills/edit-services/SKILL.md index b77710f84..2d9df2aad 100644 --- a/.agents/skills/edit-services/SKILL.md +++ b/.agents/skills/edit-services/SKILL.md @@ -57,7 +57,7 @@ Key fields: instead. Services inherit the agent environment (`MNGR_AGENT_STATE_DIR`, -`CLAUDE_CONFIG_DIR`, `MNGR_HOST_DIR`, `GH_TOKEN`, ...) from the bootstrap shell +`CLAUDE_CONFIG_DIR`, `MNGR_HOST_DIR`, `LATCHKEY_*`, ...) from the bootstrap shell that launched supervisord -- you do not need a per-program `environment=`. ## Adding a service diff --git a/.agents/skills/github-sync/SKILL.md b/.agents/skills/github-sync/SKILL.md new file mode 100644 index 000000000..e4b39eefa --- /dev/null +++ b/.agents/skills/github-sync/SKILL.md @@ -0,0 +1,184 @@ +--- +name: github-sync +description: Enable, check, or disable GitHub sync for this workspace. Enabling creates a dedicated PRIVATE GitHub repo via latchkey, points origin at it, auto-pushes every commit from every checkout, and continuously syncs runtime/ state (memory, tickets, transcripts). Use when the user asks to back up / sync the workspace to GitHub, enable auto-push, restore a previous workspace's state, or asks about GitHub sync status. +compatibility: Requires latchkey (see the latchkey skill) and the user approving GitHub permissions in the Minds app. +--- + +# GitHub sync + +GitHub sync is opt-in. Nothing syncs until this skill enables it. Once +enabled, three pieces work together (see `libs/github_sync/README.md`): + +1. `origin` points at a dedicated **private** GitHub repo for this workspace. +2. Global git wiring routes all `https://github.com/...` traffic through the + latchkey gateway (credential injected server-side; no token in the + container) and activates the `post-commit` hook, so every commit on any + checkout -- main repo and worker worktrees -- auto-pushes its branch. +3. The `[program:github-sync]` service commits + pushes `runtime/` to the + `runtime-sync` orphan branch every 60s and re-verifies the repo stays + private, halting pushes if it ever isn't. + +## Hard rules + +- **Private repos only.** Never create a public repo, never point sync at a + public repo, and never work around a visibility halt. If the user asks for + a public sync repo, decline and explain: agents can push secrets or other + sensitive data without realizing it. +- **Everything flows through latchkey.** Never ask the user for a GitHub + token and never embed credentials in URLs or git config. +- `origin` is reserved for the sync repo. Upstream-template operations keep + using `parent.toml` (see the update-self skill) and are unaffected. + +## Enable + +1. **Check current state**: `uv run github-sync status`. If `is_configured` + is already true, jump to "Status" (or "Repair" if the service is + unhealthy). Also run `supervisorctl status github-sync` (it errors when no + such program exists -- expected before enable). + +2. **Request GitHub permissions** through latchkey (see the latchkey skill + for the permission-request mechanics). Both scopes are needed, in one + combined ask to the user: + + ```bash + latchkey curl -XPOST http://latchkey-self.invalid/permission-requests \ + -H 'Content-Type: application/json' \ + -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-git", "permissions": ["github-git-read", "github-git-write"]}, "rationale": "GitHub sync: push this workspace'"'"'s branches and runtime state to your private sync repo."}' + latchkey curl -XPOST http://latchkey-self.invalid/permission-requests \ + -H 'Content-Type: application/json' \ + -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-rest-api", "permissions": ["github-read-repos", "github-write-repos"]}, "rationale": "GitHub sync: create the private sync repo and verify it stays private."}' + ``` + + Wait for the approval system message before continuing. + +3. **Pick the repo.** Default: create a brand-new private repo named after + the workspace (`$MINDS_WORKSPACE_NAME`). Confirm the name and owner with + the user first; owner defaults to their GitHub account, but they can name + an org. One exception: if `git remote get-url origin` already points at a + user-owned repo (not `imbue-ai/default-workspace-template` or another + shared template), ask whether to reuse it or create a fresh one -- + recommend a fresh dedicated repo unless they have a specific reason. + Reused repos must be verified private and writable like new ones. + +4. **Create the repo** (skip if reusing): + + ```bash + latchkey curl -s -X POST https://api.github.com/user/repos \ + -H 'Content-Type: application/json' \ + -d '{"name": "", "private": true, "description": "Private sync repo for the minds workspace"}' + ``` + + (For an org: `POST https://api.github.com/orgs//repos`.) On a 422 + name-taken error, append `-2`, `-3`, ... and retry. The response JSON must + contain `"private": true` -- if it does not, delete/abandon the repo and + stop; do not proceed with a public repo. + +5. **Point origin at it and record the config**: + + ```bash + git remote set-url origin https://github.com//.git \ + || git remote add origin https://github.com//.git + ``` + + Write `github_sync.toml` at the repo root (this file is the "sync is + enabled" marker for the service and the post-commit hook): + + ```toml + # Written by the github-sync skill. Presence of this file enables GitHub + # sync; `uv run github-sync status` reports on it. + repo_url = "https://github.com//" + ``` + +6. **Wire git through the gateway**: `uv run github-sync wire-git`. From now + on plain `git push`/`git fetch` against github.com works in every + checkout, and the post-commit auto-push hook is active. + +7. **Create (or restore) the runtime worktree**: + `uv run github-sync setup-worktree`. If origin already has a + `runtime-sync` branch (re-enabling for a workspace recreated from a + previously-synced repo), this restores the prior runtime/ state instead of + starting fresh -- tell the user their memory/tickets/transcripts are back. + +8. **Verify private before any push**: `uv run github-sync check-visibility` + must print `private` (exit 0). If not, stop and surface the problem. + +9. **Initial sync**: push the current branch, the runtime-sync branch, and + any existing worker branches: + + ```bash + git push --set-upstream origin "$(git branch --show-current)" + git -C runtime push --set-upstream origin runtime-sync + for b in $(git for-each-ref --format='%(refname:short)' refs/heads/ | grep -v -x -e "$(git branch --show-current)" -e runtime-sync); do git push origin "$b"; done + ``` + +10. **Add the service** by appending this block to `supervisord.conf`, then + `supervisorctl reread && supervisorctl update` (see the edit-services + skill): + + ```ini + # Opt-in GitHub sync (added by the github-sync skill): commits + pushes + # runtime/ to the runtime-sync branch of the private sync repo and + # re-verifies the repo stays private. See libs/github_sync/README.md. + [program:github-sync] + command=uv run github-sync run + directory=/mngr/code + autostart=true + autorestart=true + startretries=1000000 + stopasgroup=true + killasgroup=true + stdout_logfile=/var/log/supervisor/github-sync-stdout.log + stderr_logfile=/var/log/supervisor/github-sync-stderr.log + stdout_logfile_maxbytes=10MB + stderr_logfile_maxbytes=10MB + stdout_logfile_backups=3 + stderr_logfile_backups=3 + ``` + +11. **Commit the enablement** (`github_sync.toml` + `supervisord.conf`). The + now-active hook pushes the commit; this also makes sync sticky if the + repo is later used to recreate the workspace. + +12. **Report**: the repo URL, that every commit now auto-pushes, that + runtime/ syncs every minute, and that pushes queue while their machine + (the latchkey gateway) is offline (on remote hosts the per-VPS secondary + gateway usually covers that). + +## Status + +`uv run github-sync status` prints config + the service's latest status +(visibility, last push, errors); `supervisorctl status github-sync` shows the +process; logs are at `/var/log/supervisor/github-sync-*.log` and +`/tmp/github-sync.log`, hook output at `/tmp/post-commit-push.log`. Explain +findings in plain language. If `is_push_allowed` is false, the repo is public +or unverifiable -- tell the user to make it private again; sync resumes +automatically. + +## Repair (workspace recreated from a synced repo) + +A workspace created from a previously-synced private repo inherits +`github_sync.toml` and the service block, but not the latchkey permissions or +the container-local wiring/worktree. The service idles until it can reach +origin. To repair: run step 2 (permission requests); the service self-heals +within a tick (re-wires git, restores runtime/ from origin). Verify with +"Status", or accelerate with `uv run github-sync wire-git` + +`uv run github-sync setup-worktree`. + +## Disable + +Confirm with the user first, and ask separately whether to keep the remote +repo (recommend keeping it -- it costs nothing and preserves history). + +1. `supervisorctl stop github-sync`, remove the `[program:github-sync]` block + from `supervisord.conf`, then `supervisorctl reread && supervisorctl update`. +2. `uv run github-sync unwire-git` (removes the gateway git config and the + hooks path -- auto-push stops). +3. Delete `github_sync.toml`. +4. Leave the local `runtime/` worktree and its history intact (harmless, and + re-enabling picks it right back up). +5. If the user chose to delete the remote repo, try + `latchkey curl -s -X DELETE https://api.github.com/repos//`; + deleting repos needs an extra GitHub permission, so if it fails, point + them at the repo's GitHub settings page to delete it themselves. +6. Commit the removal. Note that this commit is NOT auto-pushed (the hook is + inert again). diff --git a/.agents/skills/publish-inspiration/SKILL.md b/.agents/skills/publish-inspiration/SKILL.md index 26e62f234..a849a1b39 100644 --- a/.agents/skills/publish-inspiration/SKILL.md +++ b/.agents/skills/publish-inspiration/SKILL.md @@ -591,7 +591,7 @@ exit 1 If the user never approves, surface a clear message and stop, leaving the assembled commit intact. Do NOT fall back to any other credential or -mechanism (no `GH_TOKEN`-in-URL pushes, no partial-tree API uploads -- see +mechanism (no token-in-URL pushes, no partial-tree API uploads -- see the "MUST BE BOOTABLE" callout). ## 8. Create the repo and push diff --git a/.gitignore b/.gitignore index a5b5ff95d..fc1858bf0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,8 @@ # Default Workspace Template runtime state. # memory/ now lives at runtime/memory/ (configured via .claude/settings.json's -# autoMemoryDirectory) so the runtime-backup service backs it up alongside -# the rest of runtime/. +# autoMemoryDirectory) so the opt-in github-sync service (see the github-sync +# skill) covers it alongside the rest of runtime/. **/runtime/ # chat attachment uploads (arbitrary size/format; kept out of version control) diff --git a/.mngr/settings.toml b/.mngr/settings.toml index 3c4efa66c..1d5109599 100644 --- a/.mngr/settings.toml +++ b/.mngr/settings.toml @@ -5,7 +5,7 @@ type = "claude" connect = false ensure_clean = false -pass_env__extend = ["REMOTE_SERVICE_CONNECTOR_URL", "LATCHKEY_GATEWAY", "GH_TOKEN"] +pass_env__extend = ["REMOTE_SERVICE_CONNECTOR_URL", "LATCHKEY_GATEWAY"] host_env__extend = [ "IS_SANDBOX=1", "IS_AUTONOMOUS=1", @@ -28,7 +28,8 @@ host_env__extend = [ # Disable claude.ai account MCP connector sync -- it's an OAuth side # channel to third-party services that bypasses the latchkey gateway. "ENABLE_CLAUDEAI_MCP_SERVERS=false", - # Tickets live inside runtime/ so they ride the runtime-backup branch. + # Tickets live inside runtime/ so they are covered by the opt-in GitHub + # sync (github-sync skill) when the user enables it. # tk reads TICKETS_DIR if set; absolute path so it works from worktree # subdirs too. /mngr/code/ is the container WORKDIR (set by the Dockerfile); # a /code -> /mngr/code symlink is kept in the image as a safety net. @@ -193,13 +194,13 @@ type = "main" # to it instead. # # bootstrap is the only extra_window now. It runs first-boot setup (git config, -# runtime worktree, CLAUDE_CONFIG_DIR host-env write, backup config, initial -# chat agent) and then launches supervisord, which runs every background -# service (system_interface, web, cloudflared, app-watcher, runtime-backup, -# host-backup, terminal, and the one-shot deferred-install -- see -# supervisord.conf). The old git_auth_setup window is gone (the git config it -# performed now runs inside bootstrap), and terminal is a supervisord service -# rather than its own window. +# CLAUDE_CONFIG_DIR host-env write, initial chat agent) and then launches +# supervisord, which runs every background service (system_interface, web, +# cloudflared, app-watcher, host-backup, terminal, and the one-shot +# deferred-install -- see supervisord.conf; the opt-in github-sync service is +# added by its skill). The old git_auth_setup window is gone (the git config +# it performed now runs inside bootstrap), and terminal is a supervisord +# service rather than its own window. extra_window = ["bootstrap='uv run bootstrap'"] # Write a sane default tmux config on every minds host. tuple-typed # `extra_provision_command` stacks across templates (see @@ -254,11 +255,11 @@ start_arg__extend = ["--security-opt=no-new-privileges", "--workdir=/", "--resta # Disable idle detection entirely -- the host should never auto-shutdown. # This also prevents shutdown when all agent tmux sessions exit. idle_mode = "disabled" -# Forward Anthropic creds + GH_TOKEN from minds' subprocess env onto the +# Forward Anthropic creds from minds' subprocess env onto the # new container's env. The agent process inherits the container env, so # this is what gives the agent its ANTHROPIC_API_KEY (etc.) in # non-IMBUE_CLOUD compute modes. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Seed /mngr/code from /docker_build_code (baked into the image) onto the # per-host volume. Runs synchronously after the host is online but # before any agent work_dir setup, so the workspace is in place by the @@ -299,7 +300,7 @@ start_arg__extend = ["--cpus=2", "--memory=4", "--disk=20"] # version-check disable is needed. setting__extend = ["providers.lima.is_enabled=true"] # Same forwarding as the docker template; see comment there. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Run the SAME setup as a Dockerfile-built workspace, directly in the VM as root. # These run after the project is synced to /mngr/code (agent-level provisioning, # post work_dir sync) and reuse the exact scripts the Dockerfile RUNs. There is @@ -347,7 +348,7 @@ setting__extend = [ "providers.modal.default_idle_timeout=86400", ] # Same forwarding as the docker/lima templates so the agent inherits its creds. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Modal has no Dockerfile build (like lima): run the same setup a Dockerfile-built # workspace would, directly in the sandbox over SSH. extra_provision_command__extend = [ @@ -382,7 +383,7 @@ idle_mode = "disabled" # agent container back after a VPS reboot; the entrypoint then self-heals sshd. start_arg__extend = ["--security-opt=no-new-privileges", "--workdir=/", "--restart=unless-stopped"] # Same forwarding as the docker template; see comment there. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Same first-boot seed as the docker template; see comment there. post_host_create_command__extend = ["/usr/local/bin/default-workspace-template-seed"] # Install the outer-VM boot unit so the agent recovers after a VPS reboot even @@ -449,7 +450,7 @@ idle_mode = "disabled" # relaunched by the outer boot unit installed below. start_arg__extend = ["--security-opt=no-new-privileges", "--workdir=/", "--restart=unless-stopped"] # Same forwarding as the docker template; see comment there. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Same first-boot seed as the docker template; see comment there. post_host_create_command__extend = ["/usr/local/bin/default-workspace-template-seed"] # Install a systemd unit on the OUTER pool/slice VM that, on every VM boot, @@ -523,7 +524,7 @@ idle_mode = "disabled" # agent container back after an EC2 reboot; the entrypoint then self-heals sshd. start_arg__extend = ["--security-opt=no-new-privileges", "--workdir=/", "--restart=unless-stopped"] # Same forwarding as the docker template; see comment there. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Same first-boot seed as the docker template; see comment there. post_host_create_command__extend = ["/usr/local/bin/default-workspace-template-seed"] # Install the outer-VM boot unit so the agent recovers after an EC2 reboot even @@ -594,12 +595,12 @@ build_arg__extend = ["--file=Dockerfile", "."] # pre-baked container, which already carries this policy. start_arg__extend = ["--restart=unless-stopped"] # Forward the LiteLLM-minted ANTHROPIC creds + the minds-side MNGR_PREFIX -# + an optional GH_TOKEN from minds' subprocess env onto the host's +# from minds' subprocess env onto the host's # ``/mngr/env``. minds writes these into the subprocess env it hands # ``mngr create`` (see ``run_mngr_create`` in # ``vendor/mngr/apps/minds/imbue/minds/desktop_client/agent_creator.py``) # so the host actually has them at the moment ``--pass-host-env`` reads. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "MNGR_PREFIX", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "MNGR_PREFIX"] # Seed /mngr/code from the baked image on first boot (idempotent warm-boot # fast path). Runs after create_host for both paths; on the fast path the # code is already present so default-workspace-template-seed is a no-op. Same as the vultr/ovh diff --git a/CLAUDE.md b/CLAUDE.md index 2962ea066..7867ad980 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ IF YOU FAIL TO FOLLOW ONE, YOU MUST EXPLICITLY CALL THAT OUT IN YOUR RESPONSE. - NEVER amend commits or rebase--always create new commits. - If you ever need to work with another *git* repo that is *outside* of this monorepo as a read-only dependency, you should do so by adding a git subtree under `vendor/`. - If you need to *actively develop* against an external repo (e.g. `mngr`), check out a standalone clone of it under `.external_worktrees//`. This directory is gitignored so the external clones don't pollute the monorepo. The branch in the external clone should mirror the branch you're on in this monorepo. -- This project uses a CLI ticket system (`tk`) for task management. Run `tk help` when you need to use it. Tickets live under `runtime/tickets/` (the path is set via the `TICKETS_DIR` env var so tickets ride the `mindsbackup/$MNGR_AGENT_ID` runtime-backup branch). +- This project uses a CLI ticket system (`tk`) for task management. Run `tk help` when you need to use it. Tickets live under `runtime/tickets/` (the path is set via the `TICKETS_DIR` env var so tickets sit with the rest of the workspace's runtime state). - All relative paths in this repo assume cwd = repo root (`/code`). Supervisord runs the services from there; any process started elsewhere (manual launch, subprocess from a different cwd) must either set cwd to the repo root or use absolute paths. State directories live under `runtime//`. - When adding a new web app, do NOT edit `libs/web_server/` -- it's an example placeholder. Use the `build-web-service` skill, which sets up a new lib + service entry + `forward_port.py` registration on its own port. @@ -229,7 +229,7 @@ The upstream is defined in `parent.toml`. # Memory Use Claude's built-in memory system. Your memory directory is `runtime/memory/` (configured via `autoMemoryDirectory` in `.claude/settings.json`). -Memory is gitignored from the main branch but is backed up automatically by the runtime-backup service onto the `mindsbackup/$MNGR_AGENT_ID` branch when `GH_TOKEN` is set, so it survives container loss. +Memory is gitignored from the main branch. When the user has enabled GitHub sync (the `github-sync` skill), the github-sync service ships it -- with the rest of `runtime/` -- to the `runtime-sync` branch of the workspace's private sync repo, so it survives container loss. # Services @@ -243,13 +243,11 @@ See the `edit-services` skill for details. Commit your changes locally. `runtime/` is gitignored from the main branch (it includes `runtime/memory/` for Claude memory and other transient state). -Chat file uploads (files a user attaches to a message) are stored in the top-level `uploads/` directory inside the repo working tree -- NOT under `runtime/`. Uploads can be arbitrarily large and any format, so they don't belong in version-controllable content; `uploads/` is gitignored. Being outside `runtime/`, uploads are NOT carried by the `runtime-backup` orphan branch (which ships only `runtime/`), but the host-level `host-backup` service (a restic snapshot of the whole host dir) does capture them, so uploads still survive container loss. See `libs/host_backup/README.md`. +Chat file uploads (files a user attaches to a message) are stored in the top-level `uploads/` directory inside the repo working tree -- NOT under `runtime/`. Uploads can be arbitrarily large and any format, so they don't belong in version-controllable content; `uploads/` is gitignored. Being outside `runtime/`, uploads are NOT carried by the opt-in GitHub sync (which ships only `runtime/`), but the host-level `host-backup` service (a restic snapshot of the whole host dir) does capture them, so uploads still survive container loss. See `libs/host_backup/README.md`. -A `post-commit` hook installed via `core.hooksPath = /mngr/code/scripts/git_hooks` auto-pushes the active branch to `origin` in the background, but only when `GH_TOKEN` is set in the environment. You do not need to push manually. The hook never blocks the commit; output is captured at `/tmp/post-commit-push.log`. +GitHub sync is opt-in via the `github-sync` skill. When the user has enabled it: a `post-commit` hook auto-pushes the active branch of every checkout to `origin` (the workspace's dedicated private repo) in the background -- you do not need to push manually; the hook never blocks the commit, and output is captured at `/tmp/post-commit-push.log`. The `github-sync` service additionally syncs `runtime/` onto a separate orphan branch (`runtime-sync`) on the same `origin`. See `libs/github_sync/README.md`. -`runtime/` is backed up automatically by the `runtime-backup` service onto a separate orphan branch (`mindsbackup/$MNGR_AGENT_ID`) on the same `origin`, also gated on `GH_TOKEN`. See `libs/runtime_backup/README.md`. - -If `GH_TOKEN` is unset, both auto-pushes silently no-op; commits stay local. +When GitHub sync is not enabled, there is no auto-push and no GitHub remote to push to; commits stay local (the restic `host-backup` still protects the whole host dir). - Don't include auto-generated lockfile churn (`uv.lock`, `package-lock.json`, etc.) in commits unless the change intentionally bumps a dependency. diff --git a/Dockerfile b/Dockerfile index 51a124624..c021cd8ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ COPY pyproject.toml uv.lock /mngr/code/ COPY libs/app_watcher/pyproject.toml /mngr/code/libs/app_watcher/pyproject.toml COPY libs/bootstrap/pyproject.toml /mngr/code/libs/bootstrap/pyproject.toml COPY libs/cloudflare_tunnel/pyproject.toml /mngr/code/libs/cloudflare_tunnel/pyproject.toml -COPY libs/runtime_backup/pyproject.toml /mngr/code/libs/runtime_backup/pyproject.toml +COPY libs/github_sync/pyproject.toml /mngr/code/libs/github_sync/pyproject.toml COPY libs/web_server/pyproject.toml /mngr/code/libs/web_server/pyproject.toml COPY apps/system_interface/pyproject.toml /mngr/code/apps/system_interface/pyproject.toml diff --git a/README.md b/README.md index 16952ea53..9e4d04039 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ mngr create my-workspace main -t local \ - `supervisord.conf` - Supervisord config defining the background services - `libs/bootstrap/` - First-boot setup, then launches supervisord to supervise the services - `vendor/mngr/` - A vendored, mutable copy of mngr. Note that making changes here *will* affect the behavior of the `mngr` command -- `vendor/tk/` - A vendored copy of the [tk](https://github.com/wedow/ticket) ticket tracker. The `ticket` script (also callable as `tk`) manages tickets stored as markdown. We point `TICKETS_DIR` at `runtime/tickets/` (set in `.mngr/settings.toml`'s `host_env`) so tickets are backed up alongside the rest of `runtime/` on the `mindsbackup/$MNGR_AGENT_ID` branch. +- `vendor/tk/` - A vendored copy of the [tk](https://github.com/wedow/ticket) ticket tracker. The `ticket` script (also callable as `tk`) manages tickets stored as markdown. We point `TICKETS_DIR` at `runtime/tickets/` (set in `.mngr/settings.toml`'s `host_env`) so tickets live alongside the rest of `runtime/` (and are covered by the opt-in GitHub sync when the `github-sync` skill has enabled it). ## Create templates diff --git a/apps/system_interface/imbue/system_interface/server.py b/apps/system_interface/imbue/system_interface/server.py index 691e82d94..64456df3c 100644 --- a/apps/system_interface/imbue/system_interface/server.py +++ b/apps/system_interface/imbue/system_interface/server.py @@ -419,7 +419,7 @@ def _interrupt_agent_endpoint(agent_id: str) -> Response: Refuses to interrupt agents carrying the ``is_primary=true`` label: that's the services agent for the workspace, and restarting it would stop the - bootstrap, web, cloudflared, and runtime-backup services. The + bootstrap, web, cloudflared, and other supervised services. The frontend already hides ``is_primary=true`` agents from the visible agent list; this is defense-in-depth for callers that hit the endpoint directly (curl, scripted use, etc.). @@ -952,7 +952,7 @@ def _destroy_agent(agent_id: str) -> Response: Refuses to destroy agents carrying the ``is_primary=true`` label: that's the services agent for the workspace, and destroying it would tear down - the bootstrap, web, cloudflared, and runtime-backup services + the bootstrap, web, cloudflared, and other supervised services along with it. The frontend already hides ``is_primary=true`` agents from the visible agent list; this is defense-in-depth for callers that hit the endpoint directly (curl, scripted use, etc.). diff --git a/changelog/mngr-github-backup-skill.md b/changelog/mngr-github-backup-skill.md new file mode 100644 index 000000000..c5c5eab2d --- /dev/null +++ b/changelog/mngr-github-backup-skill.md @@ -0,0 +1,11 @@ +GitHub sync is now opt-in via the new `github-sync` skill, replacing the always-on `runtime-backup` service. + +By default, nothing pushes to GitHub anymore: the `runtime-backup` supervisord service, bootstrap's `runtime/` worktree init (the `mindsbackup/$MNGR_AGENT_ID` orphan branch), the always-installed `core.hooksPath` post-commit auto-push, and every use of `GH_TOKEN` have been removed. `runtime/` is a plain gitignored directory until sync is enabled; the restic `host-backup` service remains the default durability story. + +The new `github-sync` skill enables sync on request: it creates a dedicated PRIVATE GitHub repo through latchkey (no token ever enters the container), points `origin` at it, wires plain `git push` through the latchkey gateway for every checkout (with the per-VPS secondary gateway as an offline fallback), installs the post-commit hook so all agent and worker commits auto-push their branch, and adds a `[program:github-sync]` service that syncs `runtime/` to the stable `runtime-sync` orphan branch every 60s. `libs/runtime_backup` was renamed and extended into `libs/github_sync` (`uv run github-sync run|wire-git|unwire-git|setup-worktree|check-visibility|status`). + +Private-only is enforced: the skill refuses public repos at setup and the service re-verifies visibility every 15 minutes, halting all pushes (service and hook) when the repo is public or unverifiable. + +A workspace recreated from a previously-synced repo self-heals: after the user re-grants the GitHub permissions, the service re-wires git and restores `runtime/` (memory, tickets, transcripts) from the `runtime-sync` branch automatically. Disable is supported (full unwind of the service, hook, and git wiring; the user chooses whether to keep the remote repo). + +Existing workspaces that update-self onto this version simply stop git-syncing `runtime/` until they opt in via the skill; legacy `mindsbackup/*` branches are not migrated. diff --git a/libs/bootstrap/README.md b/libs/bootstrap/README.md index 8eea7fef2..3c70b81ff 100644 --- a/libs/bootstrap/README.md +++ b/libs/bootstrap/README.md @@ -14,22 +14,17 @@ service. `uv run bootstrap` runs, in order: -1. **Global git config** - rewrites `git@`/`ssh://` GitHub remotes to `https://` - and points `core.hooksPath` at `scripts/git_hooks`. (This replaces the old - `git_auth_setup` extra_window, minus the retired `gh auth setup-git`.) -2. **runtime/ worktree init** - ensures `runtime/` is a git worktree of the - per-agent backup branch `mindsbackup/$MNGR_AGENT_ID`, so anything written - under `runtime/` (Claude memory, tickets, transcripts, the - `initial_chat_created` signal, etc.) is replicated off-box by the - `runtime-backup` service. Done before supervisord starts so the - runtime-backup / host-backup services find `runtime/` in place. -3. **CLAUDE_CONFIG_DIR host-env write** - records the services agent's per-agent +1. **Global git config** - rewrites `git@`/`ssh://` GitHub remotes to `https://`. + (`core.hooksPath` is deliberately NOT set here: the post-commit auto-push + hook only becomes active when the opt-in github-sync skill wires it up -- + see `libs/github_sync/README.md`.) +2. **CLAUDE_CONFIG_DIR host-env write** - records the services agent's per-agent Claude config dir in `$MNGR_HOST_DIR/env` so every other agent on the host inherits it. -4. **Initial chat agent** - on first boot only (gated by +3. **Initial chat agent** - on first boot only (gated by `runtime/initial_chat_created`), commits the rsynced workspace onto a clean `main` branch and creates the welcome chat agent (`--message /welcome`). -5. **Launch supervisord** - `exec supervisord -n -c supervisord.conf`. Running +4. **Launch supervisord** - `exec supervisord -n -c supervisord.conf`. Running via `exec` keeps the bootstrap tmux window alive as supervisord and lets the supervised services inherit this shell's already-sourced agent environment. diff --git a/libs/bootstrap/src/bootstrap/manager.py b/libs/bootstrap/src/bootstrap/manager.py index 9a8387f97..b0ab5da68 100644 --- a/libs/bootstrap/src/bootstrap/manager.py +++ b/libs/bootstrap/src/bootstrap/manager.py @@ -1,12 +1,10 @@ """Bootstrap: first-boot setup, then launch supervisord. `uv run bootstrap` runs once per container boot (from the `bootstrap` -extra_window). It performs first-boot setup -- global git config, ensuring -runtime/ exists as a git worktree of the per-agent backup branch -(mindsbackup/$MNGR_AGENT_ID), writing CLAUDE_CONFIG_DIR into the host env, -and creating the initial chat agent -- and then -`exec`s the system supervisord in the foreground. supervisord (configured by -supervisord.conf) owns every background service from then on. +extra_window). It performs first-boot setup -- global git config, writing +CLAUDE_CONFIG_DIR into the host env, and creating the initial chat agent -- +and then `exec`s the system supervisord in the foreground. supervisord +(configured by supervisord.conf) owns every background service from then on. Running supervisord via exec keeps the bootstrap tmux window alive as supervisord and lets the supervised services inherit this shell's already- @@ -15,7 +13,6 @@ import json import os -import shutil import subprocess from pathlib import Path @@ -29,13 +26,10 @@ SUPERVISOR_LOG_DIR = Path("/var/log/supervisor") RUNTIME_DIR = Path("runtime") -RUNTIME_PREEXISTING_DIR = Path("runtime.preexisting") -RUNTIME_BACKUP_USER_NAME = "runtime-backup" -RUNTIME_BACKUP_USER_EMAIL = "runtime-backup@mindsbackup.local" # Signal file gating exactly-once creation of the initial chat agent. Lives -# under runtime/ so the runtime-backup service replicates it to the -# mindsbackup/$MNGR_AGENT_ID branch (survives container loss). +# under runtime/, which persists with the container volume (and is synced to +# GitHub when the opt-in github-sync skill has been enabled). INITIAL_CHAT_SIGNAL = RUNTIME_DIR / "initial_chat_created" # Basename (under $MNGR_HOST_DIR) of the file holding the initial chat agent's id, # read by system_interface's welcome_resend to address the resend by id. @@ -47,9 +41,11 @@ _HOST_DIR_ENV_VAR = "MNGR_HOST_DIR" _CLAUDE_CONFIG_DIR_ENV_VAR = "CLAUDE_CONFIG_DIR" -# Global git config the old git_auth_setup extra_window used to apply (minus the -# retired `gh auth setup-git`): rewrite git@ / ssh:// GitHub remotes to https and -# point core.hooksPath at the repo's git_hooks. +# Global git config applied on every boot: rewrite git@ / ssh:// GitHub +# remotes to https (there are no SSH credentials in the container; https is +# what the latchkey gateway wiring rewrites when github-sync is enabled). +# core.hooksPath is deliberately NOT set here -- the post-commit auto-push +# hook only becomes active when the github-sync skill wires it up. _GIT_GLOBAL_CONFIG_ARGVS = ( ( "config", @@ -65,7 +61,6 @@ "url.https://github.com/.insteadOf", "ssh://git@github.com/", ), - ("config", "--global", "core.hooksPath", "/mngr/code/scripts/git_hooks"), ) @@ -412,7 +407,7 @@ def _maybe_create_initial_chat() -> None: Touches the signal file only on a successful create -- a failed create leaves the signal file absent so the next bootstrap run retries. The user's manually-destroyed initial chat agent is *not* recreated, - because the signal file persists in the runtime-backup branch. + because the signal file persists in runtime/. """ if INITIAL_CHAT_SIGNAL.exists(): logger.debug( @@ -448,12 +443,11 @@ def _bootstrap_init_chat_dir() -> None: def _configure_git_global() -> None: - """Apply the global git config the old git_auth_setup extra_window set. + """Apply the boot-time global git config. - Rewrites git@ / ssh:// GitHub remotes to https and points core.hooksPath at - the repo's git_hooks (see _GIT_GLOBAL_CONFIG_ARGVS). The retired - `gh auth setup-git` step is intentionally dropped. Best-effort: a failure - here should not block the supervisord launch. + Rewrites git@ / ssh:// GitHub remotes to https (see + _GIT_GLOBAL_CONFIG_ARGVS). Best-effort: a failure here should not block + the supervisord launch. """ for argv in _GIT_GLOBAL_CONFIG_ARGVS: result = subprocess.run( @@ -494,239 +488,13 @@ def _exec_supervisord() -> None: os.execvp("supervisord", ["supervisord", "-n", "-c", str(SUPERVISORD_CONF)]) -def _git_noninteractive_env() -> dict[str, str]: - """Environment for bootstrap git calls: never prompt for credentials. - - Git prompts for a username/password on the controlling TTY (bypassing our - captured pipes) when a remote needs auth and no credential is available -- - e.g. the "best-effort" fetch in _init_runtime_worktree against a PRIVATE - origin (a mind created from a private inspiration repo) with no GH_TOKEN. - That prompt blocks bootstrap forever, so supervisord never starts and the - workspace sits on "Loading workspace" indefinitely. GIT_TERMINAL_PROMPT=0 - turns the prompt into a fast failure, which every caller here already - handles (all bootstrap git calls are best-effort by design). - """ - return {**os.environ, "GIT_TERMINAL_PROMPT": "0"} - - -def _git_main(*args: str) -> subprocess.CompletedProcess[str]: - """Run a git command in the main checkout, never raising or prompting.""" - return subprocess.run( - ["git", *args], - capture_output=True, - text=True, - check=False, - env=_git_noninteractive_env(), - ) - - -def _git_runtime(*args: str) -> subprocess.CompletedProcess[str]: - """Run a git command inside the runtime worktree, never raising or prompting.""" - return subprocess.run( - ["git", "-C", str(RUNTIME_DIR), *args], - capture_output=True, - text=True, - check=False, - env=_git_noninteractive_env(), - ) - - -def _restore_preexisting_into_worktree() -> None: - """Move any files from runtime.preexisting/ back into runtime/.""" - if not RUNTIME_PREEXISTING_DIR.exists(): - return - for entry in list(RUNTIME_PREEXISTING_DIR.iterdir()): - target = RUNTIME_DIR / entry.name - if target.exists(): - # Don't clobber what the worktree already has (e.g. a fresh - # .gitignore we just wrote). - continue - shutil.move(str(entry), str(target)) - try: - RUNTIME_PREEXISTING_DIR.rmdir() - except OSError: - logger.warning( - "{} not empty after restore; leaving for inspection", - RUNTIME_PREEXISTING_DIR, - ) - - -def _stage_preexisting_aside() -> None: - """Move runtime/'s contents to runtime.preexisting/ so we can add a worktree. - - Only called when runtime/ exists with files but is not yet a git worktree. - """ - if RUNTIME_PREEXISTING_DIR.exists(): - # Stale leftover from a prior failed init -- clear it. - shutil.rmtree(RUNTIME_PREEXISTING_DIR) - shutil.move(str(RUNTIME_DIR), str(RUNTIME_PREEXISTING_DIR)) - - -def _runtime_dir_has_files() -> bool: - """Return True if runtime/ exists and contains anything.""" - if not RUNTIME_DIR.exists(): - return False - return any(RUNTIME_DIR.iterdir()) - - -def _create_orphan_runtime_worktree(branch: str) -> subprocess.CompletedProcess[str]: - """Add runtime/ as a worktree on a fresh orphan branch, git-version-agnostically. - - `git worktree add --orphan` only exists in git >= 2.42, but the Lima - provider's Debian 12 base ships git 2.39. So build the orphan branch with - plumbing that has worked for ages -- a parentless commit on the empty tree -- - then do a normal `git worktree add` for it. Returns the final worktree-add - CompletedProcess; if an earlier plumbing step fails, returns that failing - CompletedProcess so the caller's existing error handling fires. - """ - empty_tree = _git_main("hash-object", "-w", "-t", "tree", "/dev/null") - if empty_tree.returncode != 0: - return empty_tree - # Commit identity is passed via -c because the container may have no global - # git identity yet, and commit-tree refuses to run without one. - orphan_commit = _git_main( - "-c", - f"user.name={RUNTIME_BACKUP_USER_NAME}", - "-c", - f"user.email={RUNTIME_BACKUP_USER_EMAIL}", - "commit-tree", - empty_tree.stdout.strip(), - "-m", - "runtime backup: init", - ) - if orphan_commit.returncode != 0: - return orphan_commit - branch_result = _git_main("branch", branch, orphan_commit.stdout.strip()) - if branch_result.returncode != 0: - return branch_result - return _git_main("worktree", "add", str(RUNTIME_DIR), branch) - - -def _init_runtime_worktree() -> None: - """One-time setup of runtime/ as a worktree of mindsbackup/$MNGR_AGENT_ID. - - Best-effort: logs and returns rather than raising on any failure, so a - transient git problem does not prevent other services from starting. - """ - agent_id = os.environ.get("MNGR_AGENT_ID") - if not agent_id: - logger.warning( - "MNGR_AGENT_ID is unset; skipping runtime worktree init " - "(runtime-backup service will also no-op)" - ) - return - - branch = f"mindsbackup/{agent_id}" - - if (RUNTIME_DIR / ".git").exists(): - logger.info("runtime/ is already a worktree; skipping init") - # A prior init may have staged runtime/ content aside and been killed - # before restoring it (leaving runtime.preexisting/ behind while the - # worktree itself already exists). Recover that content now rather - # than stranding it. _restore_preexisting_into_worktree no-ops when - # runtime.preexisting/ is absent, which is the common case. - _restore_preexisting_into_worktree() - return - - logger.info("Initializing runtime worktree on branch {}", branch) - - # Best-effort fetch so we can detect a pre-existing remote branch (e.g. - # restored after a container restart on the same agent id). - fetch_result = _git_main("fetch", "origin", branch) - remote_ref = f"origin/{branch}" - has_remote = ( - fetch_result.returncode == 0 - and _git_main("rev-parse", "--verify", remote_ref).returncode == 0 - ) - - staged_aside = False - if _runtime_dir_has_files(): - logger.warning( - "runtime/ already has files; staging them aside before adding the worktree" - ) - _stage_preexisting_aside() - staged_aside = True - - if has_remote: - result = _git_main( - "worktree", "add", "-B", branch, str(RUNTIME_DIR), remote_ref - ) - else: - result = _create_orphan_runtime_worktree(branch) - - if result.returncode != 0: - logger.error( - "git worktree add failed (rc={}): {}", - result.returncode, - result.stderr.strip(), - ) - # Restore preexisting files so other services don't lose them. - if staged_aside: - if not RUNTIME_DIR.exists(): - shutil.move(str(RUNTIME_PREEXISTING_DIR), str(RUNTIME_DIR)) - else: - _restore_preexisting_into_worktree() - return - - # Configure bot identity for backup commits inside this worktree only. - _git_runtime("config", "user.name", RUNTIME_BACKUP_USER_NAME) - _git_runtime("config", "user.email", RUNTIME_BACKUP_USER_EMAIL) - - if has_remote: - # Make sure the local branch tracks the remote (some git versions - # don't set this automatically with -B + an explicit ref). - _git_runtime("branch", "--set-upstream-to", remote_ref) - else: - # Fresh orphan branch: write the .gitignore for secrets and make an - # initial empty commit so push has something to push. - gitignore = RUNTIME_DIR / ".gitignore" - gitignore.write_text("secrets\n") - _git_runtime("add", ".gitignore") - commit = _git_runtime("commit", "--allow-empty", "-m", "runtime backup: init") - if commit.returncode != 0: - logger.error( - "initial commit failed (rc={}): {}", - commit.returncode, - commit.stderr.strip(), - ) - - # Restore staged-aside content. Calling unconditionally (rather than - # gating on the `staged_aside` flag) also recovers content left by a - # prior init that staged aside but was killed before it could restore. - _restore_preexisting_into_worktree() - - if os.environ.get("GH_TOKEN"): - if has_remote: - push = _git_runtime("push") - else: - push = _git_runtime("push", "--set-upstream", "origin", branch) - if push.returncode != 0: - logger.warning( - "initial push failed (rc={}): {} (runtime-backup service will retry)", - push.returncode, - push.stderr.strip(), - ) - else: - logger.info("No GH_TOKEN; skipping initial push") - - def main() -> None: logger.info("Bootstrap starting: first-boot setup, then supervisord") - # Apply the global git config (https rewrites + repo hooksPath) before any - # service or agent runs git. Replaces the old git_auth_setup extra_window. + # Apply the global git config (https rewrites) before any service or + # agent runs git. _configure_git_global() - # Restore runtime/ FIRST so the initial_chat_created signal file (which - # lives inside the worktree and is replicated to mindsbackup/$MNGR_AGENT_ID - # by the runtime-backup service) is in place before we decide whether to - # create the initial chat agent. Without this ordering, every container - # restart sees an empty runtime/, treats the boot as first-ever, and - # re-creates the welcome chat agent (and auto-commits any uncommitted - # work_dir state). This must also happen before supervisord starts the - # runtime-backup / host-backup services that write into runtime/. - _init_runtime_worktree() - _bootstrap_init_chat_dir() # Make sure supervisord's log directory exists, then hand off: replace this diff --git a/libs/bootstrap/src/bootstrap/manager_test.py b/libs/bootstrap/src/bootstrap/manager_test.py index 2a2e2991e..d49642f18 100644 --- a/libs/bootstrap/src/bootstrap/manager_test.py +++ b/libs/bootstrap/src/bootstrap/manager_test.py @@ -16,7 +16,6 @@ INITIAL_CHAT_AGENT_ID_FILENAME, _build_create_chat_command, _configure_git_global, - _create_orphan_runtime_worktree, _ensure_host_claude_config_dir, _format_env_file, _initialize_workspace_main_branch, @@ -32,12 +31,14 @@ # --- _configure_git_global --- -def test_configure_git_global_sets_insteadof_and_hookspath( +def test_configure_git_global_sets_insteadof_but_not_hookspath( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: # Isolate the global git config to a tmp file so the test does not touch the # developer's real ~/.gitconfig. _configure_git_global should set both - # insteadOf rewrites (git@ and ssh://) plus core.hooksPath. + # insteadOf rewrites (git@ and ssh://). core.hooksPath must NOT be set: + # the post-commit auto-push hook only becomes active when the opt-in + # github-sync skill wires it up. gitconfig = tmp_path / ".gitconfig" monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(gitconfig)) @@ -59,7 +60,7 @@ def test_configure_git_global_sets_insteadof_and_hookspath( text=True, check=False, ).stdout.strip() - assert hooks_path == "/mngr/code/scripts/git_hooks" + assert hooks_path == "" # --- Env-file helpers --- @@ -513,33 +514,3 @@ def test_initialize_workspace_main_branch_is_idempotent_on_clean_main( assert branch == "main" -# --- _create_orphan_runtime_worktree --- - - -def test_create_orphan_runtime_worktree_creates_empty_orphan_branch( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """The helper must add runtime/ as a worktree on a brand-new orphan branch - (no parent, empty tree) using only plumbing that works on old git -- no - `git worktree add --orphan` (git >= 2.42), which Lima's Debian 12 lacks.""" - repo = tmp_path / "repo" - repo.mkdir() - _git_in(repo, "init", "-q", "--initial-branch=main") - _git_in(repo, "config", "user.email", "seed@test.local") - _git_in(repo, "config", "user.name", "seed") - (repo / "README.md").write_text("seed\n") - _git_in(repo, "add", "-A") - _git_in(repo, "commit", "-qm", "seed") - - monkeypatch.chdir(repo) - result = _create_orphan_runtime_worktree("mindsbackup/agent-test") - assert result.returncode == 0, result.stderr - - runtime = repo / "runtime" - assert (runtime / ".git").exists() - # The branch is an orphan: its tip commit has no parents. - parents = _git_in(runtime, "rev-list", "--parents", "-1", "HEAD").stdout.split() - assert len(parents) == 1 # just the commit sha, no parent sha - # Nothing is tracked yet (empty tree); the worktree has no repo content. - assert _git_in(runtime, "ls-files").stdout.strip() == "" - assert not (runtime / "README.md").exists() diff --git a/libs/browser/README.md b/libs/browser/README.md index b53137c69..226af7163 100644 --- a/libs/browser/README.md +++ b/libs/browser/README.md @@ -38,7 +38,7 @@ agent, identified by its `MNGR_AGENT_ID`, or the human). its own persistent Chromium profile under `$MNGR_HOST_DIR/browser-profiles/` (Tier A -- on the workspace volume), so cookies/logins/history come back; Chromium does this itself, we just point `user_data_dir` at a durable dir. A tiny manifest - (`runtime/browser-fleet.json`, Tier B -- git-backed to the mindsbackup branch) + (`runtime/browser-fleet.json`, Tier B -- covered by the opt-in GitHub sync) records which browsers existed and their tab URLs, so even a full rebuild restores the tab list (logged out, since profiles are volume-only). On daemon startup the fleet is restored **eager-sequentially** (one browser at a time, no cold-boot diff --git a/libs/browser/src/browser/manifest.py b/libs/browser/src/browser/manifest.py index 10eff02fa..34bcfc829 100644 --- a/libs/browser/src/browser/manifest.py +++ b/libs/browser/src/browser/manifest.py @@ -6,7 +6,7 @@ and dies with the old container) and NO Chromium profile bytes (cookies/logins/ history live in each browser's persistent ``user_data_dir`` on the workspace volume; see ``session.py``). Because it's tiny JSON it can live under ``runtime/`` and ride -the mindsbackup branch, so even a full container rebuild restores the tab list. +the opt-in GitHub sync, so even a full container rebuild restores the tab list. Pure synchronous file IO (no asyncio here, on purpose): writes are atomic via a temp file + ``os.replace`` so a reader on the next boot sees either the old or the diff --git a/libs/browser/src/browser/session.py b/libs/browser/src/browser/session.py index dca7b0ffb..3b20d5416 100644 --- a/libs/browser/src/browser/session.py +++ b/libs/browser/src/browser/session.py @@ -208,8 +208,8 @@ def _repo_root() -> Path: # Per-browser persistent Chromium profiles (cookies/logins/history) live here, on the # workspace volume under $MNGR_HOST_DIR -- Tier A durability: they survive stop/start # and restart of a single workspace (lost only on a permanent delete). They are NOT -# under runtime/ (which is git-backed to the mindsbackup branch) -- a fat, churny -# profile would bloat that branch. Override the root for tests / alternate layouts. +# under runtime/ (which the opt-in GitHub sync ships to a git branch) -- a fat, +# churny profile would bloat that branch. Override the root for tests / alternate layouts. _PROFILE_ROOT = Path( os.environ.get( "BROWSER_PROFILE_ROOT", diff --git a/libs/github_sync/README.md b/libs/github_sync/README.md new file mode 100644 index 000000000..1cd30100e --- /dev/null +++ b/libs/github_sync/README.md @@ -0,0 +1,73 @@ +# github_sync + +Opt-in GitHub sync for a workspace. Nothing in this library is active by +default: the `github-sync` skill enables it for a workspace whose user asks +for it, and the skill (not this library) creates the dedicated **private** +GitHub repo and points `origin` at it. + +Once enabled, three pieces work together: + +1. **The service** (`uv run github-sync run`, supervised as + `[program:github-sync]`): every 60 seconds it commits the contents of + `runtime/` (Claude memory, ticket state, transcripts, app port registry, + etc.) on the stable orphan branch `runtime-sync` -- checked out as a git + worktree at `runtime/` -- and pushes it to `origin`. +2. **The git wiring** (`uv run github-sync wire-git`): global git config that + rewrites `https://github.com/...` remotes to the latchkey gateway's git + proxy and attaches the gateway auth headers, so a plain `git push` works + for every checkout in the container (main repo, worker worktrees, the + runtime worktree). The GitHub credential is injected server-side by the + gateway; no token ever enters the container. The wiring also points + `core.hooksPath` at `scripts/git_hooks`, activating the post-commit + auto-push hook for every checkout. +3. **The post-commit hook** (`scripts/git_hooks/post-commit`, in the repo but + inert until the hooks path is wired): auto-pushes the active branch of any + checkout after each commit, so both main-agent and worker commits land on + the GitHub remote without manual pushes. + +## Behavior + +- Sync is configured iff `github_sync.toml` exists at the repo root (it holds + `repo_url`); the skill writes it. Without it the service idles. +- Tick interval: 60 seconds. Each tick: `git add -A`, commit (only if dirty) + with message `runtime sync: `, push. All `git` + operations target the runtime worktree. +- Pushes go through the latchkey gateway on the user's machine, falling back + to the per-VPS secondary gateway (remote hosts only) when the user's + machine is offline. A failed push is retried on the next tick; `--force` is + never used (the service is the branch's only writer). +- **Private-only enforcement**: the service re-checks the repo's visibility + through latchkey every 15 minutes and refuses to push while the repo is + public or its visibility cannot be confirmed. The post-commit hook consults + the same status and skips pushes during a halt. +- `runtime/secrets` is excluded via the worktree's own `.gitignore` (written + at init), so e.g. the Cloudflare tunnel token never reaches the remote. +- Each tick first clears a stale `index.lock` from the runtime worktree if + one is present and older than the tick interval (a killed git process never + cleans up its own lock, and without this every later `git add` would fail + identically and syncing would stop permanently and silently). +- On any git failure the service logs to stderr and `/tmp/github-sync.log` + and tries again on the next tick. It mirrors machine-readable status to + `/tmp/github-sync-status.json` (read by the hook and the skill). + +## Restoring on a fresh container + +If a workspace is recreated from a previously-synced repo, the synced-in +supervisord config already contains the `[program:github-sync]` block, but +the latchkey permissions and the container-local git wiring/worktree do not +carry over. The service self-heals: each tick it re-applies the wiring and +retries the worktree init, which fetches origin's `runtime-sync` branch and +materializes the prior `runtime/` state (memory, tickets, transcripts) as +soon as the user re-grants the GitHub permissions (the github-sync skill +walks them through it). + +## CLI + +``` +uv run github-sync run # the service loop (supervisord) +uv run github-sync wire-git # apply gateway git config + hooks path +uv run github-sync unwire-git # remove them (disable path) +uv run github-sync setup-worktree # create/restore the runtime/ worktree +uv run github-sync check-visibility # print private/public/unknown; rc!=0 unless private +uv run github-sync status # config + latest service status as JSON +``` diff --git a/libs/github_sync/pyproject.toml b/libs/github_sync/pyproject.toml new file mode 100644 index 000000000..de2cfd56d --- /dev/null +++ b/libs/github_sync/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "github-sync" +version = "0.1.0" +description = "Opt-in service that syncs runtime/ (and, via a post-commit hook, every branch) to a private GitHub repo through the latchkey gateway" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "click>=8.0", + "loguru>=0.7.0", +] + +[project.scripts] +github-sync = "github_sync.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/github_sync"] diff --git a/libs/runtime_backup/src/runtime_backup/__init__.py b/libs/github_sync/src/github_sync/__init__.py similarity index 100% rename from libs/runtime_backup/src/runtime_backup/__init__.py rename to libs/github_sync/src/github_sync/__init__.py diff --git a/libs/github_sync/src/github_sync/cli.py b/libs/github_sync/src/github_sync/cli.py new file mode 100644 index 000000000..67fc67f91 --- /dev/null +++ b/libs/github_sync/src/github_sync/cli.py @@ -0,0 +1,95 @@ +"""CLI for github-sync: the service loop plus the helpers the skill drives. + +`uv run github-sync run` is what the [program:github-sync] supervisord block +executes; the remaining subcommands are one-shot steps invoked by the +github-sync skill during enable / disable / status. +""" + +import json + +import click + +from github_sync import runner +from github_sync.config import GithubSyncConfigError, load_repo_url +from github_sync.visibility import VISIBILITY_PRIVATE, check_repo_visibility +from github_sync.wiring import apply_git_wiring, remove_git_wiring +from github_sync.worktree import init_runtime_worktree, is_runtime_worktree + + +@click.group() +def main() -> None: + """Manage the opt-in GitHub sync for this workspace.""" + + +@main.command() +def run() -> None: + """Run the periodic runtime/ sync service loop (used by supervisord).""" + runner.run_forever() + + +@main.command("wire-git") +def wire_git() -> None: + """Route git's GitHub access through the latchkey gateway (global config).""" + if not apply_git_wiring(): + raise SystemExit(1) + click.echo("wired") + + +@main.command("unwire-git") +def unwire_git() -> None: + """Remove the gateway git config and the hooks path (disable path).""" + remove_git_wiring() + click.echo("unwired") + + +@main.command("setup-worktree") +def setup_worktree() -> None: + """Create runtime/ as a worktree of runtime-sync, restoring from origin if it exists there.""" + if not init_runtime_worktree(): + raise SystemExit(1) + click.echo("ready") + + +@main.command("check-visibility") +def check_visibility() -> None: + """Print the sync repo's visibility; exits nonzero unless confirmed private.""" + try: + repo_url = load_repo_url() + except GithubSyncConfigError as e: + raise click.ClickException(str(e)) from e + if repo_url is None: + raise click.ClickException("sync is not configured (github_sync.toml missing)") + visibility = check_repo_visibility(repo_url) + click.echo(visibility) + if visibility != VISIBILITY_PRIVATE: + raise SystemExit(1) + + +@main.command() +def status() -> None: + """Print the sync configuration and the service's latest status as JSON.""" + try: + repo_url = load_repo_url() + config_error = None + except GithubSyncConfigError as e: + repo_url = None + config_error = str(e) + service_status = None + status_path = runner.status_file_path() + if status_path.exists(): + try: + service_status = json.loads(status_path.read_text()) + except (OSError, json.JSONDecodeError) as e: + service_status = {"error": f"unreadable status file: {e}"} + payload = { + "is_configured": repo_url is not None, + "repo_url": repo_url, + "config_error": config_error, + "is_runtime_worktree": is_runtime_worktree(), + "service": service_status, + } + click.echo(json.dumps(payload, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/libs/github_sync/src/github_sync/config.py b/libs/github_sync/src/github_sync/config.py new file mode 100644 index 000000000..41e45cc88 --- /dev/null +++ b/libs/github_sync/src/github_sync/config.py @@ -0,0 +1,105 @@ +"""Shared configuration and constants for the opt-in GitHub sync. + +GitHub sync is disabled by default. The github-sync skill enables it by +writing github_sync.toml at the repo root (the presence of that file is the +"sync is configured" marker), wiring git up to push through the latchkey +gateway, and adding the [program:github-sync] supervisord service. +""" + +import os +import tomllib +from pathlib import Path + +# All relative paths assume cwd = repo root (/mngr/code), matching +# supervisord's `directory=` and the other template services. +RUNTIME_DIR = Path("runtime") +CONFIG_PATH = Path("github_sync.toml") + +# The stable branch that runtime/ state is synced to. The sync repo is a +# dedicated private repo per workspace, so no per-agent namespacing is needed. +SYNC_BRANCH = "runtime-sync" + +GITHUB_URL_PREFIX = "https://github.com/" + +# Env vars injected by mngr_latchkey's prepare_agent_latchkey for every agent. +# The secondary gateway (remote VPS hosts only) keeps working when the user's +# own machine -- which runs the primary gateway -- is offline. +ENV_GATEWAY = "LATCHKEY_GATEWAY" +ENV_GATEWAY_SECONDARY = "LATCHKEY_GATEWAY_SECONDARY" +ENV_GATEWAY_PASSWORD = "LATCHKEY_GATEWAY_PASSWORD" +ENV_GATEWAY_PERMISSIONS_OVERRIDE = "LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE" + + +class GithubSyncError(Exception): + """Base error for the github_sync library.""" + + +class GithubSyncConfigError(GithubSyncError, ValueError): + """Raised when github_sync.toml exists but is not a valid sync config.""" + + +def load_repo_url() -> str | None: + """Return the sync repo URL from github_sync.toml, or None when sync is not configured. + + Raises GithubSyncConfigError when the file exists but is malformed: a + skill-authored config problem should be loud, not silently ignored. + """ + if not CONFIG_PATH.exists(): + return None + try: + raw_text = CONFIG_PATH.read_text() + except OSError as e: + raise GithubSyncConfigError(f"Cannot read {CONFIG_PATH}: {e}") from e + try: + data = tomllib.loads(raw_text) + except tomllib.TOMLDecodeError as e: + raise GithubSyncConfigError(f"Cannot parse {CONFIG_PATH}: {e}") from e + repo_url = data.get("repo_url") + if not isinstance(repo_url, str) or not repo_url.startswith(GITHUB_URL_PREFIX): + raise GithubSyncConfigError( + f"{CONFIG_PATH} must set repo_url to an " + f"{GITHUB_URL_PREFIX}/ URL, got {repo_url!r}" + ) + return repo_url.removesuffix(".git").rstrip("/") + + +def parse_owner_and_name(repo_url: str) -> tuple[str, str]: + """Split a GitHub repo URL into (owner, name). + + Raises GithubSyncConfigError when the URL is not of the form + https://github.com//. + """ + path = repo_url.removeprefix(GITHUB_URL_PREFIX).removesuffix(".git") + parts = [part for part in path.split("/") if part] + if len(parts) != 2: + raise GithubSyncConfigError( + f"Repo URL must be {GITHUB_URL_PREFIX}/, got {repo_url!r}" + ) + return parts[0], parts[1] + + +def get_gateway_url() -> str | None: + """The primary latchkey gateway URL (no trailing slash), or None if unset.""" + value = os.environ.get(ENV_GATEWAY, "").rstrip("/") + return value or None + + +def get_secondary_gateway_url() -> str | None: + """The per-VPS backup gateway URL (no trailing slash), or None if unset.""" + value = os.environ.get(ENV_GATEWAY_SECONDARY, "").rstrip("/") + return value or None + + +def get_gateway_password() -> str | None: + value = os.environ.get(ENV_GATEWAY_PASSWORD, "") + return value or None + + +def get_gateway_permissions_override() -> str | None: + value = os.environ.get(ENV_GATEWAY_PERMISSIONS_OVERRIDE, "") + return value or None + + +def proxied_url(gateway_url: str, target_url: str) -> str: + """The gateway's git smart-HTTP proxy URL for a GitHub target URL.""" + return f"{gateway_url}/gateway/{target_url}" diff --git a/libs/github_sync/src/github_sync/config_test.py b/libs/github_sync/src/github_sync/config_test.py new file mode 100644 index 000000000..87dc6837e --- /dev/null +++ b/libs/github_sync/src/github_sync/config_test.py @@ -0,0 +1,103 @@ +"""Unit tests for github_sync config parsing and gateway env helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from github_sync.config import ( + GithubSyncConfigError, + get_gateway_password, + get_gateway_url, + get_secondary_gateway_url, + load_repo_url, + parse_owner_and_name, + proxied_url, +) + + +def test_load_repo_url_returns_none_when_unconfigured( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + assert load_repo_url() is None + + +def test_load_repo_url_normalizes_git_suffix_and_trailing_slash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text( + 'repo_url = "https://github.com/some-user/my-workspace.git"\n' + ) + assert load_repo_url() == "https://github.com/some-user/my-workspace" + + +def test_load_repo_url_raises_on_malformed_toml( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text("repo_url = [not toml") + with pytest.raises(GithubSyncConfigError): + load_repo_url() + + +def test_load_repo_url_raises_on_non_github_url( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text( + 'repo_url = "https://gitlab.com/user/repo"\n' + ) + with pytest.raises(GithubSyncConfigError): + load_repo_url() + + +def test_load_repo_url_raises_when_repo_url_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text('other_key = "value"\n') + with pytest.raises(GithubSyncConfigError): + load_repo_url() + + +def test_parse_owner_and_name_splits_url() -> None: + assert parse_owner_and_name("https://github.com/some-user/my-repo") == ( + "some-user", + "my-repo", + ) + assert parse_owner_and_name("https://github.com/some-user/my-repo.git") == ( + "some-user", + "my-repo", + ) + + +def test_parse_owner_and_name_rejects_malformed_url() -> None: + with pytest.raises(GithubSyncConfigError): + parse_owner_and_name("https://github.com/just-an-owner") + with pytest.raises(GithubSyncConfigError): + parse_owner_and_name("https://github.com/a/b/c") + + +def test_proxied_url_composes_gateway_git_proxy() -> None: + assert ( + proxied_url("http://127.0.0.1:39000", "https://github.com/u/r.git") + == "http://127.0.0.1:39000/gateway/https://github.com/u/r.git" + ) + + +def test_gateway_env_getters_strip_and_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LATCHKEY_GATEWAY", "http://127.0.0.1:39000/") + assert get_gateway_url() == "http://127.0.0.1:39000" + monkeypatch.delenv("LATCHKEY_GATEWAY", raising=False) + assert get_gateway_url() is None + monkeypatch.setenv("LATCHKEY_GATEWAY_SECONDARY", "") + assert get_secondary_gateway_url() is None + monkeypatch.delenv("LATCHKEY_GATEWAY_PASSWORD", raising=False) + assert get_gateway_password() is None + monkeypatch.setenv("LATCHKEY_GATEWAY_PASSWORD", "pw-123") + assert get_gateway_password() == "pw-123" diff --git a/libs/github_sync/src/github_sync/conftest.py b/libs/github_sync/src/github_sync/conftest.py new file mode 100644 index 000000000..0fa1b425d --- /dev/null +++ b/libs/github_sync/src/github_sync/conftest.py @@ -0,0 +1,44 @@ +"""Fixtures shared by the github_sync test files.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + + +@pytest.fixture +def isolated_git_and_gateway_env( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> Path: + """Isolate global git config and clear all latchkey/gateway env vars. + + Keeps tests from ever touching the developer's real ~/.gitconfig (wiring + writes global config) and from picking up a real gateway from the + environment. Returns the isolated global gitconfig path. + """ + gitconfig = tmp_path / "gitconfig" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(gitconfig)) + for name in ( + "LATCHKEY_GATEWAY", + "LATCHKEY_GATEWAY_SECONDARY", + "LATCHKEY_GATEWAY_PASSWORD", + "LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE", + ): + monkeypatch.delenv(name, raising=False) + # Steer subprocesses away from any inherited status-file override. + monkeypatch.setenv( + "GITHUB_SYNC_STATUS_FILE", str(tmp_path / "github-sync-status.json") + ) + return gitconfig + + +@pytest.fixture +def fake_latchkey_bin(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """A directory prepended to PATH where tests install a fake `latchkey`.""" + bin_dir = tmp_path / "fakebin" + bin_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}") + return bin_dir diff --git a/libs/github_sync/src/github_sync/runner.py b/libs/github_sync/src/github_sync/runner.py new file mode 100644 index 000000000..b7885842b --- /dev/null +++ b/libs/github_sync/src/github_sync/runner.py @@ -0,0 +1,353 @@ +"""GitHub sync service loop. + +Polls runtime/ every TICK_INTERVAL_SECONDS; if there are uncommitted changes, +makes a sync commit on the runtime-sync branch checked out at runtime/ and +pushes it to origin through the latchkey gateway. Periodically re-verifies +that the sync repo is still private and halts pushes when it is public or its +visibility cannot be established. + +The service only exists when the github-sync skill has enabled sync (it adds +the [program:github-sync] block to supervisord.conf). The skill normally also +creates the runtime/ worktree; if it is missing (e.g. a workspace recreated +from a previously-synced repo), each tick retries the worktree init, which +restores runtime/ from origin once the latchkey GitHub permissions have been +re-granted. +""" + +import json +import os +import time +from datetime import datetime, timezone +from pathlib import Path + +from loguru import logger + +from github_sync.config import ( + SYNC_BRANCH, + GithubSyncConfigError, + get_gateway_password, + get_secondary_gateway_url, + load_repo_url, + proxied_url, +) +from github_sync.visibility import ( + VISIBILITY_PRIVATE, + VISIBILITY_PUBLIC, + VISIBILITY_UNKNOWN, + check_repo_visibility, +) +from github_sync.wiring import PASSWORD_HEADER, apply_git_wiring +from github_sync.worktree import ( + git_runtime, + init_runtime_worktree, + is_runtime_worktree, +) + +TICK_INTERVAL_SECONDS = 60 +# How long a confirmed visibility answer stays fresh before it is re-checked. +# Failed checks are retried every tick instead. +VISIBILITY_CHECK_INTERVAL_SECONDS = 900 +LOG_FILE = Path("/tmp/github-sync.log") +# Machine-readable status mirror, read by the post-commit hook (to respect a +# visibility halt) and by the github-sync skill's status report. Lives in /tmp +# deliberately: it is per-boot state, not something to sync. The default path +# is what scripts/git_hooks/post-commit reads. +DEFAULT_STATUS_FILE = Path("/tmp/github-sync-status.json") + +# Minimum age before an index.lock is treated as stale and removed. A real git +# operation on the small runtime/ tree holds the lock for well under a second, +# so a lock older than this cannot belong to a live operation. Set to the tick +# interval so a lock skipped as possibly-live on one tick is guaranteed old +# enough to clear on the next. +STALE_LOCK_MIN_AGE_SECONDS = TICK_INTERVAL_SECONDS + + +class _SyncState: + """Mutable sync status carried across ticks and mirrored to the status file.""" + + def __init__(self) -> None: + self.repo_url: str | None = None + self.visibility: str = VISIBILITY_UNKNOWN + self.visibility_checked_at: datetime | None = None + self.last_push_ok: bool | None = None + self.last_push_at: datetime | None = None + self.last_error: str | None = None + + @property + def is_push_allowed(self) -> bool: + return self.visibility == VISIBILITY_PRIVATE + + +def status_file_path() -> Path: + """The status-mirror path; overridable via env so tests stay isolated.""" + override = os.environ.get("GITHUB_SYNC_STATUS_FILE", "") + return Path(override) if override else DEFAULT_STATUS_FILE + + +def _now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def _iso_or_none(moment: datetime | None) -> str | None: + """ISO-8601 with a trailing Z (e.g. 2026-05-06T17:42:13Z), or None.""" + if moment is None: + return None + return moment.isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _clear_stale_index_lock() -> None: + """Remove a stale ``index.lock`` from the runtime worktree, if present. + + github-sync is the only writer of this worktree's git index and its ticks + run strictly sequentially, so a lock here is normally stale -- left behind + by a previous tick's git process that was killed before it could release + the lock (whenever something kills the process mid-commit). Git never + clears such a lock itself, so without this every subsequent ``git add`` + fails identically and syncing stops forever. + + The removal is deliberately *not* unconditional: to stay safe even if the + single-writer assumption is ever violated (a concurrent git process in the + worktree), the lock is only removed once it is older than + ``STALE_LOCK_MIN_AGE_SECONDS``. A live git operation on the small runtime/ + tree holds the lock for well under a second, so an older lock cannot be + live; a genuinely in-progress operation is left untouched and reconsidered + on the next tick. + + The lock path is resolved via ``--absolute-git-dir`` so it is correct + whether runtime/ is a normal repo or (as in production) a linked worktree + with a per-worktree git dir. + """ + git_dir_result = git_runtime("rev-parse", "--absolute-git-dir") + if git_dir_result.returncode != 0: + # runtime/ is not a git repo yet; nothing to clear. + return + lock_path = Path(git_dir_result.stdout.strip()) / "index.lock" + try: + lock_age_seconds = time.time() - lock_path.stat().st_mtime + except OSError: + # No lock present (the common case), or it vanished underneath us. + return + if lock_age_seconds < STALE_LOCK_MIN_AGE_SECONDS: + logger.warning( + "git index.lock at {} is only {:.0f}s old; leaving it in case a " + "git operation is in progress (will reconsider next tick)", + lock_path, + lock_age_seconds, + ) + return + logger.warning( + "Removing stale git index lock at {} ({:.0f}s old)", + lock_path, + lock_age_seconds, + ) + try: + lock_path.unlink() + except OSError as e: + logger.warning("Failed to remove stale index lock at {}: {}", lock_path, e) + + +def _has_uncommitted_changes() -> bool: + """Return True if runtime/ has anything to commit.""" + result = git_runtime("status", "--porcelain") + if result.returncode != 0: + logger.warning( + "git status failed (rc={}): {}", result.returncode, result.stderr.strip() + ) + return False + return bool(result.stdout.strip()) + + +def _commit_runtime_changes() -> None: + """Stage and commit anything new in runtime/ (no-op when clean).""" + add_result = git_runtime("add", "-A") + if add_result.returncode != 0: + logger.warning( + "git add failed (rc={}): {}", + add_result.returncode, + add_result.stderr.strip(), + ) + return + if _has_uncommitted_changes(): + commit_result = git_runtime( + "commit", "-m", f"runtime sync: {_iso_or_none(_now_utc())}" + ) + if commit_result.returncode != 0: + logger.warning( + "git commit failed (rc={}): {}", + commit_result.returncode, + commit_result.stderr.strip(), + ) + + +def _refresh_visibility(state: _SyncState, repo_url: str) -> None: + """Re-check repo visibility when the last confirmed answer has gone stale. + + A completed check updates the state; a failed check (UNKNOWN result) + leaves the previous answer in place and is retried next tick. Transitions + are logged loudly since a repo flipping public is a security condition. + """ + if state.visibility_checked_at is not None and state.visibility != VISIBILITY_UNKNOWN: + age_seconds = (_now_utc() - state.visibility_checked_at).total_seconds() + if age_seconds < VISIBILITY_CHECK_INTERVAL_SECONDS: + return + visibility = check_repo_visibility(repo_url) + if visibility == VISIBILITY_UNKNOWN: + logger.debug("Could not check visibility of {}; will retry", repo_url) + return + if visibility != state.visibility: + if visibility == VISIBILITY_PRIVATE: + logger.info("Sync repo {} confirmed private; pushes enabled", repo_url) + else: + logger.error( + "Sync repo {} is PUBLIC; halting all sync pushes until it is " + "made private again", + repo_url, + ) + state.visibility = visibility + state.visibility_checked_at = _now_utc() + + +def _push_via_secondary_gateway(repo_url: str) -> bool: + """Push runtime-sync through the per-VPS backup gateway, if one exists. + + Used when the primary gateway (on the user's machine) is unreachable. The + secondary takes only the gateway password header -- no permissions + override -- per the latchkey contract. The proxied URL is passed + explicitly so the global insteadOf rewrite (which targets the primary + gateway) does not apply. + """ + secondary_url = get_secondary_gateway_url() + password = get_gateway_password() + if secondary_url is None or password is None: + return False + push_url = proxied_url(secondary_url, f"{repo_url}.git") + result = git_runtime( + "-c", + f"http.extraHeader={PASSWORD_HEADER}: {password}", + "push", + push_url, + f"{SYNC_BRANCH}:{SYNC_BRANCH}", + ) + if result.returncode != 0: + logger.debug( + "secondary-gateway push failed (rc={}): {}", + result.returncode, + result.stderr.strip(), + ) + return result.returncode == 0 + + +def _push_runtime(state: _SyncState, repo_url: str) -> None: + """Push the runtime-sync branch, self-healing wiring drift along the way. + + Always attempts a push (even on a clean tick) so commits whose push failed + earlier still get shipped. The ladder: plain push; re-apply gateway wiring + (the gateway URL embeds a port that can change across restarts) and retry; + set-upstream (heals a lost upstream); secondary gateway. + """ + push_result = git_runtime("push") + if push_result.returncode != 0: + apply_git_wiring() + push_result = git_runtime("push") + if push_result.returncode != 0: + push_result = git_runtime("push", "--set-upstream", "origin", SYNC_BRANCH) + is_pushed = push_result.returncode == 0 + if not is_pushed: + is_pushed = _push_via_secondary_gateway(repo_url) + state.last_push_ok = is_pushed + state.last_push_at = _now_utc() + if is_pushed: + state.last_error = None + else: + state.last_error = push_result.stderr.strip() + logger.warning( + "git push failed (rc={}): {}", + push_result.returncode, + push_result.stderr.strip(), + ) + + +def _write_status(state: _SyncState) -> None: + """Mirror the sync state to the status file for the hook and the skill.""" + payload = { + "timestamp": _iso_or_none(_now_utc()), + "repo_url": state.repo_url, + "visibility": state.visibility, + "visibility_checked_at": _iso_or_none(state.visibility_checked_at), + "is_push_allowed": state.is_push_allowed, + "last_push_ok": state.last_push_ok, + "last_push_at": _iso_or_none(state.last_push_at), + "last_error": state.last_error, + } + status_path = status_file_path() + try: + status_path.write_text(json.dumps(payload, indent=2) + "\n") + except OSError as e: + logger.warning("Failed to write status file {}: {}", status_path, e) + + +def _do_tick(state: _SyncState) -> None: + """Run one sync tick: commit runtime/ changes, verify visibility, push.""" + try: + repo_url = load_repo_url() + except GithubSyncConfigError as e: + logger.error("Invalid sync config: {}", e) + state.last_error = str(e) + _write_status(state) + return + if repo_url is None: + logger.warning( + "github_sync.toml is missing; sync is not configured (run the " + "github-sync skill), idling" + ) + _write_status(state) + return + state.repo_url = repo_url + + if not is_runtime_worktree(): + # Self-healing path for a workspace recreated from a synced repo: the + # wiring and worktree are container-local, so recreate them here once + # the gateway/permissions allow it. + apply_git_wiring() + if not init_runtime_worktree(): + state.last_error = "runtime/ worktree init deferred (origin unreachable)" + _write_status(state) + return + + # Clear any stale index.lock first, so a commit interrupted by something + # killing the process cannot wedge every future tick. + _clear_stale_index_lock() + _commit_runtime_changes() + + _refresh_visibility(state, repo_url) + if state.is_push_allowed: + _push_runtime(state, repo_url) + elif state.visibility == VISIBILITY_PUBLIC: + logger.error( + "Sync repo {} is PUBLIC; refusing to push (make it private again " + "to resume syncing)", + repo_url, + ) + else: + logger.warning( + "Sync repo {} visibility not confirmed yet; holding pushes", repo_url + ) + _write_status(state) + + +def run_forever() -> None: + """Main loop: poll runtime/ on a fixed interval and sync changes.""" + # Tee stderr-bound logs into LOG_FILE so operators can `tail` the file + # across restarts of just this service window. /tmp wipes on container + # restart, which is the intended scope for the debug log. Set up here + # rather than at module import so that merely importing this module + # (e.g. from tests) does not start writing to the log file. + logger.add(LOG_FILE, level="INFO") + + logger.info("Starting github-sync (interval={}s)", TICK_INTERVAL_SECONDS) + apply_git_wiring() + + state = _SyncState() + while True: + _do_tick(state) + time.sleep(TICK_INTERVAL_SECONDS) diff --git a/libs/github_sync/src/github_sync/runner_test.py b/libs/github_sync/src/github_sync/runner_test.py new file mode 100644 index 000000000..ac4823260 --- /dev/null +++ b/libs/github_sync/src/github_sync/runner_test.py @@ -0,0 +1,233 @@ +"""Unit tests for the github-sync runner. + +Covers the stale-index-lock recovery (an interrupted commit must not wedge +every future tick) and full ticks against a local bare origin: commit+push +when the repo is confirmed private, and the push halt when it is public or +unverifiable. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import time +from pathlib import Path + +import pytest + +from github_sync.runner import ( + STALE_LOCK_MIN_AGE_SECONDS, + _clear_stale_index_lock, + _do_tick, + _SyncState, + status_file_path, +) +from github_sync.testing import init_repo, init_repo_with_origin, install_fake_latchkey +from github_sync.worktree import init_runtime_worktree + + +def _age_lock(lock_path: Path) -> None: + """Backdate a lock file's mtime so it counts as stale (past the age guard).""" + old = time.time() - STALE_LOCK_MIN_AGE_SECONDS - 60 + os.utime(lock_path, (old, old)) + + +def _git_out(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True, check=False + ) + + +def test_clear_stale_index_lock_removes_aged_lock_in_linked_worktree( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Faithful to production: runtime/ is a linked worktree, so the lock + lives in the per-worktree git dir, not in a top-level .git/.""" + monkeypatch.chdir(tmp_path) + main = tmp_path / "main" + init_repo(main) + (main / "seed.txt").write_text("seed\n") + _git_out(main, "add", "-A") + _git_out(main, "commit", "-qm", "seed") + # The runner resolves runtime/ relative to its cwd, so the worktree must + # be named exactly "runtime" and sit in tmp_path. + _git_out(main, "worktree", "add", str(tmp_path / "runtime"), "-b", "sync") + + lock_path = main / ".git" / "worktrees" / "runtime" / "index.lock" + lock_path.write_text("") + _age_lock(lock_path) + assert lock_path.exists() + + _clear_stale_index_lock() + + assert not lock_path.exists() + + +def test_clear_stale_index_lock_keeps_fresh_lock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A recently-created lock might belong to a live git operation, so it + must be left alone rather than yanked out from under that operation.""" + monkeypatch.chdir(tmp_path) + runtime = tmp_path / "runtime" + init_repo(runtime) + lock_path = runtime / ".git" / "index.lock" + lock_path.write_text("") # fresh: mtime is now + + _clear_stale_index_lock() + + assert lock_path.exists() + + +def test_clear_stale_index_lock_noop_when_no_lock_present( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + init_repo(tmp_path / "runtime") + # No lock and no git repo problems -- must simply not raise. + _clear_stale_index_lock() + + +def test_clear_stale_index_lock_noop_when_runtime_not_a_repo( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "runtime").mkdir() + # `git rev-parse` fails; the function must silently return. + _clear_stale_index_lock() + + +def _set_up_synced_workspace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[Path, Path]: + """A workspace with a local bare origin, a runtime worktree, and sync config.""" + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + (main / "github_sync.toml").write_text( + 'repo_url = "https://github.com/some-user/my-workspace"\n' + ) + assert init_runtime_worktree() is True + return main, origin + + +def test_do_tick_commits_and_pushes_when_repo_private( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """End to end against a local bare origin: new runtime state is committed + on runtime-sync and pushed once the repo is confirmed private.""" + main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": true}\'') + (main / "runtime" / "memory.txt").write_text("important state\n") + state = _SyncState() + + _do_tick(state) + + remote_log = _git_out(origin, "log", "--oneline", "runtime-sync") + assert remote_log.returncode == 0 + assert "runtime sync:" in remote_log.stdout + assert state.last_push_ok is True + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is True + assert status["visibility"] == "private" + assert status["last_push_ok"] is True + + +def test_do_tick_self_heals_stale_index_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """A stale index.lock from a killed prior tick must not wedge syncing: + the next tick clears it and commits the pending runtime state.""" + main, _ = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": true}\'') + lock_path = main / ".git" / "worktrees" / "runtime" / "index.lock" + lock_path.write_text("") + _age_lock(lock_path) + (main / "runtime" / "memory.txt").write_text("important state\n") + + _do_tick(_SyncState()) + + log = _git_out(main / "runtime", "log", "--oneline") + assert "runtime sync:" in log.stdout + + +def test_do_tick_halts_pushes_when_repo_public( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """Public repo => commit locally (nothing is lost) but never push.""" + main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": false}\'') + (main / "runtime" / "memory.txt").write_text("important state\n") + state = _SyncState() + + _do_tick(state) + + # Committed locally... + local_log = _git_out(main / "runtime", "log", "--oneline") + assert "runtime sync:" in local_log.stdout + # ...but the branch never reached origin. + remote_branch = _git_out(origin, "rev-parse", "--verify", "runtime-sync") + assert remote_branch.returncode != 0 + assert state.is_push_allowed is False + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is False + assert status["visibility"] == "public" + + +def test_do_tick_holds_pushes_while_visibility_unconfirmed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """An unreachable visibility check (gateway offline) must fail closed.""" + main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, "exit 7") + (main / "runtime" / "memory.txt").write_text("important state\n") + + _do_tick(_SyncState()) + + remote_branch = _git_out(origin, "rev-parse", "--verify", "runtime-sync") + assert remote_branch.returncode != 0 + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is False + assert status["visibility"] == "unknown" + + +def test_do_tick_idles_when_sync_not_configured( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, +) -> None: + monkeypatch.chdir(tmp_path) + + state = _SyncState() + _do_tick(state) + + assert state.repo_url is None + status = json.loads(status_file_path().read_text()) + assert status["repo_url"] is None + + +def test_do_tick_reports_malformed_config( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text("repo_url = [broken") + + state = _SyncState() + _do_tick(state) + + assert state.last_error is not None + assert "github_sync.toml" in state.last_error diff --git a/libs/github_sync/src/github_sync/testing.py b/libs/github_sync/src/github_sync/testing.py new file mode 100644 index 000000000..1cceca1d0 --- /dev/null +++ b/libs/github_sync/src/github_sync/testing.py @@ -0,0 +1,52 @@ +"""Shared test helpers for the github_sync test files.""" + +from __future__ import annotations + +import stat +import subprocess +from pathlib import Path + + +def run_git(repo: Path, *args: str) -> None: + """Run a git command in `repo`, raising on failure.""" + subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True) + + +def init_repo(repo: Path) -> None: + """Create a git repo at `repo` with a committer identity configured.""" + repo.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["git", "init", "-q", "--initial-branch=main", str(repo)], + check=True, + capture_output=True, + ) + run_git(repo, "config", "user.email", "test@test.local") + run_git(repo, "config", "user.name", "test") + + +def init_repo_with_origin(base: Path) -> tuple[Path, Path]: + """Create a seeded main repo at base/main with a bare origin at base/origin.git.""" + main = base / "main" + init_repo(main) + (main / "seed.txt").write_text("seed\n") + run_git(main, "add", "-A") + run_git(main, "commit", "-qm", "seed") + origin = base / "origin.git" + subprocess.run( + ["git", "init", "-q", "--bare", str(origin)], check=True, capture_output=True + ) + run_git(main, "remote", "add", "origin", str(origin)) + return main, origin + + +def install_fake_latchkey(bin_dir: Path, script_body: str) -> None: + """Install an executable `latchkey` shell script into `bin_dir`. + + The caller prepends `bin_dir` to PATH (via monkeypatch.setenv) so + check_repo_visibility exercises its real subprocess path against a + controllable stand-in instead of the real latchkey CLI. + """ + bin_dir.mkdir(parents=True, exist_ok=True) + script = bin_dir / "latchkey" + script.write_text(f"#!/usr/bin/env bash\n{script_body}\n") + script.chmod(script.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) diff --git a/libs/github_sync/src/github_sync/visibility.py b/libs/github_sync/src/github_sync/visibility.py new file mode 100644 index 000000000..55e0c63cc --- /dev/null +++ b/libs/github_sync/src/github_sync/visibility.py @@ -0,0 +1,105 @@ +"""Sync-repo visibility checking through latchkey. + +Public repos must never be synced to: agents might push secrets or other +sensitive data without thinking about it. The skill verifies the repo is +private at enable time, and the service re-checks periodically, halting +pushes whenever the repo is public or its visibility cannot be established. +""" + +import json +import os +import subprocess + +from loguru import logger + +from github_sync.config import ( + ENV_GATEWAY, + ENV_GATEWAY_PERMISSIONS_OVERRIDE, + get_secondary_gateway_url, + parse_owner_and_name, +) + +VISIBILITY_PRIVATE = "private" +VISIBILITY_PUBLIC = "public" +VISIBILITY_UNKNOWN = "unknown" + +_LATCHKEY_CURL_TIMEOUT_SECONDS = 60 + + +def parse_visibility_response(body: str) -> str: + """Map a GitHub `GET /repos//` response body to a visibility. + + Anything that is not an explicit `"private": true/false` (error bodies, + truncated output, a 404 for a deleted repo) is UNKNOWN, which callers + treat as push-blocking. + """ + try: + data = json.loads(body) + except json.JSONDecodeError: + return VISIBILITY_UNKNOWN + if not isinstance(data, dict): + return VISIBILITY_UNKNOWN + is_private = data.get("private") + if is_private is True: + return VISIBILITY_PRIVATE + elif is_private is False: + return VISIBILITY_PUBLIC + else: + return VISIBILITY_UNKNOWN + + +def _latchkey_curl_env(is_secondary: bool) -> dict[str, str] | None: + """Env for a `latchkey curl` call; None means inherit (primary gateway).""" + if not is_secondary: + return None + secondary_url = get_secondary_gateway_url() + if secondary_url is None: + return None + # Per the latchkey skill: the secondary gateway takes no permissions + # override, so it must be cleared alongside the gateway swap. + return { + **os.environ, + ENV_GATEWAY: secondary_url, + ENV_GATEWAY_PERMISSIONS_OVERRIDE: "", + } + + +def check_repo_visibility(repo_url: str) -> str: + """Ask GitHub (via latchkey) whether the sync repo is private. + + Tries the primary gateway first, then the secondary (per-VPS) gateway so + the check keeps working when the user's machine is offline. Returns + UNKNOWN when neither gateway produces a parseable answer. + """ + owner, name = parse_owner_and_name(repo_url) + api_url = f"https://api.github.com/repos/{owner}/{name}" + gateway_attempts = [False] + if get_secondary_gateway_url() is not None: + gateway_attempts.append(True) + for is_secondary in gateway_attempts: + # latchkey curl injects the GitHub credential server-side; -s keeps + # stdout parseable. + try: + result = subprocess.run( + ["latchkey", "curl", "-s", api_url], + capture_output=True, + text=True, + check=False, + env=_latchkey_curl_env(is_secondary), + timeout=_LATCHKEY_CURL_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired) as e: + logger.debug("latchkey curl failed (secondary={}): {}", is_secondary, e) + continue + if result.returncode != 0: + logger.debug( + "latchkey curl exited {} (secondary={}): {}", + result.returncode, + is_secondary, + result.stderr.strip(), + ) + continue + visibility = parse_visibility_response(result.stdout) + if visibility != VISIBILITY_UNKNOWN: + return visibility + return VISIBILITY_UNKNOWN diff --git a/libs/github_sync/src/github_sync/visibility_test.py b/libs/github_sync/src/github_sync/visibility_test.py new file mode 100644 index 000000000..ff19670d0 --- /dev/null +++ b/libs/github_sync/src/github_sync/visibility_test.py @@ -0,0 +1,74 @@ +"""Unit tests for repo-visibility checking (the private-only enforcement).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from github_sync.testing import install_fake_latchkey +from github_sync.visibility import ( + VISIBILITY_PRIVATE, + VISIBILITY_PUBLIC, + VISIBILITY_UNKNOWN, + check_repo_visibility, + parse_visibility_response, +) + +_REPO_URL = "https://github.com/some-user/my-workspace" + + +def test_parse_visibility_response_private() -> None: + assert parse_visibility_response('{"private": true}') == VISIBILITY_PRIVATE + + +def test_parse_visibility_response_public() -> None: + assert parse_visibility_response('{"private": false}') == VISIBILITY_PUBLIC + + +def test_parse_visibility_response_unknown_on_error_bodies() -> None: + # A 404 body (deleted repo), non-JSON output, and non-object JSON must all + # come back UNKNOWN, which blocks pushes. + assert parse_visibility_response('{"message": "Not Found"}') == VISIBILITY_UNKNOWN + assert parse_visibility_response("curl: connection refused") == VISIBILITY_UNKNOWN + assert parse_visibility_response("[1, 2]") == VISIBILITY_UNKNOWN + assert parse_visibility_response("") == VISIBILITY_UNKNOWN + + +def test_check_repo_visibility_reads_private_from_latchkey( + fake_latchkey_bin: Path, +) -> None: + install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": true}\'') + assert check_repo_visibility(_REPO_URL) == VISIBILITY_PRIVATE + + +def test_check_repo_visibility_detects_public_repo(fake_latchkey_bin: Path) -> None: + install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": false}\'') + assert check_repo_visibility(_REPO_URL) == VISIBILITY_PUBLIC + + +def test_check_repo_visibility_unknown_when_latchkey_fails( + fake_latchkey_bin: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Exit code 7 is what latchkey curl produces when the gateway is offline. + monkeypatch.delenv("LATCHKEY_GATEWAY_SECONDARY", raising=False) + install_fake_latchkey(fake_latchkey_bin, "exit 7") + assert check_repo_visibility(_REPO_URL) == VISIBILITY_UNKNOWN + + +def test_check_repo_visibility_falls_back_to_secondary_gateway( + fake_latchkey_bin: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The fake gateway only answers when called with the secondary gateway's + # env override (primary attempt fails), proving the fallback fires with + # the documented env shape (secondary URL + cleared permissions override). + monkeypatch.setenv("LATCHKEY_GATEWAY", "http://primary.invalid:1") + monkeypatch.setenv("LATCHKEY_GATEWAY_SECONDARY", "http://127.0.0.1:46123") + monkeypatch.setenv("LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE", "some-jwt") + install_fake_latchkey( + fake_latchkey_bin, + 'if [ "$LATCHKEY_GATEWAY" = "http://127.0.0.1:46123" ] ' + '&& [ -z "$LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE" ]; then ' + "echo '{\"private\": true}'; else exit 7; fi", + ) + assert check_repo_visibility(_REPO_URL) == VISIBILITY_PRIVATE diff --git a/libs/github_sync/src/github_sync/wiring.py b/libs/github_sync/src/github_sync/wiring.py new file mode 100644 index 000000000..ef5367671 --- /dev/null +++ b/libs/github_sync/src/github_sync/wiring.py @@ -0,0 +1,134 @@ +"""Global git config wiring that routes GitHub traffic through the latchkey gateway. + +Applied by the github-sync skill at enable time and re-applied by the service +when a push fails (self-healing a gateway URL that changed across container +restarts). With the wiring in place, plain `git push` / `git fetch` against any +https://github.com/... remote is transparently rewritten to the gateway's git +proxy and authenticated with the gateway headers -- for every checkout in the +container (main repo, worker worktrees, the runtime/ worktree). The GitHub +credential itself is injected server-side by the gateway; no token ever enters +the container. + +The wiring also points core.hooksPath at the repo's git_hooks so the +post-commit auto-push hook applies everywhere. +""" + +import subprocess + +from loguru import logger + +from github_sync.config import ( + GITHUB_URL_PREFIX, + get_gateway_password, + get_gateway_permissions_override, + get_gateway_url, + proxied_url, +) + +HOOKS_PATH = "/mngr/code/scripts/git_hooks" +PASSWORD_HEADER = "X-Latchkey-Gateway-Password" +PERMISSIONS_OVERRIDE_HEADER = "X-Latchkey-Gateway-Permissions-Override" + + +def _git_config(*args: str) -> subprocess.CompletedProcess[str]: + """Run a `git config --global` command, never raising.""" + return subprocess.run( + ["git", "config", "--global", *args], + capture_output=True, + text=True, + check=False, + ) + + +def _list_global_config(key_regexp: str) -> list[tuple[str, str]]: + """List (key, value) pairs of global git config entries matching a key regexp.""" + result = _git_config("--get-regexp", key_regexp) + if result.returncode != 0: + # rc=1 simply means no matching entries. + return [] + entries: list[tuple[str, str]] = [] + for line in result.stdout.splitlines(): + key, _, value = line.partition(" ") + if key: + entries.append((key, value)) + return entries + + +def _remove_gateway_entries(kept_gateway_url: str | None) -> None: + """Remove latchkey-gateway git config entries for every other gateway URL. + + With `kept_gateway_url=None` this removes all of them (the disable path). + Gateway URLs embed a reverse-tunneled local port that can change across + container restarts, so re-wiring must clean up entries pointing at a stale + port -- two insteadOf rewrites for the same prefix would be ambiguous. + """ + kept_insteadof_key = ( + f"url.{proxied_url(kept_gateway_url, GITHUB_URL_PREFIX)}.insteadof" + if kept_gateway_url is not None + else None + ) + for key, value in _list_global_config(r"url\..*\.insteadof"): + is_gateway_rewrite = value == GITHUB_URL_PREFIX and "/gateway/" in key + if is_gateway_rewrite and key.lower() != (kept_insteadof_key or "").lower(): + _git_config("--unset-all", key) + + kept_header_key = ( + f"http.{kept_gateway_url}/.extraheader" if kept_gateway_url is not None else None + ) + for key, value in _list_global_config(r"http\..*\.extraheader"): + is_gateway_header = value.startswith( + (PASSWORD_HEADER, PERMISSIONS_OVERRIDE_HEADER) + ) + if is_gateway_header and key.lower() != (kept_header_key or "").lower(): + _git_config("--unset-all", key) + + +def apply_git_wiring() -> bool: + """Install the gateway rewrite, auth headers, and hooks path in global git config. + + Idempotent; safe to re-run on every service self-heal. Returns False (with + a logged warning) when the latchkey gateway env vars are not all present. + """ + gateway_url = get_gateway_url() + password = get_gateway_password() + permissions_override = get_gateway_permissions_override() + if not gateway_url or not password or not permissions_override: + logger.warning( + "Latchkey gateway env is incomplete (need LATCHKEY_GATEWAY, " + "LATCHKEY_GATEWAY_PASSWORD, LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE); " + "cannot wire git for GitHub sync" + ) + return False + + _remove_gateway_entries(gateway_url) + + header_key = f"http.{gateway_url}/.extraHeader" + config_steps = ( + ( + "--replace-all", + f"url.{proxied_url(gateway_url, GITHUB_URL_PREFIX)}.insteadOf", + GITHUB_URL_PREFIX, + ), + ("--replace-all", header_key, f"{PASSWORD_HEADER}: {password}"), + ("--add", header_key, f"{PERMISSIONS_OVERRIDE_HEADER}: {permissions_override}"), + ("--replace-all", "core.hooksPath", HOOKS_PATH), + ) + for argv in config_steps: + result = _git_config(*argv) + if result.returncode != 0: + logger.warning( + "git config --global {} failed (rc={}): {}", + " ".join(argv), + result.returncode, + result.stderr.strip(), + ) + return False + logger.debug("Wired git GitHub access through gateway {}", gateway_url) + return True + + +def remove_git_wiring() -> None: + """Remove every gateway git config entry and the hooks path (disable path).""" + _remove_gateway_entries(None) + _git_config("--unset-all", "core.hooksPath") + logger.info("Removed GitHub sync git wiring from global git config") diff --git a/libs/github_sync/src/github_sync/wiring_test.py b/libs/github_sync/src/github_sync/wiring_test.py new file mode 100644 index 000000000..27d9150df --- /dev/null +++ b/libs/github_sync/src/github_sync/wiring_test.py @@ -0,0 +1,98 @@ +"""Unit tests for the latchkey-gateway git config wiring.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from github_sync.wiring import HOOKS_PATH, apply_git_wiring, remove_git_wiring + + +def _get_all_global(key: str) -> list[str]: + result = subprocess.run( + ["git", "config", "--global", "--get-all", key], + capture_output=True, + text=True, + check=False, + ) + return result.stdout.splitlines() + + +def _set_gateway_env(monkeypatch: pytest.MonkeyPatch, gateway_url: str) -> None: + monkeypatch.setenv("LATCHKEY_GATEWAY", gateway_url) + monkeypatch.setenv("LATCHKEY_GATEWAY_PASSWORD", "pw-abc") + monkeypatch.setenv("LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE", "jwt-xyz") + + +def test_apply_git_wiring_writes_rewrite_headers_and_hookspath( + isolated_git_and_gateway_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_gateway_env(monkeypatch, "http://127.0.0.1:41234") + + assert apply_git_wiring() is True + + insteadof_key = "url.http://127.0.0.1:41234/gateway/https://github.com/.insteadOf" + assert _get_all_global(insteadof_key) == ["https://github.com/"] + headers = _get_all_global("http.http://127.0.0.1:41234/.extraHeader") + assert headers == [ + "X-Latchkey-Gateway-Password: pw-abc", + "X-Latchkey-Gateway-Permissions-Override: jwt-xyz", + ] + assert _get_all_global("core.hooksPath") == [HOOKS_PATH] + + +def test_apply_git_wiring_fails_without_gateway_env( + isolated_git_and_gateway_env: Path, +) -> None: + assert apply_git_wiring() is False + assert _get_all_global("core.hooksPath") == [] + + +def test_apply_git_wiring_replaces_stale_gateway_entries( + isolated_git_and_gateway_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A gateway URL embeds a port that changes across restarts; re-wiring must + remove the previous port's entries or git would see two ambiguous + rewrites for the same prefix.""" + _set_gateway_env(monkeypatch, "http://127.0.0.1:41234") + assert apply_git_wiring() is True + _set_gateway_env(monkeypatch, "http://127.0.0.1:59999") + + assert apply_git_wiring() is True + + stale_key = "url.http://127.0.0.1:41234/gateway/https://github.com/.insteadOf" + fresh_key = "url.http://127.0.0.1:59999/gateway/https://github.com/.insteadOf" + assert _get_all_global(stale_key) == [] + assert _get_all_global(fresh_key) == ["https://github.com/"] + assert _get_all_global("http.http://127.0.0.1:41234/.extraHeader") == [] + assert len(_get_all_global("http.http://127.0.0.1:59999/.extraHeader")) == 2 + + +def test_remove_git_wiring_clears_everything_but_keeps_ssh_rewrites( + isolated_git_and_gateway_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The bootstrap-owned ssh->https rewrite shares the value shape but is not + # gateway wiring and must survive a disable. + subprocess.run( + [ + "git", + "config", + "--global", + "url.https://github.com/.insteadOf", + "git@github.com:", + ], + check=True, + capture_output=True, + ) + _set_gateway_env(monkeypatch, "http://127.0.0.1:41234") + assert apply_git_wiring() is True + + remove_git_wiring() + + gateway_key = "url.http://127.0.0.1:41234/gateway/https://github.com/.insteadOf" + assert _get_all_global(gateway_key) == [] + assert _get_all_global("http.http://127.0.0.1:41234/.extraHeader") == [] + assert _get_all_global("core.hooksPath") == [] + assert _get_all_global("url.https://github.com/.insteadOf") == ["git@github.com:"] diff --git a/libs/github_sync/src/github_sync/worktree.py b/libs/github_sync/src/github_sync/worktree.py new file mode 100644 index 000000000..25a4e62ba --- /dev/null +++ b/libs/github_sync/src/github_sync/worktree.py @@ -0,0 +1,258 @@ +"""Setup and restore of runtime/ as a git worktree on the runtime-sync branch. + +Run by the github-sync skill at enable time, and retried by the service on +every tick while runtime/ is not yet a worktree (the self-healing path for a +workspace recreated from a previously-synced repo: once the user re-grants +the latchkey GitHub permissions, the next tick fetches origin's runtime-sync +branch and materializes the prior runtime/ state -- memory, tickets, +transcripts -- automatically). + +Any files already sitting in runtime/ (a fresh boot writes signal files and +tickets there before sync is enabled) are staged aside during worktree +creation and restored on top afterwards, never clobbering restored state. +""" + +import os +import shutil +import subprocess + +from loguru import logger + +from github_sync.config import RUNTIME_DIR, SYNC_BRANCH + +SYNC_USER_NAME = "github-sync" +SYNC_USER_EMAIL = "github-sync@minds.local" + +_RUNTIME_PREEXISTING_DIR = RUNTIME_DIR.with_name(RUNTIME_DIR.name + ".preexisting") + +# `git ls-remote --exit-code` exits 2 when the remote is reachable but has no +# matching refs. +_LS_REMOTE_NO_MATCHING_REFS = 2 + + +def _git_noninteractive_env() -> dict[str, str]: + """Environment for sync git calls: never prompt for credentials. + + GIT_TERMINAL_PROMPT=0 turns any credential prompt (e.g. a fetch through a + gateway whose permission grant lapsed) into a fast failure instead of a + TTY prompt that would wedge the caller forever; every caller here already + logs-and-continues on a nonzero exit. + """ + return {**os.environ, "GIT_TERMINAL_PROMPT": "0"} + + +def git_main(*args: str) -> subprocess.CompletedProcess[str]: + """Run a git command in the main checkout, never raising or prompting.""" + return subprocess.run( + ["git", *args], + capture_output=True, + text=True, + check=False, + env=_git_noninteractive_env(), + ) + + +def git_runtime(*args: str) -> subprocess.CompletedProcess[str]: + """Run a git command inside the runtime worktree, never raising or prompting.""" + return subprocess.run( + ["git", "-C", str(RUNTIME_DIR), *args], + capture_output=True, + text=True, + check=False, + env=_git_noninteractive_env(), + ) + + +def is_runtime_worktree() -> bool: + """Return True when runtime/ is already a git worktree.""" + return (RUNTIME_DIR / ".git").exists() + + +def _restore_preexisting_into_worktree() -> None: + """Move any files from runtime.preexisting/ back into runtime/.""" + if not _RUNTIME_PREEXISTING_DIR.exists(): + return + for entry in list(_RUNTIME_PREEXISTING_DIR.iterdir()): + target = RUNTIME_DIR / entry.name + if target.exists(): + # Don't clobber what the worktree already has (e.g. restored + # state fetched from origin, or a fresh .gitignore). + continue + shutil.move(str(entry), str(target)) + try: + _RUNTIME_PREEXISTING_DIR.rmdir() + except OSError: + logger.warning( + "{} not empty after restore; leaving for inspection", + _RUNTIME_PREEXISTING_DIR, + ) + + +def _stage_preexisting_aside() -> None: + """Move runtime/'s contents to runtime.preexisting/ so we can add a worktree. + + Only called when runtime/ exists with files but is not yet a git worktree. + """ + if _RUNTIME_PREEXISTING_DIR.exists(): + # Stale leftover from a prior failed init -- clear it. + shutil.rmtree(_RUNTIME_PREEXISTING_DIR) + shutil.move(str(RUNTIME_DIR), str(_RUNTIME_PREEXISTING_DIR)) + + +def _runtime_dir_has_files() -> bool: + """Return True if runtime/ exists and contains anything.""" + if not RUNTIME_DIR.exists(): + return False + return any(RUNTIME_DIR.iterdir()) + + +def _create_orphan_runtime_worktree(branch: str) -> subprocess.CompletedProcess[str]: + """Add runtime/ as a worktree on a fresh orphan branch, git-version-agnostically. + + `git worktree add --orphan` only exists in git >= 2.42, but the Lima + provider's Debian 12 base ships git 2.39. So build the orphan branch with + plumbing that has worked for ages -- a parentless commit on the empty tree -- + then do a normal `git worktree add` for it. Returns the final worktree-add + CompletedProcess; if an earlier plumbing step fails, returns that failing + CompletedProcess so the caller's existing error handling fires. + """ + empty_tree = git_main("hash-object", "-w", "-t", "tree", "/dev/null") + if empty_tree.returncode != 0: + return empty_tree + # Commit identity is passed via -c because the container may have no global + # git identity yet, and commit-tree refuses to run without one. + orphan_commit = git_main( + "-c", + f"user.name={SYNC_USER_NAME}", + "-c", + f"user.email={SYNC_USER_EMAIL}", + "commit-tree", + empty_tree.stdout.strip(), + "-m", + "runtime sync: init", + ) + if orphan_commit.returncode != 0: + return orphan_commit + branch_result = git_main("branch", branch, orphan_commit.stdout.strip()) + if branch_result.returncode != 0: + return branch_result + return git_main("worktree", "add", str(RUNTIME_DIR), branch) + + +def init_runtime_worktree() -> bool: + """One-time setup of runtime/ as a worktree of the runtime-sync branch. + + Returns True when runtime/ is a worktree afterwards. Never raises: a + failure (most commonly the origin remote being unreachable because the + latchkey GitHub permission has not been granted yet) is logged and the + caller retries later. Deliberately refuses to create a *fresh* orphan + branch while origin is unreachable -- a remote runtime-sync branch might + exist (workspace recreated from a synced repo), and starting an unrelated + local history would leave the two permanently diverged. + """ + if is_runtime_worktree(): + # A prior init may have staged runtime/ content aside and been killed + # before restoring it. Recover that content now rather than stranding + # it; this no-ops when runtime.preexisting/ is absent (the common case). + _restore_preexisting_into_worktree() + return True + + local_branch_exists = ( + git_main("rev-parse", "--verify", "--quiet", SYNC_BRANCH).returncode == 0 + ) + + # Classify the remote state: branch exists (restore it), remote reachable + # but branch absent (fresh orphan), or remote unreachable (retry later). + ls_remote = git_main( + "ls-remote", "--exit-code", "--heads", "origin", SYNC_BRANCH + ) + if ls_remote.returncode == 0: + has_remote_branch = True + elif ls_remote.returncode == _LS_REMOTE_NO_MATCHING_REFS: + has_remote_branch = False + elif local_branch_exists: + # Offline, but a local runtime-sync branch already exists (e.g. a + # prior init whose worktree was removed) -- reattaching to it cannot + # diverge from origin any further than it already has. + has_remote_branch = False + else: + logger.warning( + "origin is unreachable (rc={}): {}; deferring runtime worktree init", + ls_remote.returncode, + ls_remote.stderr.strip(), + ) + return False + + logger.info("Initializing runtime worktree on branch {}", SYNC_BRANCH) + + remote_ref = f"origin/{SYNC_BRANCH}" + if has_remote_branch: + fetch_result = git_main("fetch", "origin", SYNC_BRANCH) + if fetch_result.returncode != 0: + logger.warning( + "git fetch origin {} failed (rc={}): {}; deferring init", + SYNC_BRANCH, + fetch_result.returncode, + fetch_result.stderr.strip(), + ) + return False + + staged_aside = False + if _runtime_dir_has_files(): + logger.info( + "runtime/ already has files; staging them aside before adding the worktree" + ) + _stage_preexisting_aside() + staged_aside = True + + if has_remote_branch: + result = git_main( + "worktree", "add", "-B", SYNC_BRANCH, str(RUNTIME_DIR), remote_ref + ) + elif local_branch_exists: + result = git_main("worktree", "add", str(RUNTIME_DIR), SYNC_BRANCH) + else: + result = _create_orphan_runtime_worktree(SYNC_BRANCH) + + if result.returncode != 0: + logger.error( + "git worktree add failed (rc={}): {}", + result.returncode, + result.stderr.strip(), + ) + # Restore preexisting files so other services don't lose them. + if staged_aside: + if not RUNTIME_DIR.exists(): + shutil.move(str(_RUNTIME_PREEXISTING_DIR), str(RUNTIME_DIR)) + else: + _restore_preexisting_into_worktree() + return False + + # Configure bot identity for sync commits inside this worktree only. + git_runtime("config", "user.name", SYNC_USER_NAME) + git_runtime("config", "user.email", SYNC_USER_EMAIL) + + if has_remote_branch: + # Make sure the local branch tracks the remote (some git versions + # don't set this automatically with -B + an explicit ref). + git_runtime("branch", "--set-upstream-to", remote_ref) + elif not local_branch_exists: + # Fresh orphan branch: write the .gitignore for secrets and make an + # initial commit so push has something to push. runtime/secrets holds + # e.g. the Cloudflare tunnel token, which must never reach the remote. + gitignore = RUNTIME_DIR / ".gitignore" + gitignore.write_text("secrets\n") + git_runtime("add", ".gitignore") + commit = git_runtime("commit", "-m", "runtime sync: init") + if commit.returncode != 0: + logger.error( + "initial commit failed (rc={}): {}", + commit.returncode, + commit.stderr.strip(), + ) + + # Restore staged-aside content. Calling unconditionally (rather than + # gating on the `staged_aside` flag) also recovers content left by a + # prior init that staged aside but was killed before it could restore. + _restore_preexisting_into_worktree() + return True diff --git a/libs/github_sync/src/github_sync/worktree_test.py b/libs/github_sync/src/github_sync/worktree_test.py new file mode 100644 index 000000000..710984ba9 --- /dev/null +++ b/libs/github_sync/src/github_sync/worktree_test.py @@ -0,0 +1,127 @@ +"""Unit tests for runtime/ worktree init and restore-from-origin.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from github_sync.testing import init_repo, init_repo_with_origin, run_git +from github_sync.worktree import init_runtime_worktree, is_runtime_worktree + + +def _git_out(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True, check=False + ).stdout + + +def test_init_creates_orphan_worktree_when_origin_has_no_sync_branch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + + assert init_runtime_worktree() is True + + runtime = main / "runtime" + assert is_runtime_worktree() + # The branch is an orphan: a single parentless root, sharing no history + # with main (whose seed commit must not be reachable). + roots = _git_out(runtime, "rev-list", "--max-parents=0", "HEAD").split() + assert len(roots) == 1 + assert "seed" not in _git_out(runtime, "log", "--format=%s") + assert _git_out(runtime, "branch", "--show-current").strip() == "runtime-sync" + # Secrets are excluded from the sync branch. + assert (runtime / ".gitignore").read_text() == "secrets\n" + + +def test_init_stages_aside_and_restores_preexisting_runtime_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + runtime = main / "runtime" + runtime.mkdir() + (runtime / "initial_chat_created").write_text("") + + assert init_runtime_worktree() is True + + assert (runtime / "initial_chat_created").exists() + assert not (main / "runtime.preexisting").exists() + + +def test_init_restores_runtime_state_from_origin_sync_branch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The recreated-workspace path: origin already has runtime-sync, so init + materializes the prior runtime/ state instead of starting fresh.""" + # A first workspace creates and pushes runtime state. + first_base = tmp_path / "first" + first_base.mkdir() + first_main, origin = init_repo_with_origin(first_base) + monkeypatch.chdir(first_main) + assert init_runtime_worktree() is True + (first_main / "runtime" / "memory.md").write_text("remember me\n") + run_git(first_main / "runtime", "add", "-A") + run_git(first_main / "runtime", "commit", "-qm", "state") + run_git(first_main / "runtime", "push", "--set-upstream", "origin", "runtime-sync") + + # A second workspace pointing at the same origin restores that state. + second_main = tmp_path / "second" + init_repo(second_main) + (second_main / "seed.txt").write_text("seed\n") + run_git(second_main, "add", "-A") + run_git(second_main, "commit", "-qm", "seed") + run_git(second_main, "remote", "add", "origin", str(origin)) + # Fresh-boot files already in runtime/ must survive the restore. + runtime = second_main / "runtime" + runtime.mkdir() + (runtime / "initial_chat_created").write_text("") + monkeypatch.chdir(second_main) + + assert init_runtime_worktree() is True + + assert (runtime / "memory.md").read_text() == "remember me\n" + assert (runtime / "initial_chat_created").exists() + upstream = _git_out( + runtime, "rev-parse", "--abbrev-ref", "runtime-sync@{upstream}" + ).strip() + assert upstream == "origin/runtime-sync" + + +def test_init_defers_when_origin_unreachable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With origin unreachable we cannot know whether a remote runtime-sync + branch exists, so init must NOT create a fresh orphan branch (it could + permanently diverge from a restorable remote); it defers instead.""" + main = tmp_path / "main" + init_repo(main) + (main / "seed.txt").write_text("seed\n") + run_git(main, "add", "-A") + run_git(main, "commit", "-qm", "seed") + run_git(main, "remote", "add", "origin", str(tmp_path / "does-not-exist.git")) + runtime = main / "runtime" + runtime.mkdir() + (runtime / "precious.txt").write_text("keep\n") + monkeypatch.chdir(main) + + assert init_runtime_worktree() is False + + assert not is_runtime_worktree() + assert (runtime / "precious.txt").read_text() == "keep\n" + + +def test_init_noops_when_already_a_worktree( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + assert init_runtime_worktree() is True + head_before = _git_out(main / "runtime", "rev-parse", "HEAD") + + assert init_runtime_worktree() is True + + assert _git_out(main / "runtime", "rev-parse", "HEAD") == head_before diff --git a/libs/runtime_backup/test_runtime_backup_ratchets.py b/libs/github_sync/test_github_sync_ratchets.py similarity index 100% rename from libs/runtime_backup/test_runtime_backup_ratchets.py rename to libs/github_sync/test_github_sync_ratchets.py diff --git a/libs/host_backup/README.md b/libs/host_backup/README.md index 282624d28..7246de6f1 100644 --- a/libs/host_backup/README.md +++ b/libs/host_backup/README.md @@ -3,8 +3,8 @@ Background service that continuously backs up the agent's full `host_dir` (`/mngr/`) to a remote restic repository (Cloudflare R2 by default). -Distinct from `runtime_backup`, which only ships `runtime/` to a GitHub -orphan branch as a fine-grained checkpoint. `host_backup` covers the whole +Distinct from the opt-in `github_sync` service, which only ships `runtime/` +to a GitHub orphan branch as a fine-grained checkpoint. `host_backup` covers the whole host_dir (code, worktrees, agent state, chat sessions, logs) and pushes to an encrypted restic repo on cheaper object storage. @@ -31,7 +31,8 @@ an encrypted restic repo on cheaper object storage. `AWS_SECRET_ACCESS_KEY` for an S3/R2 backend). Written only by the minds app (injected whole); a missing file means backups are not configured. `restic.env` is gitignored (rides nothing). `backup.toml` is *not* - gitignored so it survives container loss via runtime-backup. + gitignored, so it also rides the opt-in GitHub sync of runtime/ when + that is enabled. - A tick only runs once both `RESTIC_REPOSITORY` and `RESTIC_PASSWORD` are set in `restic.env`. Backend credentials are not gated by host_backup -- restic reports its own error if the chosen backend needs one that is @@ -58,8 +59,8 @@ an encrypted restic repo on cheaper object storage. M --keep-weekly W --keep-monthly O` runs (cheap, index-only). At most once per `prune_interval_hours` (default 24) we additionally run `restic prune` (the slow data deletion step); gated by - `runtime/last-restic-prune` (a timestamp file that rides runtime-backup - so it survives container loss). + `runtime/last-restic-prune` (a timestamp file under runtime/, covered by + the opt-in GitHub sync when enabled). - The outer loop never exits. Every exception is logged with full traceback to loguru and as a `tick_error` event in the jsonl stream; the loop continues to the next tick. diff --git a/libs/host_backup/src/host_backup/config.py b/libs/host_backup/src/host_backup/config.py index 109a6877f..7aed4926c 100644 --- a/libs/host_backup/src/host_backup/config.py +++ b/libs/host_backup/src/host_backup/config.py @@ -8,7 +8,7 @@ `[snapshot]` section old bootstraps keep writing forever) and malformed values produce log warnings and fall back to defaults -- they never crash the service and never block the remaining valid settings from applying. - Rides the runtime-backup git push. + Rides the opt-in GitHub sync of runtime/ when that is enabled. - `runtime/secrets/restic.env` -- restic's repository address + all secrets (`RESTIC_REPOSITORY`, `RESTIC_PASSWORD`, and any backend credentials restic reads from the environment, e.g. `AWS_ACCESS_KEY_ID` / diff --git a/libs/runtime_backup/README.md b/libs/runtime_backup/README.md deleted file mode 100644 index 1db7c063b..000000000 --- a/libs/runtime_backup/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# runtime_backup - -Background service that periodically commits and pushes the contents of -`runtime/` (which holds Claude memory, ticket state, transcripts, app port -registry, etc.) to a per-agent orphan branch named -`mindsbackup/$MNGR_AGENT_ID` on the same `origin` as the main checkout. - -The branch and the worktree at `runtime/` itself are created by -`libs/bootstrap` during its pre-services init step. This service assumes -that has already happened and just polls. - -## Behavior - -- Tick interval: 60 seconds. -- Each tick: `git add -A`, commit (only if dirty) with message - `runtime backup: `, push (only when `GH_TOKEN` - is set in env). All `git` operations target the runtime worktree. -- Push first tries default args (no `--force`). Bootstrap normally sets - the upstream during init, but if that best-effort push failed, the - next tick's plain `git push` fails with "no upstream" -- so on push - failure the runner retries once with `git push --set-upstream origin - ` to self-heal that case (mirroring the post-commit hook). - `--force` is never used; the per-agent branch model means there is - only ever one writer, so non-fast-forwards should not happen. -- `runtime/secrets` is excluded via the worktree's own `.gitignore` - (written by bootstrap during init), so the Cloudflare tunnel token - never reaches the remote. -- Each tick first clears a stale `index.lock` from the runtime worktree if - one is present. This service is the worktree's only git writer and its - ticks are sequential, so a lock here was normally left by a prior tick's - git process that was killed before releasing it (if something kills the - process mid-commit). Git never clears such a lock itself, so without this - every later `git add` would fail identically and backups would stop - permanently and silently. As a safeguard against the unlikely case of a - concurrent writer, the lock is only removed once it is older than the tick - interval -- a live git operation on the small `runtime/` tree holds the - lock far more briefly than that, so an in-progress operation is never - disturbed. -- On any git failure the service logs to stderr and `/tmp/runtime-backup.log` - and tries again on the next tick. It does not exit, so bootstrap's - `restart = "on-failure"` policy is only triggered on a hard crash. -- Without `GH_TOKEN` the service still commits locally; pushes are - skipped until a token appears (typically on container restart). - -## Restoring on a fresh container - -If `mindsbackup/$MNGR_AGENT_ID` already exists on origin (e.g. the same -agent is recreated), bootstrap fetches and materializes it into `runtime/` -on first boot, so prior memory and runtime state come back automatically. - -Migration to a *different* `MNGR_AGENT_ID` is intentionally manual. diff --git a/libs/runtime_backup/pyproject.toml b/libs/runtime_backup/pyproject.toml deleted file mode 100644 index 0cf347675..000000000 --- a/libs/runtime_backup/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[project] -name = "runtime-backup" -version = "0.1.0" -description = "Service that periodically commits + pushes runtime/ to a per-agent orphan branch (mindsbackup/$MNGR_AGENT_ID)" -readme = "README.md" -requires-python = ">=3.11" -dependencies = [ - "loguru>=0.7.0", -] - -[project.scripts] -runtime-backup = "runtime_backup.runner:main" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/runtime_backup"] diff --git a/libs/runtime_backup/src/runtime_backup/runner.py b/libs/runtime_backup/src/runtime_backup/runner.py deleted file mode 100644 index a145ecd2e..000000000 --- a/libs/runtime_backup/src/runtime_backup/runner.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Runtime backup service. - -Polls runtime/ every TICK_INTERVAL_SECONDS; if there are uncommitted changes, -makes a backup commit on the orphan branch checked out at runtime/ and (when -GH_TOKEN is set) pushes it to origin. - -The orphan branch (mindsbackup/$MNGR_AGENT_ID) and the worktree at runtime/ -are created by libs/bootstrap during its pre-services init step, so this -service can assume runtime/ is already a git worktree on that branch. -""" - -import os -import subprocess -import time -from datetime import datetime, timezone -from pathlib import Path - -from loguru import logger - -RUNTIME_DIR = Path("runtime") -TICK_INTERVAL_SECONDS = 60 -LOG_FILE = Path("/tmp/runtime-backup.log") - -# Minimum age before an index.lock is treated as stale and removed. A real git -# operation on the small runtime/ tree holds the lock for well under a second, -# so a lock older than this cannot belong to a live operation. Set to the tick -# interval so a lock skipped as possibly-live on one tick is guaranteed old -# enough to clear on the next. -STALE_LOCK_MIN_AGE_SECONDS = TICK_INTERVAL_SECONDS - - -def _git_noninteractive_env() -> dict[str, str]: - """Environment for backup git calls: never prompt for credentials. - - GIT_TERMINAL_PROMPT=0 turns any credential prompt (e.g. a push to a - private origin whose token went stale) into a fast failure instead of a - TTY prompt that would wedge the backup loop forever; every caller already - logs-and-continues on a nonzero exit. Mirrors the helper of the same name - in bootstrap.manager (kept local so this lib stays dependency-free). - """ - return {**os.environ, "GIT_TERMINAL_PROMPT": "0"} - - -def _git(*args: str) -> subprocess.CompletedProcess[str]: - """Run a git command inside the runtime worktree, never raising or prompting.""" - return subprocess.run( - ["git", "-C", str(RUNTIME_DIR), *args], - capture_output=True, - text=True, - check=False, - env=_git_noninteractive_env(), - ) - - -def _clear_stale_index_lock() -> None: - """Remove a stale ``index.lock`` from the runtime worktree, if present. - - runtime-backup is the only writer of this worktree's git index and its - ticks run strictly sequentially, so a lock here is normally stale -- left - behind by a previous tick's git process that was killed before it could - release the lock (whenever something kills the process mid-commit). Git - never clears such a lock itself, so without this every subsequent - ``git add`` fails identically and backups stop forever. - - The removal is deliberately *not* unconditional: to stay safe even if the - single-writer assumption is ever violated (a concurrent git process in the - worktree), the lock is only removed once it is older than - ``STALE_LOCK_MIN_AGE_SECONDS``. A live git operation on the small runtime/ - tree holds the lock for well under a second, so an older lock cannot be - live; a genuinely in-progress operation is left untouched and reconsidered - on the next tick. - - The lock path is resolved via ``--absolute-git-dir`` so it is correct - whether runtime/ is a normal repo or (as in production) a linked worktree - with a per-worktree git dir. - """ - git_dir_result = _git("rev-parse", "--absolute-git-dir") - if git_dir_result.returncode != 0: - # runtime/ is not a git repo yet; nothing to clear. - return - lock_path = Path(git_dir_result.stdout.strip()) / "index.lock" - try: - lock_age_seconds = time.time() - lock_path.stat().st_mtime - except OSError: - # No lock present (the common case), or it vanished underneath us. - return - if lock_age_seconds < STALE_LOCK_MIN_AGE_SECONDS: - logger.warning( - "git index.lock at {} is only {:.0f}s old; leaving it in case a " - "git operation is in progress (will reconsider next tick)", - lock_path, - lock_age_seconds, - ) - return - logger.warning( - "Removing stale git index lock at {} ({:.0f}s old)", - lock_path, - lock_age_seconds, - ) - try: - lock_path.unlink() - except OSError as e: - logger.warning("Failed to remove stale index lock at {}: {}", lock_path, e) - - -def _has_uncommitted_changes() -> bool: - """Return True if runtime/ has anything to commit.""" - result = _git("status", "--porcelain") - if result.returncode != 0: - logger.warning( - "git status failed (rc={}): {}", result.returncode, result.stderr.strip() - ) - return False - return bool(result.stdout.strip()) - - -def _now_iso_utc() -> str: - """Current UTC time as ISO-8601 with a trailing Z (e.g. 2026-05-06T17:42:13Z).""" - return ( - datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") - ) - - -def _do_tick(should_push: bool) -> None: - """Run one backup tick: add, commit-if-dirty, push-if-token.""" - # Clear any stale index.lock first, so a commit interrupted by something - # killing the process cannot wedge every future tick. - _clear_stale_index_lock() - - add_result = _git("add", "-A") - if add_result.returncode != 0: - logger.warning( - "git add failed (rc={}): {}", - add_result.returncode, - add_result.stderr.strip(), - ) - return - - if _has_uncommitted_changes(): - commit_result = _git("commit", "-m", f"runtime backup: {_now_iso_utc()}") - if commit_result.returncode != 0: - logger.warning( - "git commit failed (rc={}): {}", - commit_result.returncode, - commit_result.stderr.strip(), - ) - # Fall through to push: any prior unpushed commits should still - # be shipped even if this tick's commit failed. - - if should_push: - # Always attempt push: covers the case where a prior tick committed but - # failed to push, so the next tick still ships the unpushed commit. - # Bootstrap's initial `--set-upstream` push is best-effort; if it - # failed, plain `git push` will fail forever with "no upstream". Fall - # back to `--set-upstream origin ` to self-heal that case - # (mirrors the post-commit hook's chain). - push_result = _git("push") - if push_result.returncode != 0: - branch_result = _git("symbolic-ref", "--short", "HEAD") - branch = branch_result.stdout.strip() - if branch_result.returncode == 0 and branch: - push_result = _git("push", "--set-upstream", "origin", branch) - if push_result.returncode != 0: - logger.warning( - "git push failed (rc={}): {}", - push_result.returncode, - push_result.stderr.strip(), - ) - - -def main() -> None: - """Main loop: poll runtime/ on a fixed interval and back up changes.""" - # Tee stderr-bound logs into LOG_FILE so operators can `tail` the file - # across restarts of just this service window. /tmp wipes on container - # restart, which is the intended scope for the debug log. Set up here - # rather than at module import so that merely importing this module - # (e.g. from tests) does not start writing to the log file. - logger.add(LOG_FILE, level="INFO") - - logger.info("Starting runtime-backup (interval={}s)", TICK_INTERVAL_SECONDS) - - if not (RUNTIME_DIR / ".git").exists(): - logger.warning( - "runtime/ is not a git worktree; bootstrap should have created it" - ) - - has_token = bool(os.environ.get("GH_TOKEN")) - if not has_token: - logger.info("No GH_TOKEN; will commit locally but skip push") - - while True: - _do_tick(should_push=has_token) - time.sleep(TICK_INTERVAL_SECONDS) - - -if __name__ == "__main__": - main() diff --git a/libs/runtime_backup/src/runtime_backup/runner_test.py b/libs/runtime_backup/src/runtime_backup/runner_test.py deleted file mode 100644 index 1286b4ce6..000000000 --- a/libs/runtime_backup/src/runtime_backup/runner_test.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Unit tests for the runtime-backup runner. - -Focus: the stale-index-lock recovery that keeps an interrupted commit from -permanently wedging every future backup tick, and the age guard that keeps it -from disturbing a genuinely in-progress git operation. -""" - -from __future__ import annotations - -import os -import subprocess -import time -from pathlib import Path - -import pytest - -from runtime_backup.runner import ( - STALE_LOCK_MIN_AGE_SECONDS, - _clear_stale_index_lock, - _do_tick, -) - - -def _git(repo: Path, *args: str) -> None: - """Run a git command in `repo`, raising on failure.""" - subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True) - - -def _init_repo(repo: Path) -> None: - """Create a git repo at `repo` with a committer identity configured.""" - repo.mkdir(parents=True, exist_ok=True) - subprocess.run(["git", "init", "-q", str(repo)], check=True, capture_output=True) - _git(repo, "config", "user.email", "test@test.local") - _git(repo, "config", "user.name", "test") - - -def _age_lock(lock_path: Path) -> None: - """Backdate a lock file's mtime so it counts as stale (past the age guard).""" - old = time.time() - STALE_LOCK_MIN_AGE_SECONDS - 60 - os.utime(lock_path, (old, old)) - - -def test_clear_stale_index_lock_removes_aged_lock_in_linked_worktree( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Faithful to production: runtime/ is a linked worktree, so the lock - lives in the per-worktree git dir, not in a top-level .git/.""" - monkeypatch.chdir(tmp_path) - main = tmp_path / "main" - _init_repo(main) - (main / "seed.txt").write_text("seed\n") - _git(main, "add", "-A") - _git(main, "commit", "-qm", "seed") - # The runner resolves runtime/ relative to its cwd, so the worktree must - # be named exactly "runtime" and sit in tmp_path. - _git(main, "worktree", "add", str(tmp_path / "runtime"), "-b", "backup") - - lock_path = main / ".git" / "worktrees" / "runtime" / "index.lock" - lock_path.write_text("") - _age_lock(lock_path) - assert lock_path.exists() - - _clear_stale_index_lock() - - assert not lock_path.exists() - - -def test_clear_stale_index_lock_keeps_fresh_lock( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A recently-created lock might belong to a live git operation, so it - must be left alone rather than yanked out from under that operation.""" - monkeypatch.chdir(tmp_path) - runtime = tmp_path / "runtime" - _init_repo(runtime) - lock_path = runtime / ".git" / "index.lock" - lock_path.write_text("") # fresh: mtime is now - - _clear_stale_index_lock() - - assert lock_path.exists() - - -def test_clear_stale_index_lock_noop_when_no_lock_present( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - _init_repo(tmp_path / "runtime") - # No lock and no git repo problems -- must simply not raise. - _clear_stale_index_lock() - - -def test_clear_stale_index_lock_noop_when_runtime_not_a_repo( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - (tmp_path / "runtime").mkdir() - # `git rev-parse` fails; the function must silently return. - _clear_stale_index_lock() - - -def test_do_tick_self_heals_stale_index_lock( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A stale index.lock from a killed prior tick must not wedge backups: - the next tick clears it and commits the pending runtime state.""" - monkeypatch.chdir(tmp_path) - runtime = tmp_path / "runtime" - _init_repo(runtime) - # A leftover lock from a prior tick's git process that was killed; aged - # past the guard so it is recognized as stale. - lock_path = runtime / ".git" / "index.lock" - lock_path.write_text("") - _age_lock(lock_path) - # New runtime state waiting to be backed up. - (runtime / "memory.txt").write_text("important state\n") - - _do_tick(should_push=False) - - log = subprocess.run( - ["git", "-C", str(runtime), "log", "--oneline"], - check=True, - capture_output=True, - text=True, - ) - assert "runtime backup:" in log.stdout diff --git a/pyproject.toml b/pyproject.toml index c63e4b581..2ce47d968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "cloudflare-tunnel", "app-watcher", "web-server", - "runtime-backup", + "github-sync", "host-backup", "tk-command-parsing", "imbue-common", @@ -55,14 +55,14 @@ package = false override-dependencies = ["openai>=2.20.0"] [tool.uv.workspace] -members = ["libs/bootstrap", "libs/cloudflare_tunnel", "libs/app_watcher", "libs/web_server", "libs/runtime_backup", "libs/host_backup", "libs/mngr_cli_contract", "libs/tk_command_parsing", "apps/system_interface", "libs/browser"] +members = ["libs/bootstrap", "libs/cloudflare_tunnel", "libs/app_watcher", "libs/web_server", "libs/github_sync", "libs/host_backup", "libs/mngr_cli_contract", "libs/tk_command_parsing", "apps/system_interface", "libs/browser"] [tool.uv.sources] bootstrap = { workspace = true } cloudflare-tunnel = { workspace = true } app-watcher = { workspace = true } web-server = { workspace = true } -runtime-backup = { workspace = true } +github-sync = { workspace = true } host-backup = { workspace = true } mngr-cli-contract = { workspace = true } tk-command-parsing = { workspace = true } diff --git a/scripts/claude_open_tickets_reminder.sh b/scripts/claude_open_tickets_reminder.sh index bcb3e38e1..734ee2633 100755 --- a/scripts/claude_open_tickets_reminder.sh +++ b/scripts/claude_open_tickets_reminder.sh @@ -14,7 +14,7 @@ set -euo pipefail repo_root="${MNGR_AGENT_WORK_DIR:-$(pwd)}" # Honor any externally-set TICKETS_DIR (the agent's env normally pins it # via .mngr/settings.toml -- e.g. /code/runtime/tickets -- so the tk -# tickets live alongside the runtime-backup branch). Fall back to tk's +# tickets live under runtime/ with the rest of the synced state). Fall back to tk's # unset-default of /.tickets when nothing is set. tickets_dir="${TICKETS_DIR:-${repo_root}/.tickets}" diff --git a/scripts/claude_open_tickets_stop_nudge.sh b/scripts/claude_open_tickets_stop_nudge.sh index 63075d5d0..0cbcd3ca3 100755 --- a/scripts/claude_open_tickets_stop_nudge.sh +++ b/scripts/claude_open_tickets_stop_nudge.sh @@ -9,7 +9,7 @@ set -euo pipefail repo_root="${MNGR_AGENT_WORK_DIR:-$(pwd)}" # Honor any externally-set TICKETS_DIR (the agent's env normally pins it # via .mngr/settings.toml -- e.g. /code/runtime/tickets -- so the tk -# tickets live alongside the runtime-backup branch). Fall back to tk's +# tickets live under runtime/ with the rest of the synced state). Fall back to tk's # unset-default of /.tickets when nothing is set. tickets_dir="${TICKETS_DIR:-${repo_root}/.tickets}" diff --git a/scripts/git_hooks/post-commit b/scripts/git_hooks/post-commit index 324a9c01d..dd7a47b2b 100755 --- a/scripts/git_hooks/post-commit +++ b/scripts/git_hooks/post-commit @@ -1,16 +1,24 @@ #!/usr/bin/env bash -# Auto-push the active branch after every commit, gated on GH_TOKEN being set. +# Auto-push the active branch after every commit, part of the opt-in GitHub +# sync (see libs/github_sync/README.md and the github-sync skill). # -# Installed for every checkout in the container via: -# git config --global core.hooksPath /mngr/code/scripts/git_hooks -# (set up by the git_auth_setup extra_window in .mngr/settings.toml). +# This hook is inert by default: it only takes effect once the github-sync +# skill points `git config --global core.hooksPath` at this directory (done +# by `uv run github-sync wire-git`). The same wiring makes plain `git push` +# authenticate through the latchkey gateway, so this hook needs no +# credentials of its own -- no token ever enters the container. # -# Skips silently in three cases that are normal, not errors: -# 1. GH_TOKEN is unset (e.g. local dev, or worker sub-agents that should -# never push). +# Because core.hooksPath is global, the hook fires for every checkout in the +# container: the main /mngr/code repo and every worker worktree, each pushing +# its own current branch. +# +# Skips silently in these cases that are normal, not errors: +# 1. GitHub sync is not configured (github_sync.toml absent). # 2. HEAD is detached (no branch to push to). -# 3. The current branch is under mindsbackup/ -- those branches belong to -# the runtime-backup polling service, which handles its own pushes. +# 3. The current branch is runtime-sync -- that branch belongs to the +# github-sync polling service, which handles its own pushes. +# 4. The github-sync service has halted pushes because the sync repo is +# public or its visibility could not be confirmed. # # Runs the push in the background so the commit never blocks. All output # (success or failure) is appended to /tmp/post-commit-push.log for debugging; @@ -18,10 +26,12 @@ set -u +CONFIG_FILE=/mngr/code/github_sync.toml +STATUS_FILE=/tmp/github-sync-status.json LOG_FILE=/tmp/post-commit-push.log -# 1. No token, nothing to do. -if [ -z "${GH_TOKEN:-}" ]; then +# 1. Sync not configured, nothing to do. +if [ ! -f "$CONFIG_FILE" ]; then exit 0 fi @@ -31,17 +41,36 @@ if [ -z "$branch" ]; then exit 0 fi -# 3. Runtime backup branches push themselves via the runtime-backup service. +# 3. The runtime-sync branch pushes itself via the github-sync service. case "$branch" in - mindsbackup/*) exit 0 ;; + runtime-sync) exit 0 ;; esac +# 4. Respect a visibility halt: the service refuses to push while the sync +# repo is public (or unconfirmed), and so must we. +if [ -f "$STATUS_FILE" ] && command -v jq >/dev/null 2>&1; then + allowed="$(jq -r '.is_push_allowed // true' "$STATUS_FILE" 2>/dev/null)" + if [ "$allowed" = "false" ]; then + echo "[$(date -Is)] skipping push of $branch: sync repo not confirmed private" >> "$LOG_FILE" + exit 0 + fi +fi + # Background push, never blocking the commit. First try a plain push; if the -# branch has no upstream yet, fall back to setting upstream as we push. +# branch has no upstream yet, fall back to setting upstream as we push. If the +# primary latchkey gateway is unreachable and a secondary (per-VPS) gateway +# exists, push the proxied URL directly through it (password header only -- +# the secondary takes no permissions override). { echo "[$(date -Is)] post-commit push for $branch" if ! git push 2>&1; then - git push --set-upstream origin "$branch" 2>&1 + if ! git push --set-upstream origin "$branch" 2>&1; then + repo_url="$(sed -n 's/^repo_url *= *"\(.*\)".*/\1/p' "$CONFIG_FILE")" + if [ -n "${LATCHKEY_GATEWAY_SECONDARY:-}" ] && [ -n "${LATCHKEY_GATEWAY_PASSWORD:-}" ] && [ -n "$repo_url" ]; then + git -c "http.extraHeader=X-Latchkey-Gateway-Password: $LATCHKEY_GATEWAY_PASSWORD" \ + push "${LATCHKEY_GATEWAY_SECONDARY%/}/gateway/$repo_url.git" "$branch" 2>&1 + fi + fi fi } >> "$LOG_FILE" 2>&1 & diff --git a/supervisord.conf b/supervisord.conf index f8a6f9ed3..e814507d5 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -6,8 +6,8 @@ # # Conventions: # - Services inherit the agent environment (MNGR_AGENT_STATE_DIR, -# CLAUDE_CONFIG_DIR, MNGR_HOST_DIR, MNGR_AGENT_ID, GH_TOKEN, ...) from the -# bootstrap shell that exec'd supervisord -- there is no per-service +# CLAUDE_CONFIG_DIR, MNGR_HOST_DIR, MNGR_AGENT_ID, LATCHKEY_*, ...) from +# the bootstrap shell that exec'd supervisord -- there is no per-service # `environment=` enumeration. # - All commands run from the repo root (directory=/mngr/code). # - Commands that chain with `&&` (or set inline env) are wrapped in @@ -108,24 +108,10 @@ stderr_logfile_maxbytes=10MB stdout_logfile_backups=3 stderr_logfile_backups=3 -# Periodically commits + pushes runtime/ to mindsbackup/$MNGR_AGENT_ID so that -# Claude memory, ticket state, transcripts, etc. survive container loss. The -# runtime/ worktree is created by bootstrap before supervisord starts, so this -# service always finds it in place. See libs/runtime_backup/README.md. -[program:runtime-backup] -command=uv run runtime-backup -directory=/mngr/code -autostart=true -autorestart=true -startretries=1000000 -stopasgroup=true -killasgroup=true -stdout_logfile=/var/log/supervisor/runtime-backup-stdout.log -stderr_logfile=/var/log/supervisor/runtime-backup-stderr.log -stdout_logfile_maxbytes=10MB -stderr_logfile_maxbytes=10MB -stdout_logfile_backups=3 -stderr_logfile_backups=3 +# NOTE: GitHub sync ([program:github-sync]) is opt-in and deliberately NOT in +# this default config. The github-sync skill appends its program block (and +# sets up the private repo, git wiring, and runtime/ worktree) when the user +# asks for it. See libs/github_sync/README.md. # Continuous restic-based backup of the full host_dir (/mngr/) to a remote # repository. runtime/secrets/restic.env is injected by the minds app; diff --git a/uv.lock b/uv.lock index 4bb33df13..b9f134622 100644 --- a/uv.lock +++ b/uv.lock @@ -29,9 +29,9 @@ members = [ "browser", "cloudflare-tunnel", "default-workspace-template", + "github-sync", "host-backup", "mngr-cli-contract", - "runtime-backup", "system-interface", "tk-command-parsing", "web-server", @@ -942,11 +942,11 @@ dependencies = [ { name = "bootstrap" }, { name = "browser" }, { name = "cloudflare-tunnel" }, + { name = "github-sync" }, { name = "host-backup" }, { name = "imbue-common" }, { name = "litellm" }, { name = "playwright" }, - { name = "runtime-backup" }, { name = "tk-command-parsing" }, { name = "tomlkit" }, { name = "web-server" }, @@ -973,11 +973,11 @@ requires-dist = [ { name = "bootstrap", editable = "libs/bootstrap" }, { name = "browser", editable = "libs/browser" }, { name = "cloudflare-tunnel", editable = "libs/cloudflare_tunnel" }, + { name = "github-sync", editable = "libs/github_sync" }, { name = "host-backup", editable = "libs/host_backup" }, { name = "imbue-common", editable = "vendor/mngr/libs/imbue_common" }, { name = "litellm", specifier = ">=1.88.1" }, { name = "playwright", specifier = "==1.58.0" }, - { name = "runtime-backup", editable = "libs/runtime_backup" }, { name = "tk-command-parsing", editable = "libs/tk_command_parsing" }, { name = "tomlkit", specifier = ">=0.12" }, { name = "web-server", editable = "libs/web_server" }, @@ -1264,6 +1264,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/b9/7dd37b6001d16f692b1bfb6e68cad642beb38b34a753c29bbff312f46e4b/gevent-26.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:d1c08bc9bb6bd79732a26710a99588b5e9b67b668e165dd609704b876f41baab", size = 1703189, upload-time = "2026-04-08T22:48:31.713Z" }, ] +[[package]] +name = "github-sync" +version = "0.1.0" +source = { editable = "libs/github_sync" } +dependencies = [ + { name = "click" }, + { name = "loguru" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.0" }, + { name = "loguru", specifier = ">=0.7.0" }, +] + [[package]] name = "google-api-core" version = "2.29.0" @@ -6375,17 +6390,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/7e/61c42657f6e4614a4258f1c3b0c5b93adc4d1f8575f5229d1906b483099b/ruff-0.12.12-py3-none-win_arm64.whl", hash = "sha256:2a8199cab4ce4d72d158319b63370abf60991495fb733db96cd923a34c52d093", size = 12256762, upload-time = "2025-09-04T16:50:15.737Z" }, ] -[[package]] -name = "runtime-backup" -version = "0.1.0" -source = { editable = "libs/runtime_backup" } -dependencies = [ - { name = "loguru" }, -] - -[package.metadata] -requires-dist = [{ name = "loguru", specifier = ">=0.7.0" }] - [[package]] name = "s3transfer" version = "0.19.1" From ba87186a01020ad2b5c632911fbd0bf7cfd4b40e Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:34:03 -0700 Subject: [PATCH 02/24] Fix post-commit hook visibility halt defeated by jq's // operator Problem: scripts/git_hooks/post-commit read the halt flag with jq -r '.is_push_allowed // true'. jq's // operator falls through to its right side when the left side is false OR null, so an explicit "is_push_allowed": false evaluated to true and the hook kept auto-pushing branches while the github-sync service had halted pushes (sync repo public or unverifiable), defeating the private-only enforcement. Fix: read the field with jq -r '.is_push_allowed' and compare against "false" (missing file/field reads as null/empty, preserving the intended default-allow before the service's first report). Made the hook's config/status/log paths env-overridable (production defaults unchanged; GITHUB_SYNC_STATUS_FILE matches the override the runner already honors) and added hook tests in libs/github_sync, including a regression test proven to fail against the old jq expression. The github_sync time.sleep ratchet count is bumped 1 -> 2 for the bounded condition-poll awaiting the hook's intentionally-backgrounded push. Co-Authored-By: Claude Fable 5 --- .../src/github_sync/post_commit_hook_test.py | 128 ++++++++++++++++++ libs/github_sync/test_github_sync_ratchets.py | 5 +- scripts/git_hooks/post-commit | 17 ++- 3 files changed, 144 insertions(+), 6 deletions(-) create mode 100644 libs/github_sync/src/github_sync/post_commit_hook_test.py diff --git a/libs/github_sync/src/github_sync/post_commit_hook_test.py b/libs/github_sync/src/github_sync/post_commit_hook_test.py new file mode 100644 index 000000000..b8cff3c74 --- /dev/null +++ b/libs/github_sync/src/github_sync/post_commit_hook_test.py @@ -0,0 +1,128 @@ +"""Tests for the scripts/git_hooks/post-commit auto-push hook. + +The hook is the third piece of the opt-in GitHub sync (see the README): it +auto-pushes the committed branch of any checkout, but only when sync is +configured, and never while the github-sync service has halted pushes because +the sync repo is not confirmed private. The halt is the security-critical +behavior, so it gets a direct regression test (an earlier version used +jq's `//` operator, which turned an explicit ``"is_push_allowed": false`` +into ``true`` and silently disabled the halt). +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import time +from pathlib import Path + +import pytest + +from github_sync.testing import init_repo_with_origin + +_HOOK_DIR = Path(__file__).parents[4] / "scripts" / "git_hooks" + +_PUSH_WAIT_SECONDS = 20.0 + + +def _commit_with_hook( + repo: Path, tmp_path: Path, is_push_allowed: bool | None, is_configured: bool = True +) -> Path: + """Make a hook-firing commit in `repo`; returns the hook log path. + + `is_push_allowed=None` means no status file (service has not reported yet). + """ + config_path = tmp_path / "github_sync.toml" + if is_configured: + config_path.write_text( + 'repo_url = "https://github.com/some-user/my-workspace"\n' + ) + status_path = tmp_path / "github-sync-status.json" + if is_push_allowed is not None: + status_path.write_text(json.dumps({"is_push_allowed": is_push_allowed})) + log_path = tmp_path / "post-commit-push.log" + + subprocess.run( + ["git", "-C", str(repo), "config", "core.hooksPath", str(_HOOK_DIR)], + check=True, + capture_output=True, + ) + (repo / "change.txt").write_text("change\n") + subprocess.run( + ["git", "-C", str(repo), "add", "-A"], check=True, capture_output=True + ) + hook_env = { + **os.environ, + # Never let a push attempt block on a TTY credential prompt. + "GIT_TERMINAL_PROMPT": "0", + "GITHUB_SYNC_CONFIG_FILE": str(config_path), + "GITHUB_SYNC_STATUS_FILE": str(status_path), + "GITHUB_SYNC_HOOK_LOG_FILE": str(log_path), + } + subprocess.run( + ["git", "-C", str(repo), "commit", "-qm", "change"], + check=True, + capture_output=True, + env=hook_env, + ) + return log_path + + +def _origin_has_branch(origin: Path, branch: str) -> bool: + result = subprocess.run( + ["git", "-C", str(origin), "rev-parse", "--verify", branch], + capture_output=True, + text=True, + check=False, + ) + return result.returncode == 0 + + +def test_hook_pushes_committed_branch_when_push_allowed( + tmp_path: Path, isolated_git_and_gateway_env: Path +) -> None: + """With sync configured and the repo confirmed private, a commit's branch + lands on origin (the push runs in the background, so poll for it).""" + main, origin = init_repo_with_origin(tmp_path) + + _commit_with_hook(main, tmp_path, is_push_allowed=True) + + deadline = time.monotonic() + _PUSH_WAIT_SECONDS + while not _origin_has_branch(origin, "main"): + assert time.monotonic() < deadline, "hook never pushed the branch" + time.sleep(0.1) + + +def test_hook_skips_push_during_visibility_halt( + tmp_path: Path, isolated_git_and_gateway_env: Path +) -> None: + """Regression: an explicit `"is_push_allowed": false` must block the push + (jq's `//` operator once turned it into `true`). The halt path is fully + synchronous -- no background push job is spawned -- so asserting right + after the commit is race-free.""" + if shutil.which("jq") is None: + pytest.skip("jq is not installed; the hook's halt check requires it") + main, origin = init_repo_with_origin(tmp_path) + + log_path = _commit_with_hook(main, tmp_path, is_push_allowed=False) + + assert not _origin_has_branch(origin, "main") + assert "skipping push of main: sync repo not confirmed private" in ( + log_path.read_text() + ) + + +def test_hook_does_nothing_when_sync_not_configured( + tmp_path: Path, isolated_git_and_gateway_env: Path +) -> None: + """Without github_sync.toml the hook must exit before doing anything.""" + main, origin = init_repo_with_origin(tmp_path) + + log_path = _commit_with_hook( + main, tmp_path, is_push_allowed=None, is_configured=False + ) + + assert not _origin_has_branch(origin, "main") + assert not log_path.exists() diff --git a/libs/github_sync/test_github_sync_ratchets.py b/libs/github_sync/test_github_sync_ratchets.py index 30889dc13..dcf119a5a 100644 --- a/libs/github_sync/test_github_sync_ratchets.py +++ b/libs/github_sync/test_github_sync_ratchets.py @@ -26,7 +26,10 @@ def test_prevent_while_true() -> None: def test_prevent_time_sleep() -> None: - rc.check_time_sleep(_DIR, snapshot(1)) + # 1 = the service loop's tick interval; 1 = the bounded condition-poll in + # post_commit_hook_test.py waiting for the hook's intentionally-backgrounded + # push (the polling pattern the rule_description itself recommends). + rc.check_time_sleep(_DIR, snapshot(2)) def test_prevent_global_keyword() -> None: diff --git a/scripts/git_hooks/post-commit b/scripts/git_hooks/post-commit index dd7a47b2b..a0c8d19e0 100755 --- a/scripts/git_hooks/post-commit +++ b/scripts/git_hooks/post-commit @@ -26,9 +26,12 @@ set -u -CONFIG_FILE=/mngr/code/github_sync.toml -STATUS_FILE=/tmp/github-sync-status.json -LOG_FILE=/tmp/post-commit-push.log +# Env overrides exist for tests; production always uses the defaults. +# GITHUB_SYNC_STATUS_FILE is the same override the github-sync service honors +# when writing the file this hook reads. +CONFIG_FILE="${GITHUB_SYNC_CONFIG_FILE:-/mngr/code/github_sync.toml}" +STATUS_FILE="${GITHUB_SYNC_STATUS_FILE:-/tmp/github-sync-status.json}" +LOG_FILE="${GITHUB_SYNC_HOOK_LOG_FILE:-/tmp/post-commit-push.log}" # 1. Sync not configured, nothing to do. if [ ! -f "$CONFIG_FILE" ]; then @@ -47,9 +50,13 @@ case "$branch" in esac # 4. Respect a visibility halt: the service refuses to push while the sync -# repo is public (or unconfirmed), and so must we. +# repo is public (or unconfirmed), and so must we. A missing status file or +# field reads as "null"/empty, which is not "false", so the default is to +# allow (the service simply hasn't reported yet). Note: jq's `//` operator +# must NOT be used here -- `.is_push_allowed // true` would turn an +# explicit `false` into `true` and defeat the halt. if [ -f "$STATUS_FILE" ] && command -v jq >/dev/null 2>&1; then - allowed="$(jq -r '.is_push_allowed // true' "$STATUS_FILE" 2>/dev/null)" + allowed="$(jq -r '.is_push_allowed' "$STATUS_FILE" 2>/dev/null)" if [ "$allowed" = "false" ]; then echo "[$(date -Is)] skipping push of $branch: sync repo not confirmed private" >> "$LOG_FILE" exit 0 From 5b99f11ebdea951432739c40b74f950157c5337b Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:35:43 -0700 Subject: [PATCH 03/24] Clarify visibility-halt docs to match the keep-last-confirmed semantics Problem: libs/github_sync/README.md and the runner module docstring said pushes are refused whenever the repo's visibility "cannot be confirmed", but _refresh_visibility deliberately keeps the last confirmed answer when a re-check fails outright (retrying every tick), so a prior private confirmation is not invalidated by e.g. an offline gateway. Fix: reworded both to state the exact semantics -- pushes held until the first confirmed-private answer, halted whenever the repo is confirmed public, and a failed re-check retains the last confirmed answer (in that failure mode pushes would fail too, since they use the same gateway). Co-Authored-By: Claude Fable 5 --- libs/github_sync/README.md | 9 ++++++--- libs/github_sync/src/github_sync/runner.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/libs/github_sync/README.md b/libs/github_sync/README.md index 1cd30100e..38a440934 100644 --- a/libs/github_sync/README.md +++ b/libs/github_sync/README.md @@ -37,9 +37,12 @@ Once enabled, three pieces work together: machine is offline. A failed push is retried on the next tick; `--force` is never used (the service is the branch's only writer). - **Private-only enforcement**: the service re-checks the repo's visibility - through latchkey every 15 minutes and refuses to push while the repo is - public or its visibility cannot be confirmed. The post-commit hook consults - the same status and skips pushes during a halt. + through latchkey every 15 minutes; pushes are held until the first + confirmed-private answer and halted whenever the repo is confirmed public. + A re-check that fails outright (e.g. the gateway is offline -- in which + case pushes would fail too) keeps the last confirmed answer and is retried + every tick. The post-commit hook consults the same status and skips pushes + during a halt. - `runtime/secrets` is excluded via the worktree's own `.gitignore` (written at init), so e.g. the Cloudflare tunnel token never reaches the remote. - Each tick first clears a stale `index.lock` from the runtime worktree if diff --git a/libs/github_sync/src/github_sync/runner.py b/libs/github_sync/src/github_sync/runner.py index b7885842b..d09e1b42b 100644 --- a/libs/github_sync/src/github_sync/runner.py +++ b/libs/github_sync/src/github_sync/runner.py @@ -3,8 +3,10 @@ Polls runtime/ every TICK_INTERVAL_SECONDS; if there are uncommitted changes, makes a sync commit on the runtime-sync branch checked out at runtime/ and pushes it to origin through the latchkey gateway. Periodically re-verifies -that the sync repo is still private and halts pushes when it is public or its -visibility cannot be established. +that the sync repo is still private: pushes are held until visibility is +first confirmed private and halted whenever the repo is confirmed public. A +re-check that fails outright keeps the last confirmed answer and is retried +every tick (see _refresh_visibility). The service only exists when the github-sync skill has enabled sync (it adds the [program:github-sync] block to supervisord.conf). The skill normally also From 5b1172ba606a293c20ce094868bd1ad796d0c2e8 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:36:28 -0700 Subject: [PATCH 04/24] Normalize repo_url in the hook's secondary-gateway fallback Problem: scripts/git_hooks/post-commit composed the secondary-gateway push URL by appending .git to the raw repo_url extracted from github_sync.toml. A repo_url written with a trailing .git suffix or trailing slash (which github_sync.config.load_repo_url normalizes and therefore the Python side accepts) produced a broken URL like .../repo.git.git, silently breaking the offline-primary fallback. Fix: strip a trailing slash and .git suffix after extraction, mirroring load_repo_url's normalization; a no-op for the canonical bare form the skill writes. Co-Authored-By: Claude Fable 5 --- scripts/git_hooks/post-commit | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/git_hooks/post-commit b/scripts/git_hooks/post-commit index a0c8d19e0..86a080d92 100755 --- a/scripts/git_hooks/post-commit +++ b/scripts/git_hooks/post-commit @@ -73,6 +73,11 @@ fi if ! git push 2>&1; then if ! git push --set-upstream origin "$branch" 2>&1; then repo_url="$(sed -n 's/^repo_url *= *"\(.*\)".*/\1/p' "$CONFIG_FILE")" + # Normalize like github_sync.config.load_repo_url does, so a + # repo_url written with a trailing slash or .git suffix still + # composes a valid proxied URL below. + repo_url="${repo_url%/}" + repo_url="${repo_url%.git}" if [ -n "${LATCHKEY_GATEWAY_SECONDARY:-}" ] && [ -n "${LATCHKEY_GATEWAY_PASSWORD:-}" ] && [ -n "$repo_url" ]; then git -c "http.extraHeader=X-Latchkey-Gateway-Password: $LATCHKEY_GATEWAY_PASSWORD" \ push "${LATCHKEY_GATEWAY_SECONDARY%/}/gateway/$repo_url.git" "$branch" 2>&1 From 4350ce9f3520fde1ca14d81edfa3691bed41ce4f Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:37:02 -0700 Subject: [PATCH 05/24] Remove trailing blank lines left in manager_test.py Problem: deleting the _create_orphan_runtime_worktree test section at the end of libs/bootstrap/src/bootstrap/manager_test.py left two trailing blank lines, so the file was no longer ruff-format clean. Fix: stripped the trailing blank lines (file now passes ruff format --check). Co-Authored-By: Claude Fable 5 --- libs/bootstrap/src/bootstrap/manager_test.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/libs/bootstrap/src/bootstrap/manager_test.py b/libs/bootstrap/src/bootstrap/manager_test.py index d49642f18..6dc14679a 100644 --- a/libs/bootstrap/src/bootstrap/manager_test.py +++ b/libs/bootstrap/src/bootstrap/manager_test.py @@ -512,5 +512,3 @@ def test_initialize_workspace_main_branch_is_idempotent_on_clean_main( _initialize_workspace_main_branch() branch = _git_in(work_dir, "branch", "--show-current").stdout.strip() assert branch == "main" - - From 8088b6f23bf7fbb1eccbe17f5d6ea8b8ff9fddde Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:46:24 -0700 Subject: [PATCH 06/24] Write the github-sync status file atomically Problem: runner._write_status wrote /tmp/github-sync-status.json with a truncate-then-write (Path.write_text) while the post-commit hook reads the same file with jq on every commit to honor a visibility halt. A read landing mid-write saw truncated JSON, jq failed silently, and the hook defaulted to allowing the push -- defeating the security halt during the recurring 60s rewrite window. Fix: write the payload to a sibling .tmp file and os.replace it into place (the same atomic-write pattern libs/browser/src/browser/manifest.py uses), so the hook always reads either the old or the new complete file. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/runner.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/libs/github_sync/src/github_sync/runner.py b/libs/github_sync/src/github_sync/runner.py index d09e1b42b..3d1c4c303 100644 --- a/libs/github_sync/src/github_sync/runner.py +++ b/libs/github_sync/src/github_sync/runner.py @@ -270,7 +270,14 @@ def _push_runtime(state: _SyncState, repo_url: str) -> None: def _write_status(state: _SyncState) -> None: - """Mirror the sync state to the status file for the hook and the skill.""" + """Mirror the sync state to the status file for the hook and the skill. + + Written atomically (temp file in the same directory + ``os.replace``) so + the post-commit hook -- which reads this file concurrently to decide + whether a visibility halt is in effect -- can never observe a truncated + file. A truncated read would make jq fail and the hook would default to + allowing the push, silently defeating the halt. + """ payload = { "timestamp": _iso_or_none(_now_utc()), "repo_url": state.repo_url, @@ -282,8 +289,10 @@ def _write_status(state: _SyncState) -> None: "last_error": state.last_error, } status_path = status_file_path() + tmp_path = status_path.with_name(status_path.name + ".tmp") try: - status_path.write_text(json.dumps(payload, indent=2) + "\n") + tmp_path.write_text(json.dumps(payload, indent=2) + "\n") + os.replace(tmp_path, status_path) except OSError as e: logger.warning("Failed to write status file {}: {}", status_path, e) From d9a0783e4341af459cadf9445abe29843cc03cb6 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:47:28 -0700 Subject: [PATCH 07/24] Normalize repo_url trailing slash before the .git suffix Problem: load_repo_url in libs/github_sync/src/github_sync/config.py stripped the .git suffix before the trailing slash, so a repo_url written as 'https://github.com/u/repo.git/' kept its .git suffix; the post-commit hook normalizes the same config value in the opposite (correct) order, so the service and hook could disagree on the remote URL and the visibility check could query a wrong repo name. Fix: strip the trailing slash first, then the .git suffix, matching the hook's sed normalization; extended the normalization unit test with the combined '.git/' case as a regression test. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/config.py | 5 ++++- libs/github_sync/src/github_sync/config_test.py | 9 ++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/libs/github_sync/src/github_sync/config.py b/libs/github_sync/src/github_sync/config.py index 41e45cc88..0f2303e14 100644 --- a/libs/github_sync/src/github_sync/config.py +++ b/libs/github_sync/src/github_sync/config.py @@ -60,7 +60,10 @@ def load_repo_url() -> str | None: f"{CONFIG_PATH} must set repo_url to an " f"{GITHUB_URL_PREFIX}/ URL, got {repo_url!r}" ) - return repo_url.removesuffix(".git").rstrip("/") + # Strip any trailing slash before the .git suffix so a URL written as + # ".../repo.git/" also normalizes fully (same order as the post-commit + # hook's sed normalization). + return repo_url.rstrip("/").removesuffix(".git") def parse_owner_and_name(repo_url: str) -> tuple[str, str]: diff --git a/libs/github_sync/src/github_sync/config_test.py b/libs/github_sync/src/github_sync/config_test.py index 87dc6837e..ce595a76f 100644 --- a/libs/github_sync/src/github_sync/config_test.py +++ b/libs/github_sync/src/github_sync/config_test.py @@ -28,10 +28,17 @@ def test_load_repo_url_normalizes_git_suffix_and_trailing_slash( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.chdir(tmp_path) - (tmp_path / "github_sync.toml").write_text( + config_path = tmp_path / "github_sync.toml" + config_path.write_text( 'repo_url = "https://github.com/some-user/my-workspace.git"\n' ) assert load_repo_url() == "https://github.com/some-user/my-workspace" + # A .git suffix followed by a trailing slash must also normalize fully + # (the slash has to come off before the suffix). + config_path.write_text( + 'repo_url = "https://github.com/some-user/my-workspace.git/"\n' + ) + assert load_repo_url() == "https://github.com/some-user/my-workspace" def test_load_repo_url_raises_on_malformed_toml( From 21c96ada60f199bb4d1e6079dddcd6638e763287 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:56:27 -0700 Subject: [PATCH 08/24] Clarify the post-commit hook's skip-case comment Problem: the hook header said all four skip cases are silent and normal, but case 4 (the github-sync visibility halt) writes a log line and is a security condition, not a normal state. Fix: reword the comment to distinguish the silent normal skips (1-3) from the logged security halt (4). Comment-only; no behavior change. Co-Authored-By: Claude Fable 5 --- scripts/git_hooks/post-commit | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/git_hooks/post-commit b/scripts/git_hooks/post-commit index 86a080d92..47e6ed861 100755 --- a/scripts/git_hooks/post-commit +++ b/scripts/git_hooks/post-commit @@ -12,7 +12,8 @@ # container: the main /mngr/code repo and every worker worktree, each pushing # its own current branch. # -# Skips silently in these cases that are normal, not errors: +# Skips the push in these cases -- 1-3 are normal, not errors, and skip +# silently; 4 is a security halt and logs a line to the log file: # 1. GitHub sync is not configured (github_sync.toml absent). # 2. HEAD is detached (no branch to push to). # 3. The current branch is runtime-sync -- that branch belongs to the From df63b4fdd861cda02ff6f1b8e00c8eb1a7cad10f Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 18:57:46 -0700 Subject: [PATCH 09/24] Test _do_tick's self-healing worktree restore and deferral Problem: every _do_tick test pre-created the runtime worktree, leaving the recreated-workspace self-heal branch of runner._do_tick untested (restore runtime/ from origin's runtime-sync branch when the worktree is missing, and defer with a status error when origin is unreachable). Fix: add two runner tests driving that branch through a full tick -- one restoring prior runtime state from a local bare origin and pushing, one asserting the deferred-init error when origin cannot be reached. Co-Authored-By: Claude Fable 5 --- .../src/github_sync/runner_test.py | 82 ++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/libs/github_sync/src/github_sync/runner_test.py b/libs/github_sync/src/github_sync/runner_test.py index ac4823260..498e05430 100644 --- a/libs/github_sync/src/github_sync/runner_test.py +++ b/libs/github_sync/src/github_sync/runner_test.py @@ -23,8 +23,13 @@ _SyncState, status_file_path, ) -from github_sync.testing import init_repo, init_repo_with_origin, install_fake_latchkey -from github_sync.worktree import init_runtime_worktree +from github_sync.testing import ( + init_repo, + init_repo_with_origin, + install_fake_latchkey, + run_git, +) +from github_sync.worktree import init_runtime_worktree, is_runtime_worktree def _age_lock(lock_path: Path) -> None: @@ -203,6 +208,79 @@ def test_do_tick_holds_pushes_while_visibility_unconfirmed( assert status["visibility"] == "unknown" +def test_do_tick_restores_missing_worktree_from_origin( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """The recreated-workspace self-heal path: when runtime/ is not yet a + worktree, a tick restores it from origin's runtime-sync branch (prior + memory/tickets come back) and then syncs as usual.""" + # A first workspace creates runtime state and pushes it to the origin. + first_base = tmp_path / "first" + first_base.mkdir() + first_main, origin = init_repo_with_origin(first_base) + monkeypatch.chdir(first_main) + assert init_runtime_worktree() is True + (first_main / "runtime" / "memory.md").write_text("remember me\n") + run_git(first_main / "runtime", "add", "-A") + run_git(first_main / "runtime", "commit", "-qm", "state") + run_git(first_main / "runtime", "push", "--set-upstream", "origin", "runtime-sync") + + # A workspace recreated from the synced repo: github_sync.toml came along + # with the checkout, but the container-local runtime worktree did not. + second_main = tmp_path / "second" + init_repo(second_main) + (second_main / "seed.txt").write_text("seed\n") + run_git(second_main, "add", "-A") + run_git(second_main, "commit", "-qm", "seed") + run_git(second_main, "remote", "add", "origin", str(origin)) + (second_main / "github_sync.toml").write_text( + 'repo_url = "https://github.com/some-user/my-workspace"\n' + ) + install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": true}\'') + monkeypatch.chdir(second_main) + state = _SyncState() + + _do_tick(state) + + assert is_runtime_worktree() + assert (second_main / "runtime" / "memory.md").read_text() == "remember me\n" + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is True + assert status["last_push_ok"] is True + + +def test_do_tick_defers_when_worktree_missing_and_origin_unreachable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, +) -> None: + """A recreated workspace whose origin cannot be reached yet (e.g. the + GitHub permissions have not been re-granted) must defer the worktree init + and report that in the status, not create a fresh orphan branch.""" + main = tmp_path / "main" + init_repo(main) + (main / "seed.txt").write_text("seed\n") + run_git(main, "add", "-A") + run_git(main, "commit", "-qm", "seed") + run_git(main, "remote", "add", "origin", str(tmp_path / "does-not-exist.git")) + (main / "github_sync.toml").write_text( + 'repo_url = "https://github.com/some-user/my-workspace"\n' + ) + monkeypatch.chdir(main) + state = _SyncState() + + _do_tick(state) + + assert not is_runtime_worktree() + assert state.last_error is not None + assert "deferred" in state.last_error + status = json.loads(status_file_path().read_text()) + assert "deferred" in status["last_error"] + + def test_do_tick_idles_when_sync_not_configured( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 6380b13baa823aff99800e5e19b4b5c32b4c5799 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:05:14 -0700 Subject: [PATCH 10/24] Validate the sync repo URL's owner/repo shape at load time Problem: load_repo_url only checked the https://github.com/ prefix, so a github_sync.toml with e.g. repo_url = "https://github.com/owner-only" loaded fine and the GithubSyncConfigError was raised later by parse_owner_and_name inside check_repo_visibility. runner._do_tick only guards load_repo_url, so the error escaped the tick and crash-looped the supervised service (and made check-visibility traceback) instead of taking the graceful config-error status path. Fix: load_repo_url now validates the full / shape via parse_owner_and_name after normalizing, so every malformed-config case raises at the single load point that all callers already handle; added a unit test covering missing and extra path segments. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/config.py | 8 +++++++- libs/github_sync/src/github_sync/config_test.py | 16 ++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/libs/github_sync/src/github_sync/config.py b/libs/github_sync/src/github_sync/config.py index 0f2303e14..cfbdfd306 100644 --- a/libs/github_sync/src/github_sync/config.py +++ b/libs/github_sync/src/github_sync/config.py @@ -63,7 +63,13 @@ def load_repo_url() -> str | None: # Strip any trailing slash before the .git suffix so a URL written as # ".../repo.git/" also normalizes fully (same order as the post-commit # hook's sed normalization). - return repo_url.rstrip("/").removesuffix(".git") + normalized_url = repo_url.rstrip("/").removesuffix(".git") + # Validate the full / shape here, at the single load point, + # so a malformed URL surfaces as a GithubSyncConfigError that every caller + # already handles -- rather than escaping later from + # parse_owner_and_name inside the service tick and crash-looping it. + parse_owner_and_name(normalized_url) + return normalized_url def parse_owner_and_name(repo_url: str) -> tuple[str, str]: diff --git a/libs/github_sync/src/github_sync/config_test.py b/libs/github_sync/src/github_sync/config_test.py index ce595a76f..d7baf8223 100644 --- a/libs/github_sync/src/github_sync/config_test.py +++ b/libs/github_sync/src/github_sync/config_test.py @@ -70,6 +70,22 @@ def test_load_repo_url_raises_when_repo_url_missing( load_repo_url() +def test_load_repo_url_raises_when_owner_or_repo_segment_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A github.com URL without the full / shape must fail at load + # time (where callers handle GithubSyncConfigError), not later inside the + # service tick via parse_owner_and_name. + monkeypatch.chdir(tmp_path) + config_path = tmp_path / "github_sync.toml" + config_path.write_text('repo_url = "https://github.com/just-an-owner"\n') + with pytest.raises(GithubSyncConfigError): + load_repo_url() + config_path.write_text('repo_url = "https://github.com/owner/repo/extra"\n') + with pytest.raises(GithubSyncConfigError): + load_repo_url() + + def test_parse_owner_and_name_splits_url() -> None: assert parse_owner_and_name("https://github.com/some-user/my-repo") == ( "some-user", From 52d5db1a9e18332fbd28735f614f84128cc7e6c1 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:06:56 -0700 Subject: [PATCH 11/24] Test the visibility re-check caching and keep-last-confirmed policy Problem: _refresh_visibility's re-check policy -- a confirmed answer is cached for VISIBILITY_CHECK_INTERVAL_SECONDS, and a stale answer whose re-check fails outright keeps the last confirmed answer with checked_at unchanged so it retries next tick -- had no direct tests; only the first-check-from-scratch states were covered, so a regression in either behavior would pass the suite. Fix: add three _refresh_visibility unit tests: a fresh private confirmation is trusted without re-asking GitHub, a stale confirmation whose re-check fails keeps pushes enabled and retries next tick, and a stale confirmation that re-checks to public halts pushes. Co-Authored-By: Claude Fable 5 --- .../src/github_sync/runner_test.py | 82 ++++++++++++++++++- 1 file changed, 79 insertions(+), 3 deletions(-) diff --git a/libs/github_sync/src/github_sync/runner_test.py b/libs/github_sync/src/github_sync/runner_test.py index 498e05430..4b551148b 100644 --- a/libs/github_sync/src/github_sync/runner_test.py +++ b/libs/github_sync/src/github_sync/runner_test.py @@ -1,9 +1,11 @@ """Unit tests for the github-sync runner. Covers the stale-index-lock recovery (an interrupted commit must not wedge -every future tick) and full ticks against a local bare origin: commit+push -when the repo is confirmed private, and the push halt when it is public or -unverifiable. +every future tick), the visibility re-check policy (a confirmed answer is +cached for the check interval; a failed re-check keeps the last confirmed +answer and retries next tick), and full ticks against a local bare origin: +commit+push when the repo is confirmed private, and the push halt when it is +public or unverifiable. """ from __future__ import annotations @@ -12,14 +14,17 @@ import os import subprocess import time +from datetime import datetime, timedelta, timezone from pathlib import Path import pytest from github_sync.runner import ( STALE_LOCK_MIN_AGE_SECONDS, + VISIBILITY_CHECK_INTERVAL_SECONDS, _clear_stale_index_lock, _do_tick, + _refresh_visibility, _SyncState, status_file_path, ) @@ -29,8 +34,11 @@ install_fake_latchkey, run_git, ) +from github_sync.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC from github_sync.worktree import init_runtime_worktree, is_runtime_worktree +_REPO_URL = "https://github.com/some-user/my-workspace" + def _age_lock(lock_path: Path) -> None: """Backdate a lock file's mtime so it counts as stale (past the age guard).""" @@ -208,6 +216,74 @@ def test_do_tick_holds_pushes_while_visibility_unconfirmed( assert status["visibility"] == "unknown" +def _confirmed_private_state(age_seconds: float) -> _SyncState: + """A state whose private confirmation is `age_seconds` old.""" + state = _SyncState() + state.repo_url = _REPO_URL + state.visibility = VISIBILITY_PRIVATE + state.visibility_checked_at = datetime.now(timezone.utc) - timedelta( + seconds=age_seconds + ) + return state + + +def test_refresh_visibility_skips_recheck_while_confirmed_answer_is_fresh( + isolated_git_and_gateway_env: Path, fake_latchkey_bin: Path +) -> None: + """A confirmed answer younger than the check interval must be trusted + without re-asking GitHub (the fake would answer public, so any re-check + would flip the state and fail the assertion).""" + install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": false}\'') + state = _confirmed_private_state(age_seconds=1) + checked_at_before = state.visibility_checked_at + + _refresh_visibility(state, _REPO_URL) + + assert state.visibility == VISIBILITY_PRIVATE + assert state.is_push_allowed is True + assert state.visibility_checked_at == checked_at_before + + +def test_refresh_visibility_keeps_last_confirmed_answer_when_recheck_fails( + isolated_git_and_gateway_env: Path, fake_latchkey_bin: Path +) -> None: + """A stale confirmation whose re-check fails outright (gateway offline) + must keep the last confirmed answer -- pushes would fail too in that + state, so halting adds nothing -- and leave checked_at unchanged so the + re-check is retried on the next tick rather than in 15 minutes.""" + install_fake_latchkey(fake_latchkey_bin, "exit 7") + state = _confirmed_private_state( + age_seconds=VISIBILITY_CHECK_INTERVAL_SECONDS + 60 + ) + checked_at_before = state.visibility_checked_at + + _refresh_visibility(state, _REPO_URL) + + assert state.visibility == VISIBILITY_PRIVATE + assert state.is_push_allowed is True + assert state.visibility_checked_at == checked_at_before + + +def test_refresh_visibility_rechecks_stale_answer_and_halts_on_public( + isolated_git_and_gateway_env: Path, fake_latchkey_bin: Path +) -> None: + """Once the confirmed answer goes stale, a re-check happens and a repo + that flipped public halts pushes.""" + install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": false}\'') + state = _confirmed_private_state( + age_seconds=VISIBILITY_CHECK_INTERVAL_SECONDS + 60 + ) + checked_at_before = state.visibility_checked_at + assert checked_at_before is not None + + _refresh_visibility(state, _REPO_URL) + + assert state.visibility == VISIBILITY_PUBLIC + assert state.is_push_allowed is False + assert state.visibility_checked_at is not None + assert state.visibility_checked_at > checked_at_before + + def test_do_tick_restores_missing_worktree_from_origin( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From 88daf046b6bc28eb96653d772b029a58fc3ba340 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:15:23 -0700 Subject: [PATCH 12/24] Align the visibility.py docstring with the actual halt policy Problem: the visibility.py module docstring said pushes halt whenever the repo's visibility cannot be established, but the implemented policy (and the README / runner docs) keep the last confirmed answer when a re-check fails outright -- only the initial establishment and a confirmed-public answer block pushes. Fix: reworded the docstring to describe the real policy and point at runner._refresh_visibility. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/visibility.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/github_sync/src/github_sync/visibility.py b/libs/github_sync/src/github_sync/visibility.py index 55e0c63cc..f4edec5bb 100644 --- a/libs/github_sync/src/github_sync/visibility.py +++ b/libs/github_sync/src/github_sync/visibility.py @@ -2,8 +2,10 @@ Public repos must never be synced to: agents might push secrets or other sensitive data without thinking about it. The skill verifies the repo is -private at enable time, and the service re-checks periodically, halting -pushes whenever the repo is public or its visibility cannot be established. +private at enable time, and the service re-checks periodically: pushes are +held until visibility is first confirmed private and halted whenever the +repo is confirmed public, while a re-check that fails outright keeps the +last confirmed answer (see runner._refresh_visibility for that policy). """ import json From 29fe66e5b56cd38188aed139abb266b7af1da869 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:16:02 -0700 Subject: [PATCH 13/24] Terminate init_runtime_worktree's branch-setup chain with an explicit else Problem: the if/elif chain that configures the branch after a successful worktree add in libs/github_sync/src/github_sync/worktree.py had no else clause, leaving the reattach-to-existing-local-branch no-op case implicit; style_guide.md requires every if/elif chain to end with an else. Fix: added an else branch with a pass and a comment explaining why the reattach case needs no setup. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/worktree.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/libs/github_sync/src/github_sync/worktree.py b/libs/github_sync/src/github_sync/worktree.py index 25a4e62ba..f871e7e24 100644 --- a/libs/github_sync/src/github_sync/worktree.py +++ b/libs/github_sync/src/github_sync/worktree.py @@ -250,6 +250,11 @@ def init_runtime_worktree() -> bool: commit.returncode, commit.stderr.strip(), ) + else: + # Reattached to an existing local runtime-sync branch: its history + # (including the secrets .gitignore) is already in place, and there is + # no remote ref to track yet, so there is nothing to set up. + pass # Restore staged-aside content. Calling unconditionally (rather than # gating on the `staged_aside` flag) also recovers content left by a From daaa9c530b456199405fcb0c36a5ea57d50fffce Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:16:27 -0700 Subject: [PATCH 14/24] Rename local_branch_exists to has_local_branch for symmetric naming Problem: init_runtime_worktree named its two parallel branch-existence booleans asymmetrically (has_remote_branch vs local_branch_exists), and the style guide's prefixed-boolean convention favors the has_/is_ shape. Fix: renamed the function-local variable to has_local_branch to mirror has_remote_branch. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/worktree.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/github_sync/src/github_sync/worktree.py b/libs/github_sync/src/github_sync/worktree.py index f871e7e24..25a15badb 100644 --- a/libs/github_sync/src/github_sync/worktree.py +++ b/libs/github_sync/src/github_sync/worktree.py @@ -157,7 +157,7 @@ def init_runtime_worktree() -> bool: _restore_preexisting_into_worktree() return True - local_branch_exists = ( + has_local_branch = ( git_main("rev-parse", "--verify", "--quiet", SYNC_BRANCH).returncode == 0 ) @@ -170,7 +170,7 @@ def init_runtime_worktree() -> bool: has_remote_branch = True elif ls_remote.returncode == _LS_REMOTE_NO_MATCHING_REFS: has_remote_branch = False - elif local_branch_exists: + elif has_local_branch: # Offline, but a local runtime-sync branch already exists (e.g. a # prior init whose worktree was removed) -- reattaching to it cannot # diverge from origin any further than it already has. @@ -209,7 +209,7 @@ def init_runtime_worktree() -> bool: result = git_main( "worktree", "add", "-B", SYNC_BRANCH, str(RUNTIME_DIR), remote_ref ) - elif local_branch_exists: + elif has_local_branch: result = git_main("worktree", "add", str(RUNTIME_DIR), SYNC_BRANCH) else: result = _create_orphan_runtime_worktree(SYNC_BRANCH) @@ -236,7 +236,7 @@ def init_runtime_worktree() -> bool: # Make sure the local branch tracks the remote (some git versions # don't set this automatically with -B + an explicit ref). git_runtime("branch", "--set-upstream-to", remote_ref) - elif not local_branch_exists: + elif not has_local_branch: # Fresh orphan branch: write the .gitignore for secrets and make an # initial commit so push has something to push. runtime/secrets holds # e.g. the Cloudflare tunnel token, which must never reach the remote. From 88876d5de9edb7a4016fe0e9c918c6af7add5211 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:17:01 -0700 Subject: [PATCH 15/24] Format libs/github_sync with the repo's ruff formatter Problem: six files in libs/github_sync were not clean under the formatter the pre-commit hook enforces (uvx ruff format), so any future commit touching them would produce noise reformat diffs. Fix: ran the hook's exact ruff check --fix and ruff format commands over libs/github_sync; the changes are wrapping and quote-style only. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/runner.py | 5 ++++- .../src/github_sync/runner_test.py | 20 ++++++++----------- .../src/github_sync/visibility_test.py | 4 ++-- libs/github_sync/src/github_sync/wiring.py | 4 +++- libs/github_sync/src/github_sync/worktree.py | 4 +--- libs/github_sync/test_github_sync_ratchets.py | 1 - 6 files changed, 18 insertions(+), 20 deletions(-) diff --git a/libs/github_sync/src/github_sync/runner.py b/libs/github_sync/src/github_sync/runner.py index 3d1c4c303..bb613e440 100644 --- a/libs/github_sync/src/github_sync/runner.py +++ b/libs/github_sync/src/github_sync/runner.py @@ -188,7 +188,10 @@ def _refresh_visibility(state: _SyncState, repo_url: str) -> None: leaves the previous answer in place and is retried next tick. Transitions are logged loudly since a repo flipping public is a security condition. """ - if state.visibility_checked_at is not None and state.visibility != VISIBILITY_UNKNOWN: + if ( + state.visibility_checked_at is not None + and state.visibility != VISIBILITY_UNKNOWN + ): age_seconds = (_now_utc() - state.visibility_checked_at).total_seconds() if age_seconds < VISIBILITY_CHECK_INTERVAL_SECONDS: return diff --git a/libs/github_sync/src/github_sync/runner_test.py b/libs/github_sync/src/github_sync/runner_test.py index 4b551148b..e8d434a93 100644 --- a/libs/github_sync/src/github_sync/runner_test.py +++ b/libs/github_sync/src/github_sync/runner_test.py @@ -133,7 +133,7 @@ def test_do_tick_commits_and_pushes_when_repo_private( """End to end against a local bare origin: new runtime state is committed on runtime-sync and pushed once the repo is confirmed private.""" main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) - install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": true}\'') + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": true}'") (main / "runtime" / "memory.txt").write_text("important state\n") state = _SyncState() @@ -158,7 +158,7 @@ def test_do_tick_self_heals_stale_index_lock( """A stale index.lock from a killed prior tick must not wedge syncing: the next tick clears it and commits the pending runtime state.""" main, _ = _set_up_synced_workspace(tmp_path, monkeypatch) - install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": true}\'') + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": true}'") lock_path = main / ".git" / "worktrees" / "runtime" / "index.lock" lock_path.write_text("") _age_lock(lock_path) @@ -178,7 +178,7 @@ def test_do_tick_halts_pushes_when_repo_public( ) -> None: """Public repo => commit locally (nothing is lost) but never push.""" main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) - install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": false}\'') + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": false}'") (main / "runtime" / "memory.txt").write_text("important state\n") state = _SyncState() @@ -233,7 +233,7 @@ def test_refresh_visibility_skips_recheck_while_confirmed_answer_is_fresh( """A confirmed answer younger than the check interval must be trusted without re-asking GitHub (the fake would answer public, so any re-check would flip the state and fail the assertion).""" - install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": false}\'') + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": false}'") state = _confirmed_private_state(age_seconds=1) checked_at_before = state.visibility_checked_at @@ -252,9 +252,7 @@ def test_refresh_visibility_keeps_last_confirmed_answer_when_recheck_fails( state, so halting adds nothing -- and leave checked_at unchanged so the re-check is retried on the next tick rather than in 15 minutes.""" install_fake_latchkey(fake_latchkey_bin, "exit 7") - state = _confirmed_private_state( - age_seconds=VISIBILITY_CHECK_INTERVAL_SECONDS + 60 - ) + state = _confirmed_private_state(age_seconds=VISIBILITY_CHECK_INTERVAL_SECONDS + 60) checked_at_before = state.visibility_checked_at _refresh_visibility(state, _REPO_URL) @@ -269,10 +267,8 @@ def test_refresh_visibility_rechecks_stale_answer_and_halts_on_public( ) -> None: """Once the confirmed answer goes stale, a re-check happens and a repo that flipped public halts pushes.""" - install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": false}\'') - state = _confirmed_private_state( - age_seconds=VISIBILITY_CHECK_INTERVAL_SECONDS + 60 - ) + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": false}'") + state = _confirmed_private_state(age_seconds=VISIBILITY_CHECK_INTERVAL_SECONDS + 60) checked_at_before = state.visibility_checked_at assert checked_at_before is not None @@ -315,7 +311,7 @@ def test_do_tick_restores_missing_worktree_from_origin( (second_main / "github_sync.toml").write_text( 'repo_url = "https://github.com/some-user/my-workspace"\n' ) - install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": true}\'') + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": true}'") monkeypatch.chdir(second_main) state = _SyncState() diff --git a/libs/github_sync/src/github_sync/visibility_test.py b/libs/github_sync/src/github_sync/visibility_test.py index ff19670d0..d8466c827 100644 --- a/libs/github_sync/src/github_sync/visibility_test.py +++ b/libs/github_sync/src/github_sync/visibility_test.py @@ -38,12 +38,12 @@ def test_parse_visibility_response_unknown_on_error_bodies() -> None: def test_check_repo_visibility_reads_private_from_latchkey( fake_latchkey_bin: Path, ) -> None: - install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": true}\'') + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": true}'") assert check_repo_visibility(_REPO_URL) == VISIBILITY_PRIVATE def test_check_repo_visibility_detects_public_repo(fake_latchkey_bin: Path) -> None: - install_fake_latchkey(fake_latchkey_bin, 'echo \'{"private": false}\'') + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": false}'") assert check_repo_visibility(_REPO_URL) == VISIBILITY_PUBLIC diff --git a/libs/github_sync/src/github_sync/wiring.py b/libs/github_sync/src/github_sync/wiring.py index ef5367671..5ccbc3f14 100644 --- a/libs/github_sync/src/github_sync/wiring.py +++ b/libs/github_sync/src/github_sync/wiring.py @@ -73,7 +73,9 @@ def _remove_gateway_entries(kept_gateway_url: str | None) -> None: _git_config("--unset-all", key) kept_header_key = ( - f"http.{kept_gateway_url}/.extraheader" if kept_gateway_url is not None else None + f"http.{kept_gateway_url}/.extraheader" + if kept_gateway_url is not None + else None ) for key, value in _list_global_config(r"http\..*\.extraheader"): is_gateway_header = value.startswith( diff --git a/libs/github_sync/src/github_sync/worktree.py b/libs/github_sync/src/github_sync/worktree.py index 25a15badb..fc170e7e1 100644 --- a/libs/github_sync/src/github_sync/worktree.py +++ b/libs/github_sync/src/github_sync/worktree.py @@ -163,9 +163,7 @@ def init_runtime_worktree() -> bool: # Classify the remote state: branch exists (restore it), remote reachable # but branch absent (fresh orphan), or remote unreachable (retry later). - ls_remote = git_main( - "ls-remote", "--exit-code", "--heads", "origin", SYNC_BRANCH - ) + ls_remote = git_main("ls-remote", "--exit-code", "--heads", "origin", SYNC_BRANCH) if ls_remote.returncode == 0: has_remote_branch = True elif ls_remote.returncode == _LS_REMOTE_NO_MATCHING_REFS: diff --git a/libs/github_sync/test_github_sync_ratchets.py b/libs/github_sync/test_github_sync_ratchets.py index dcf119a5a..548576101 100644 --- a/libs/github_sync/test_github_sync_ratchets.py +++ b/libs/github_sync/test_github_sync_ratchets.py @@ -75,4 +75,3 @@ def test_prevent_asyncio_import() -> None: def test_prevent_dataclasses_import() -> None: rc.check_dataclasses_import(_DIR, snapshot(0)) - From 7fb7b2a3eb7182965c6e0e5fc11036f4e0c74066 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:25:15 -0700 Subject: [PATCH 16/24] Reset visibility state when the configured sync repo changes Problem: _do_tick in libs/github_sync/src/github_sync/runner.py carried the previous repo's confirmed-private visibility answer across a github_sync.toml repo_url change, so a repointed config could receive pushes for up to 15 minutes without the new repo's visibility ever being confirmed -- violating the per-repo pushes-held-until-confirmed- private invariant. Fix: detect a repo_url change in _do_tick and reset visibility to UNKNOWN (clearing visibility_checked_at) so the swapped-in repo must earn its own confirmed-private answer; added a regression test proving a fresh confirmation for a different repo no longer authorizes pushes. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/runner.py | 15 ++++++++++- .../src/github_sync/runner_test.py | 27 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/libs/github_sync/src/github_sync/runner.py b/libs/github_sync/src/github_sync/runner.py index bb613e440..9253ebaf6 100644 --- a/libs/github_sync/src/github_sync/runner.py +++ b/libs/github_sync/src/github_sync/runner.py @@ -316,7 +316,20 @@ def _do_tick(state: _SyncState) -> None: ) _write_status(state) return - state.repo_url = repo_url + if repo_url != state.repo_url: + if state.repo_url is not None: + # The confirmed visibility answer belongs to the previously + # configured repo; a swapped-in repo must earn its own + # confirmed-private answer before any push. + logger.info( + "Sync repo changed from {} to {}; holding pushes until the " + "new repo is confirmed private", + state.repo_url, + repo_url, + ) + state.visibility = VISIBILITY_UNKNOWN + state.visibility_checked_at = None + state.repo_url = repo_url if not is_runtime_worktree(): # Self-healing path for a workspace recreated from a synced repo: the diff --git a/libs/github_sync/src/github_sync/runner_test.py b/libs/github_sync/src/github_sync/runner_test.py index e8d434a93..98a9fae90 100644 --- a/libs/github_sync/src/github_sync/runner_test.py +++ b/libs/github_sync/src/github_sync/runner_test.py @@ -280,6 +280,33 @@ def test_refresh_visibility_rechecks_stale_answer_and_halts_on_public( assert state.visibility_checked_at > checked_at_before +def test_do_tick_rechecks_visibility_when_repo_url_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """A repointed github_sync.toml must not inherit the previous repo's + confirmed-private answer: the new repo has to earn its own confirmation + before any push (here the re-check fails, so pushes stay held).""" + main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, "exit 7") + (main / "runtime" / "memory.txt").write_text("important state\n") + # State carried over from ticks against a different, previously-configured + # repo, with a still-fresh private confirmation. + state = _confirmed_private_state(age_seconds=1) + state.repo_url = "https://github.com/some-user/previous-repo" + + _do_tick(state) + + assert state.repo_url == _REPO_URL + remote_branch = _git_out(origin, "rev-parse", "--verify", "runtime-sync") + assert remote_branch.returncode != 0 + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is False + assert status["visibility"] == "unknown" + + def test_do_tick_restores_missing_worktree_from_origin( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From b2a95b36b5224418e47f44e1ecf14c173d840287 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:25:46 -0700 Subject: [PATCH 17/24] Correct the bootstrap comment about insteadOf rewrite chaining Problem: the comment above _GIT_GLOBAL_CONFIG_ARGVS in libs/bootstrap/src/bootstrap/manager.py implied that the ssh->https rewrite's output is then routed through github-sync's gateway wiring, but git applies at most one insteadOf rewrite per URL (verified empirically) -- ssh-form remotes end at plain https and never reach the gateway. Fix: reword the comment to state the non-chaining behavior explicitly and note that only remotes stored as https URLs (the shape the skill always configures) route through the gateway. Co-Authored-By: Claude Fable 5 --- libs/bootstrap/src/bootstrap/manager.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libs/bootstrap/src/bootstrap/manager.py b/libs/bootstrap/src/bootstrap/manager.py index b0ab5da68..2de6024d4 100644 --- a/libs/bootstrap/src/bootstrap/manager.py +++ b/libs/bootstrap/src/bootstrap/manager.py @@ -42,8 +42,11 @@ _CLAUDE_CONFIG_DIR_ENV_VAR = "CLAUDE_CONFIG_DIR" # Global git config applied on every boot: rewrite git@ / ssh:// GitHub -# remotes to https (there are no SSH credentials in the container; https is -# what the latchkey gateway wiring rewrites when github-sync is enabled). +# remotes to https (there are no SSH credentials in the container). Note that +# git applies at most one insteadOf rewrite per URL, so this rewrite's output +# is NOT further rewritten by github-sync's latchkey gateway wiring: only +# remotes stored as https://github.com/ URLs (the shape the github-sync skill +# always configures) route through the gateway. # core.hooksPath is deliberately NOT set here -- the post-commit auto-push # hook only becomes active when the github-sync skill wires it up. _GIT_GLOBAL_CONFIG_ARGVS = ( From 8a04e69396a6875369a0c2a5131739970507b80b Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:46:24 -0700 Subject: [PATCH 18/24] Clarify that use-inspiration's commit push is gated on GitHub sync Problem: .agents/skills/use-inspiration/SKILL.md (section 7) still said 'the post-commit hook handles any push', implying the auto-push hook is always active, but on this branch the hook is inert until the opt-in github-sync skill wires it up. Fix: reword the parenthetical to condition the push on GitHub sync being enabled, matching the branch-wide doc sweep. Co-Authored-By: Claude Fable 5 --- .agents/skills/use-inspiration/SKILL.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.agents/skills/use-inspiration/SKILL.md b/.agents/skills/use-inspiration/SKILL.md index 084b6842d..160f51d6e 100644 --- a/.agents/skills/use-inspiration/SKILL.md +++ b/.agents/skills/use-inspiration/SKILL.md @@ -158,7 +158,8 @@ one never removes or overwrites the manifests of the others. ## 7. Commit -Commit the adaptation per the repo's git conventions (a plain local commit; the -post-commit hook handles any push). Include the merged-in tree, the modified +Commit the adaptation per the repo's git conventions (a plain local commit; +when the user has enabled GitHub sync, the post-commit hook handles any push). +Include the merged-in tree, the modified files from filling holes, and the updated manifest with its new `Adaptation history` entry. From 0ed2415355dc50bf3f31b9273798b75f2cd9bef3 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:46:51 -0700 Subject: [PATCH 19/24] Update show-files-in-chat's runtime/ durability wording for opt-in sync Problem: .agents/skills/show-files-in-chat/SKILL.md said runtime/chat-images/ 'is backed up along with the rest of runtime/', phrasing written for the removed always-on runtime-backup service; runtime/ git sync is now opt-in. Fix: reword to say the directory persists with runtime/ and is covered by the opt-in GitHub sync when enabled, matching the branch-wide doc sweep. Co-Authored-By: Claude Fable 5 --- .agents/skills/show-files-in-chat/SKILL.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.agents/skills/show-files-in-chat/SKILL.md b/.agents/skills/show-files-in-chat/SKILL.md index 77360fe88..b1a308471 100644 --- a/.agents/skills/show-files-in-chat/SKILL.md +++ b/.agents/skills/show-files-in-chat/SKILL.md @@ -15,7 +15,8 @@ any other file is offered as a download. 1. Write the image to disk under `runtime/chat-images/` (create it once with `mkdir -p runtime/chat-images` if it does not exist). That directory is - gitignored and is backed up along with the rest of `runtime/`. + gitignored and persists with the rest of `runtime/` (covered by the opt-in + GitHub sync when the user has enabled the `github-sync` skill). Give each image a unique, descriptive filename, e.g. `revenue-by-quarter-2026.png`. Served image URLs are cached immutably, so From 15760a9d2c1599f0bc520d1319b3db8df3ed3cd1 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 19:55:34 -0700 Subject: [PATCH 20/24] Disable TTY credential prompts in the post-commit auto-push hook Problem: the hook's background push attempts (plain push, --set-upstream fallback, secondary-gateway fallback) in scripts/git_hooks/post-commit ran without GIT_TERMINAL_PROMPT=0. A push hitting an auth challenge (e.g. a 401 from a lapsed latchkey gateway permission) would make git prompt for a username on the controlling TTY, writing a stray prompt into the committing agent's tmux session and hanging the background job forever. The hook's own test already had to inject GIT_TERMINAL_PROMPT=0 to guard against this, and the Python-side git helpers (github_sync.worktree._git_noninteractive_env) set it for the same reason. Fix: export GIT_TERMINAL_PROMPT=0 at the top of the hook. Credentials are injected server-side via the gateway headers, so a prompt can never lead to a successful push; disabling it turns the hang into a fast failure recorded in the hook log. Co-Authored-By: Claude Fable 5 --- scripts/git_hooks/post-commit | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/scripts/git_hooks/post-commit b/scripts/git_hooks/post-commit index 47e6ed861..dcd232378 100755 --- a/scripts/git_hooks/post-commit +++ b/scripts/git_hooks/post-commit @@ -27,6 +27,14 @@ set -u +# Never prompt for credentials on the controlling TTY: the gateway wiring +# authenticates pushes via injected headers, so a prompt (e.g. after a 401 +# from a lapsed gateway permission) can never succeed -- it would only write +# a stray username prompt into the committing agent's tmux session and hang +# the background push forever. With prompts disabled the push fails fast and +# the failure lands in the log file. +export GIT_TERMINAL_PROMPT=0 + # Env overrides exist for tests; production always uses the defaults. # GITHUB_SYNC_STATUS_FILE is the same override the github-sync service honors # when writing the file this hook reads. From 5c47203d3b408842134c6375ea54cf7a282eea28 Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Sat, 11 Jul 2026 20:03:49 -0700 Subject: [PATCH 21/24] Ensure the secrets .gitignore exists on every runtime worktree init path Problem: init_runtime_worktree wrote the secrets-excluding .gitignore only on the fresh-orphan path. _create_orphan_runtime_worktree creates the runtime-sync branch (pointing at the bare empty-tree init commit) before its final `git worktree add`; if that add failed, the retry reattached to a branch with no .gitignore in its history, and the next sync tick's `git add -A` would have staged and pushed runtime/secrets (Cloudflare tunnel token, restic credentials) to the GitHub remote. Fix: after any successful worktree add, write and commit the .gitignore whenever it is missing (never clobbering an existing one), instead of only on the fresh-orphan path. Added a regression test that reattaches to an interrupted-init branch and asserts the exclusion is in place and committed. Co-Authored-By: Claude Fable 5 --- libs/github_sync/src/github_sync/worktree.py | 23 +++++++++------- .../src/github_sync/worktree_test.py | 26 +++++++++++++++++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/libs/github_sync/src/github_sync/worktree.py b/libs/github_sync/src/github_sync/worktree.py index fc170e7e1..00a0f3f11 100644 --- a/libs/github_sync/src/github_sync/worktree.py +++ b/libs/github_sync/src/github_sync/worktree.py @@ -234,11 +234,19 @@ def init_runtime_worktree() -> bool: # Make sure the local branch tracks the remote (some git versions # don't set this automatically with -B + an explicit ref). git_runtime("branch", "--set-upstream-to", remote_ref) - elif not has_local_branch: - # Fresh orphan branch: write the .gitignore for secrets and make an - # initial commit so push has something to push. runtime/secrets holds - # e.g. the Cloudflare tunnel token, which must never reach the remote. - gitignore = RUNTIME_DIR / ".gitignore" + + # Every path must end with the secrets-excluding .gitignore in place: + # runtime/secrets holds e.g. the Cloudflare tunnel token, which must never + # reach the remote. A restored or reattached branch normally already + # carries the file in its history, but a branch left behind by an + # interrupted prior init (created just before its `worktree add` failed) + # does not -- and without it the next sync tick's `git add -A` would ship + # the secrets. So write and commit it whenever it is missing, never + # clobbering an existing (possibly customized) one. On the fresh-orphan + # path this doubles as the initial commit that gives push something to + # push. + gitignore = RUNTIME_DIR / ".gitignore" + if not gitignore.exists(): gitignore.write_text("secrets\n") git_runtime("add", ".gitignore") commit = git_runtime("commit", "-m", "runtime sync: init") @@ -248,11 +256,6 @@ def init_runtime_worktree() -> bool: commit.returncode, commit.stderr.strip(), ) - else: - # Reattached to an existing local runtime-sync branch: its history - # (including the secrets .gitignore) is already in place, and there is - # no remote ref to track yet, so there is nothing to set up. - pass # Restore staged-aside content. Calling unconditionally (rather than # gating on the `staged_aside` flag) also recovers content left by a diff --git a/libs/github_sync/src/github_sync/worktree_test.py b/libs/github_sync/src/github_sync/worktree_test.py index 710984ba9..9cf94c839 100644 --- a/libs/github_sync/src/github_sync/worktree_test.py +++ b/libs/github_sync/src/github_sync/worktree_test.py @@ -114,6 +114,32 @@ def test_init_defers_when_origin_unreachable( assert (runtime / "precious.txt").read_text() == "keep\n" +def test_init_reattach_writes_missing_secrets_gitignore( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A runtime-sync branch left behind by an interrupted prior init (the + branch is created just before the `worktree add` that failed) has no + .gitignore in its history. Reattaching to it must still put the secrets + exclusion in place, or the next sync tick's `git add -A` would push + runtime/secrets to the remote.""" + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + # Mirror what the interrupted init leaves: a parentless empty-tree commit + # on runtime-sync and no worktree. + empty_tree = _git_out(main, "hash-object", "-w", "-t", "tree", "/dev/null").strip() + orphan_commit = _git_out( + main, "commit-tree", empty_tree, "-m", "runtime sync: init" + ).strip() + run_git(main, "branch", "runtime-sync", orphan_commit) + + assert init_runtime_worktree() is True + + runtime = main / "runtime" + assert (runtime / ".gitignore").read_text() == "secrets\n" + # Committed, not just written: the worktree is clean afterwards. + assert _git_out(runtime, "status", "--porcelain").strip() == "" + + def test_init_noops_when_already_a_worktree( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: From 147293fd22cfa321de0d9721c8349dad24574be3 Mon Sep 17 00:00:00 2001 From: github-sync Date: Tue, 14 Jul 2026 14:48:57 +0000 Subject: [PATCH 22/24] Fix github-sync enable flow to request github-write-all for repo creation The enable flow's REST-API permission ask requested github-write-repos, but the latchkey gateway scopes that (and github-read-repos) only to /repos/{owner}/{repo} paths -- existing-repo operations. Repo creation via POST /user/repos requires github-write-all, so the old ask forced a third, avoidable approval prompt mid-flow. Request github-write-all in the same combined ask (keeping github-read-repos for the least-privilege visibility read via GET /repos/{owner}/{repo}), and note at the repo-creation step why the broader permission is needed. Co-Authored-By: Claude Opus 4.8 (1M context) --- .agents/skills/github-sync/SKILL.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.agents/skills/github-sync/SKILL.md b/.agents/skills/github-sync/SKILL.md index e4b39eefa..2baf7ee89 100644 --- a/.agents/skills/github-sync/SKILL.md +++ b/.agents/skills/github-sync/SKILL.md @@ -46,7 +46,7 @@ enabled, three pieces work together (see `libs/github_sync/README.md`): -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-git", "permissions": ["github-git-read", "github-git-write"]}, "rationale": "GitHub sync: push this workspace'"'"'s branches and runtime state to your private sync repo."}' latchkey curl -XPOST http://latchkey-self.invalid/permission-requests \ -H 'Content-Type: application/json' \ - -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-rest-api", "permissions": ["github-read-repos", "github-write-repos"]}, "rationale": "GitHub sync: create the private sync repo and verify it stays private."}' + -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-rest-api", "permissions": ["github-read-repos", "github-write-all"]}, "rationale": "GitHub sync: create the private sync repo (needs github-write-all) and verify it stays private (github-read-repos)."}' ``` Wait for the approval system message before continuing. @@ -73,6 +73,11 @@ enabled, three pieces work together (see `libs/github_sync/README.md`): contain `"private": true` -- if it does not, delete/abandon the repo and stop; do not proceed with a public repo. + Repo creation (`POST /user/repos`) needs `github-write-all`, not + `github-write-repos`: the scoped `github-write-repos` permission only covers + existing-repo (`/repos/{owner}/{repo}`) paths, which is why step 2 requests + `github-write-all`. + 5. **Point origin at it and record the config**: ```bash From 18c70a9c495b025a0dcfe62c1b8a772ba3acf4ca Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Tue, 14 Jul 2026 08:09:15 -0700 Subject: [PATCH 23/24] Make the github-sync permission ask correct and self-sufficient Live-testing the enable flow surfaced two skill bugs that made the user approve GitHub access three times instead of one round. 1. Repo creation needs github-write-all (fixed in the previous commit): github-write-repos only covers existing-repo /repos/{owner}/{repo} paths, so POST /user/repos was rejected after the user had already approved, forcing a third mid-flow request. 2. The agent probed 'GET /user' to check whether the grants had landed, but that endpoint needs github-read-user, which the skill never requested. The probe therefore returned "Request not permitted" even after both grants had been approved, so the agent concluded the approval had not arrived and nagged the user to approve it again. Fix: request the exact set the flow needs in one up-front round (github-read-user + github-read-repos + github-write-all on the REST scope; read/write on the git scope), document what each permission is for, and state plainly that a rejected endpoint means that endpoint is out of scope -- not that a grant is missing -- with GET /user named as the one legitimate post-grant probe. Also drop the stale disable-path caveat that repo deletion needs an extra permission (github-write-all covers it) and use the create response's full_name as the authoritative owner/repo. Two approval dialogs remain the floor: a latchkey permission request carries exactly one scope, and GitHub's git and REST access are separate scopes. Both are now fired back-to-back before any GitHub call, so the user approves once and the flow runs to completion without returning. Co-Authored-By: Claude Fable 5 --- .agents/skills/github-sync/SKILL.md | 66 +++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 17 deletions(-) diff --git a/.agents/skills/github-sync/SKILL.md b/.agents/skills/github-sync/SKILL.md index 2baf7ee89..892ed3335 100644 --- a/.agents/skills/github-sync/SKILL.md +++ b/.agents/skills/github-sync/SKILL.md @@ -37,8 +37,11 @@ enabled, three pieces work together (see `libs/github_sync/README.md`): such program exists -- expected before enable). 2. **Request GitHub permissions** through latchkey (see the latchkey skill - for the permission-request mechanics). Both scopes are needed, in one - combined ask to the user: + for the permission-request mechanics). GitHub exposes two latchkey scopes + and a permission request carries exactly one scope, so this is two + requests -- **fire both back-to-back, before any other GitHub call**, so + the user approves them in a single sitting. Never dribble out further + requests later in the flow. ```bash latchkey curl -XPOST http://latchkey-self.invalid/permission-requests \ @@ -46,15 +49,47 @@ enabled, three pieces work together (see `libs/github_sync/README.md`): -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-git", "permissions": ["github-git-read", "github-git-write"]}, "rationale": "GitHub sync: push this workspace'"'"'s branches and runtime state to your private sync repo."}' latchkey curl -XPOST http://latchkey-self.invalid/permission-requests \ -H 'Content-Type: application/json' \ - -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-rest-api", "permissions": ["github-read-repos", "github-write-all"]}, "rationale": "GitHub sync: create the private sync repo (needs github-write-all) and verify it stays private (github-read-repos)."}' + -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-rest-api", "permissions": ["github-read-user", "github-read-repos", "github-write-all"]}, "rationale": "GitHub sync: create the private sync repo (needs github-write-all), confirm which GitHub account it lands under (github-read-user), and verify it stays private (github-read-repos)."}' ``` - Wait for the approval system message before continuing. + This exact permission set is what the flow needs -- do not trim it, or the + user gets asked again mid-flow: + + - `github-write-all` -- repo creation (`POST /user/repos`). The narrower + `github-write-repos` covers only existing-repo (`/repos/{owner}/{repo}`) + paths and is **not** enough to create a repo. It also covers the + optional repo deletion on disable. + - `github-read-repos` -- the recurring private-visibility check + (`GET /repos/{owner}/{repo}`), which the service repeats forever. + - `github-read-user` -- `GET /user`, to name the account the repo will be + created under (step 3) and as the one legitimate "did the grants land?" + probe (below). + - `github-git-read` / `github-git-write` -- clone/fetch and push. + + Then **wait for both approval system messages** ("Your permission request + for GitHub (git) / (REST API) was granted..."). Those messages are the + authoritative signal; they are what tells you to proceed. + + **Do not treat a rejected API call as evidence that a grant is missing.** + `{"error": "Error: Request not permitted by the user."}` means *that + endpoint* is not covered by the granted permissions -- it does not mean + the approval failed to arrive. Probing an endpoint outside the set above + (e.g. `GET /user/repos`, which needs broader read access) will be rejected + even when everything is granted correctly. If you want to sanity-check + after the grant messages arrive, the only probe to use is: + + ```bash + latchkey curl -s https://api.github.com/user + ``` + + Never re-ask the user for a permission you have already been told was + granted; re-read this list and check *which endpoint* you called instead. 3. **Pick the repo.** Default: create a brand-new private repo named after - the workspace (`$MINDS_WORKSPACE_NAME`). Confirm the name and owner with - the user first; owner defaults to their GitHub account, but they can name - an org. One exception: if `git remote get-url origin` already points at a + the workspace (`$MINDS_WORKSPACE_NAME`), owned by the authenticated GitHub + account (read its `login` from `latchkey curl -s https://api.github.com/user`). + Confirm the name and owner with the user first; they can name an org + instead. One exception: if `git remote get-url origin` already points at a user-owned repo (not `imbue-ai/default-workspace-template` or another shared template), ask whether to reuse it or create a fresh one -- recommend a fresh dedicated repo unless they have a specific reason. @@ -71,12 +106,8 @@ enabled, three pieces work together (see `libs/github_sync/README.md`): (For an org: `POST https://api.github.com/orgs//repos`.) On a 422 name-taken error, append `-2`, `-3`, ... and retry. The response JSON must contain `"private": true` -- if it does not, delete/abandon the repo and - stop; do not proceed with a public repo. - - Repo creation (`POST /user/repos`) needs `github-write-all`, not - `github-write-repos`: the scoped `github-write-repos` permission only covers - existing-repo (`/repos/{owner}/{repo}`) paths, which is why step 2 requests - `github-write-all`. + stop; do not proceed with a public repo. The response's `full_name` is the + authoritative `/` to use from here on. 5. **Point origin at it and record the config**: @@ -181,9 +212,10 @@ repo (recommend keeping it -- it costs nothing and preserves history). 3. Delete `github_sync.toml`. 4. Leave the local `runtime/` worktree and its history intact (harmless, and re-enabling picks it right back up). -5. If the user chose to delete the remote repo, try - `latchkey curl -s -X DELETE https://api.github.com/repos//`; - deleting repos needs an extra GitHub permission, so if it fails, point - them at the repo's GitHub settings page to delete it themselves. +5. If the user chose to delete the remote repo: + `latchkey curl -s -X DELETE https://api.github.com/repos//` + (covered by the `github-write-all` granted at enable). If the grant has + since been revoked, do not re-request it just for this -- point them at + the repo's GitHub settings page to delete it themselves. 6. Commit the removal. Note that this commit is NOT auto-pushed (the hook is inert again). From c735b74cc78bc6f4f649db5248208d9c606f212d Mon Sep 17 00:00:00 2001 From: Josh Albrecht Date: Tue, 14 Jul 2026 08:09:24 -0700 Subject: [PATCH 24/24] Note the permission-ask correction in the changelog Co-Authored-By: Claude Fable 5 --- changelog/mngr-github-backup-skill.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog/mngr-github-backup-skill.md b/changelog/mngr-github-backup-skill.md index c5c5eab2d..0ab0851ae 100644 --- a/changelog/mngr-github-backup-skill.md +++ b/changelog/mngr-github-backup-skill.md @@ -6,6 +6,8 @@ The new `github-sync` skill enables sync on request: it creates a dedicated PRIV Private-only is enforced: the skill refuses public repos at setup and the service re-verifies visibility every 15 minutes, halting all pushes (service and hook) when the repo is public or unverifiable. +The skill's permission ask was validated against a live workspace and corrected: creating the repo needs `github-write-all` (the narrower `github-write-repos` covers only existing-repo paths, not `POST /user/repos`), and `github-read-user` is requested so the skill can name the owning account and verify the grants landed. Both requests now go out back-to-back before any GitHub call, so the user approves once and setup runs to completion. + A workspace recreated from a previously-synced repo self-heals: after the user re-grants the GitHub permissions, the service re-wires git and restores `runtime/` (memory, tickets, transcripts) from the `runtime-sync` branch automatically. Disable is supported (full unwind of the service, hook, and git wiring; the user chooses whether to keep the remote repo). Existing workspaces that update-self onto this version simply stop git-syncing `runtime/` until they opt in via the skill; legacy `mindsbackup/*` branches are not migrated.