diff --git a/.agents/skills/build-web-service/scripts/scaffold_flask_lib.py b/.agents/skills/build-web-service/scripts/scaffold_flask_lib.py index ed9fb078c..b6990998a 100644 --- a/.agents/skills/build-web-service/scripts/scaffold_flask_lib.py +++ b/.agents/skills/build-web-service/scripts/scaffold_flask_lib.py @@ -43,7 +43,7 @@ "cloudflare-tunnel", "app-watcher", "bootstrap", - "runtime-backup", + "github-sync", "host-backup", "terminal", "deferred-install", diff --git a/.agents/skills/edit-services/SKILL.md b/.agents/skills/edit-services/SKILL.md index b77710f84..2d9df2aad 100644 --- a/.agents/skills/edit-services/SKILL.md +++ b/.agents/skills/edit-services/SKILL.md @@ -57,7 +57,7 @@ Key fields: instead. Services inherit the agent environment (`MNGR_AGENT_STATE_DIR`, -`CLAUDE_CONFIG_DIR`, `MNGR_HOST_DIR`, `GH_TOKEN`, ...) from the bootstrap shell +`CLAUDE_CONFIG_DIR`, `MNGR_HOST_DIR`, `LATCHKEY_*`, ...) from the bootstrap shell that launched supervisord -- you do not need a per-program `environment=`. ## Adding a service diff --git a/.agents/skills/github-sync/SKILL.md b/.agents/skills/github-sync/SKILL.md new file mode 100644 index 000000000..892ed3335 --- /dev/null +++ b/.agents/skills/github-sync/SKILL.md @@ -0,0 +1,221 @@ +--- +name: github-sync +description: Enable, check, or disable GitHub sync for this workspace. Enabling creates a dedicated PRIVATE GitHub repo via latchkey, points origin at it, auto-pushes every commit from every checkout, and continuously syncs runtime/ state (memory, tickets, transcripts). Use when the user asks to back up / sync the workspace to GitHub, enable auto-push, restore a previous workspace's state, or asks about GitHub sync status. +compatibility: Requires latchkey (see the latchkey skill) and the user approving GitHub permissions in the Minds app. +--- + +# GitHub sync + +GitHub sync is opt-in. Nothing syncs until this skill enables it. Once +enabled, three pieces work together (see `libs/github_sync/README.md`): + +1. `origin` points at a dedicated **private** GitHub repo for this workspace. +2. Global git wiring routes all `https://github.com/...` traffic through the + latchkey gateway (credential injected server-side; no token in the + container) and activates the `post-commit` hook, so every commit on any + checkout -- main repo and worker worktrees -- auto-pushes its branch. +3. The `[program:github-sync]` service commits + pushes `runtime/` to the + `runtime-sync` orphan branch every 60s and re-verifies the repo stays + private, halting pushes if it ever isn't. + +## Hard rules + +- **Private repos only.** Never create a public repo, never point sync at a + public repo, and never work around a visibility halt. If the user asks for + a public sync repo, decline and explain: agents can push secrets or other + sensitive data without realizing it. +- **Everything flows through latchkey.** Never ask the user for a GitHub + token and never embed credentials in URLs or git config. +- `origin` is reserved for the sync repo. Upstream-template operations keep + using `parent.toml` (see the update-self skill) and are unaffected. + +## Enable + +1. **Check current state**: `uv run github-sync status`. If `is_configured` + is already true, jump to "Status" (or "Repair" if the service is + unhealthy). Also run `supervisorctl status github-sync` (it errors when no + such program exists -- expected before enable). + +2. **Request GitHub permissions** through latchkey (see the latchkey skill + for the permission-request mechanics). GitHub exposes two latchkey scopes + and a permission request carries exactly one scope, so this is two + requests -- **fire both back-to-back, before any other GitHub call**, so + the user approves them in a single sitting. Never dribble out further + requests later in the flow. + + ```bash + latchkey curl -XPOST http://latchkey-self.invalid/permission-requests \ + -H 'Content-Type: application/json' \ + -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-git", "permissions": ["github-git-read", "github-git-write"]}, "rationale": "GitHub sync: push this workspace'"'"'s branches and runtime state to your private sync repo."}' + latchkey curl -XPOST http://latchkey-self.invalid/permission-requests \ + -H 'Content-Type: application/json' \ + -d '{"agent_id": "'"$MNGR_AGENT_ID"'", "type": "predefined", "payload": {"scope": "github-rest-api", "permissions": ["github-read-user", "github-read-repos", "github-write-all"]}, "rationale": "GitHub sync: create the private sync repo (needs github-write-all), confirm which GitHub account it lands under (github-read-user), and verify it stays private (github-read-repos)."}' + ``` + + This exact permission set is what the flow needs -- do not trim it, or the + user gets asked again mid-flow: + + - `github-write-all` -- repo creation (`POST /user/repos`). The narrower + `github-write-repos` covers only existing-repo (`/repos/{owner}/{repo}`) + paths and is **not** enough to create a repo. It also covers the + optional repo deletion on disable. + - `github-read-repos` -- the recurring private-visibility check + (`GET /repos/{owner}/{repo}`), which the service repeats forever. + - `github-read-user` -- `GET /user`, to name the account the repo will be + created under (step 3) and as the one legitimate "did the grants land?" + probe (below). + - `github-git-read` / `github-git-write` -- clone/fetch and push. + + Then **wait for both approval system messages** ("Your permission request + for GitHub (git) / (REST API) was granted..."). Those messages are the + authoritative signal; they are what tells you to proceed. + + **Do not treat a rejected API call as evidence that a grant is missing.** + `{"error": "Error: Request not permitted by the user."}` means *that + endpoint* is not covered by the granted permissions -- it does not mean + the approval failed to arrive. Probing an endpoint outside the set above + (e.g. `GET /user/repos`, which needs broader read access) will be rejected + even when everything is granted correctly. If you want to sanity-check + after the grant messages arrive, the only probe to use is: + + ```bash + latchkey curl -s https://api.github.com/user + ``` + + Never re-ask the user for a permission you have already been told was + granted; re-read this list and check *which endpoint* you called instead. + +3. **Pick the repo.** Default: create a brand-new private repo named after + the workspace (`$MINDS_WORKSPACE_NAME`), owned by the authenticated GitHub + account (read its `login` from `latchkey curl -s https://api.github.com/user`). + Confirm the name and owner with the user first; they can name an org + instead. One exception: if `git remote get-url origin` already points at a + user-owned repo (not `imbue-ai/default-workspace-template` or another + shared template), ask whether to reuse it or create a fresh one -- + recommend a fresh dedicated repo unless they have a specific reason. + Reused repos must be verified private and writable like new ones. + +4. **Create the repo** (skip if reusing): + + ```bash + latchkey curl -s -X POST https://api.github.com/user/repos \ + -H 'Content-Type: application/json' \ + -d '{"name": "", "private": true, "description": "Private sync repo for the minds workspace"}' + ``` + + (For an org: `POST https://api.github.com/orgs//repos`.) On a 422 + name-taken error, append `-2`, `-3`, ... and retry. The response JSON must + contain `"private": true` -- if it does not, delete/abandon the repo and + stop; do not proceed with a public repo. The response's `full_name` is the + authoritative `/` to use from here on. + +5. **Point origin at it and record the config**: + + ```bash + git remote set-url origin https://github.com//.git \ + || git remote add origin https://github.com//.git + ``` + + Write `github_sync.toml` at the repo root (this file is the "sync is + enabled" marker for the service and the post-commit hook): + + ```toml + # Written by the github-sync skill. Presence of this file enables GitHub + # sync; `uv run github-sync status` reports on it. + repo_url = "https://github.com//" + ``` + +6. **Wire git through the gateway**: `uv run github-sync wire-git`. From now + on plain `git push`/`git fetch` against github.com works in every + checkout, and the post-commit auto-push hook is active. + +7. **Create (or restore) the runtime worktree**: + `uv run github-sync setup-worktree`. If origin already has a + `runtime-sync` branch (re-enabling for a workspace recreated from a + previously-synced repo), this restores the prior runtime/ state instead of + starting fresh -- tell the user their memory/tickets/transcripts are back. + +8. **Verify private before any push**: `uv run github-sync check-visibility` + must print `private` (exit 0). If not, stop and surface the problem. + +9. **Initial sync**: push the current branch, the runtime-sync branch, and + any existing worker branches: + + ```bash + git push --set-upstream origin "$(git branch --show-current)" + git -C runtime push --set-upstream origin runtime-sync + for b in $(git for-each-ref --format='%(refname:short)' refs/heads/ | grep -v -x -e "$(git branch --show-current)" -e runtime-sync); do git push origin "$b"; done + ``` + +10. **Add the service** by appending this block to `supervisord.conf`, then + `supervisorctl reread && supervisorctl update` (see the edit-services + skill): + + ```ini + # Opt-in GitHub sync (added by the github-sync skill): commits + pushes + # runtime/ to the runtime-sync branch of the private sync repo and + # re-verifies the repo stays private. See libs/github_sync/README.md. + [program:github-sync] + command=uv run github-sync run + directory=/mngr/code + autostart=true + autorestart=true + startretries=1000000 + stopasgroup=true + killasgroup=true + stdout_logfile=/var/log/supervisor/github-sync-stdout.log + stderr_logfile=/var/log/supervisor/github-sync-stderr.log + stdout_logfile_maxbytes=10MB + stderr_logfile_maxbytes=10MB + stdout_logfile_backups=3 + stderr_logfile_backups=3 + ``` + +11. **Commit the enablement** (`github_sync.toml` + `supervisord.conf`). The + now-active hook pushes the commit; this also makes sync sticky if the + repo is later used to recreate the workspace. + +12. **Report**: the repo URL, that every commit now auto-pushes, that + runtime/ syncs every minute, and that pushes queue while their machine + (the latchkey gateway) is offline (on remote hosts the per-VPS secondary + gateway usually covers that). + +## Status + +`uv run github-sync status` prints config + the service's latest status +(visibility, last push, errors); `supervisorctl status github-sync` shows the +process; logs are at `/var/log/supervisor/github-sync-*.log` and +`/tmp/github-sync.log`, hook output at `/tmp/post-commit-push.log`. Explain +findings in plain language. If `is_push_allowed` is false, the repo is public +or unverifiable -- tell the user to make it private again; sync resumes +automatically. + +## Repair (workspace recreated from a synced repo) + +A workspace created from a previously-synced private repo inherits +`github_sync.toml` and the service block, but not the latchkey permissions or +the container-local wiring/worktree. The service idles until it can reach +origin. To repair: run step 2 (permission requests); the service self-heals +within a tick (re-wires git, restores runtime/ from origin). Verify with +"Status", or accelerate with `uv run github-sync wire-git` + +`uv run github-sync setup-worktree`. + +## Disable + +Confirm with the user first, and ask separately whether to keep the remote +repo (recommend keeping it -- it costs nothing and preserves history). + +1. `supervisorctl stop github-sync`, remove the `[program:github-sync]` block + from `supervisord.conf`, then `supervisorctl reread && supervisorctl update`. +2. `uv run github-sync unwire-git` (removes the gateway git config and the + hooks path -- auto-push stops). +3. Delete `github_sync.toml`. +4. Leave the local `runtime/` worktree and its history intact (harmless, and + re-enabling picks it right back up). +5. If the user chose to delete the remote repo: + `latchkey curl -s -X DELETE https://api.github.com/repos//` + (covered by the `github-write-all` granted at enable). If the grant has + since been revoked, do not re-request it just for this -- point them at + the repo's GitHub settings page to delete it themselves. +6. Commit the removal. Note that this commit is NOT auto-pushed (the hook is + inert again). diff --git a/.agents/skills/publish-inspiration/SKILL.md b/.agents/skills/publish-inspiration/SKILL.md index 26e62f234..a849a1b39 100644 --- a/.agents/skills/publish-inspiration/SKILL.md +++ b/.agents/skills/publish-inspiration/SKILL.md @@ -591,7 +591,7 @@ exit 1 If the user never approves, surface a clear message and stop, leaving the assembled commit intact. Do NOT fall back to any other credential or -mechanism (no `GH_TOKEN`-in-URL pushes, no partial-tree API uploads -- see +mechanism (no token-in-URL pushes, no partial-tree API uploads -- see the "MUST BE BOOTABLE" callout). ## 8. Create the repo and push diff --git a/.agents/skills/show-files-in-chat/SKILL.md b/.agents/skills/show-files-in-chat/SKILL.md index 77360fe88..b1a308471 100644 --- a/.agents/skills/show-files-in-chat/SKILL.md +++ b/.agents/skills/show-files-in-chat/SKILL.md @@ -15,7 +15,8 @@ any other file is offered as a download. 1. Write the image to disk under `runtime/chat-images/` (create it once with `mkdir -p runtime/chat-images` if it does not exist). That directory is - gitignored and is backed up along with the rest of `runtime/`. + gitignored and persists with the rest of `runtime/` (covered by the opt-in + GitHub sync when the user has enabled the `github-sync` skill). Give each image a unique, descriptive filename, e.g. `revenue-by-quarter-2026.png`. Served image URLs are cached immutably, so diff --git a/.agents/skills/use-inspiration/SKILL.md b/.agents/skills/use-inspiration/SKILL.md index 084b6842d..160f51d6e 100644 --- a/.agents/skills/use-inspiration/SKILL.md +++ b/.agents/skills/use-inspiration/SKILL.md @@ -158,7 +158,8 @@ one never removes or overwrites the manifests of the others. ## 7. Commit -Commit the adaptation per the repo's git conventions (a plain local commit; the -post-commit hook handles any push). Include the merged-in tree, the modified +Commit the adaptation per the repo's git conventions (a plain local commit; +when the user has enabled GitHub sync, the post-commit hook handles any push). +Include the merged-in tree, the modified files from filling holes, and the updated manifest with its new `Adaptation history` entry. diff --git a/.gitignore b/.gitignore index a5b5ff95d..fc1858bf0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,8 +9,8 @@ # Default Workspace Template runtime state. # memory/ now lives at runtime/memory/ (configured via .claude/settings.json's -# autoMemoryDirectory) so the runtime-backup service backs it up alongside -# the rest of runtime/. +# autoMemoryDirectory) so the opt-in github-sync service (see the github-sync +# skill) covers it alongside the rest of runtime/. **/runtime/ # chat attachment uploads (arbitrary size/format; kept out of version control) diff --git a/.mngr/settings.toml b/.mngr/settings.toml index 3c4efa66c..1d5109599 100644 --- a/.mngr/settings.toml +++ b/.mngr/settings.toml @@ -5,7 +5,7 @@ type = "claude" connect = false ensure_clean = false -pass_env__extend = ["REMOTE_SERVICE_CONNECTOR_URL", "LATCHKEY_GATEWAY", "GH_TOKEN"] +pass_env__extend = ["REMOTE_SERVICE_CONNECTOR_URL", "LATCHKEY_GATEWAY"] host_env__extend = [ "IS_SANDBOX=1", "IS_AUTONOMOUS=1", @@ -28,7 +28,8 @@ host_env__extend = [ # Disable claude.ai account MCP connector sync -- it's an OAuth side # channel to third-party services that bypasses the latchkey gateway. "ENABLE_CLAUDEAI_MCP_SERVERS=false", - # Tickets live inside runtime/ so they ride the runtime-backup branch. + # Tickets live inside runtime/ so they are covered by the opt-in GitHub + # sync (github-sync skill) when the user enables it. # tk reads TICKETS_DIR if set; absolute path so it works from worktree # subdirs too. /mngr/code/ is the container WORKDIR (set by the Dockerfile); # a /code -> /mngr/code symlink is kept in the image as a safety net. @@ -193,13 +194,13 @@ type = "main" # to it instead. # # bootstrap is the only extra_window now. It runs first-boot setup (git config, -# runtime worktree, CLAUDE_CONFIG_DIR host-env write, backup config, initial -# chat agent) and then launches supervisord, which runs every background -# service (system_interface, web, cloudflared, app-watcher, runtime-backup, -# host-backup, terminal, and the one-shot deferred-install -- see -# supervisord.conf). The old git_auth_setup window is gone (the git config it -# performed now runs inside bootstrap), and terminal is a supervisord service -# rather than its own window. +# CLAUDE_CONFIG_DIR host-env write, initial chat agent) and then launches +# supervisord, which runs every background service (system_interface, web, +# cloudflared, app-watcher, host-backup, terminal, and the one-shot +# deferred-install -- see supervisord.conf; the opt-in github-sync service is +# added by its skill). The old git_auth_setup window is gone (the git config +# it performed now runs inside bootstrap), and terminal is a supervisord +# service rather than its own window. extra_window = ["bootstrap='uv run bootstrap'"] # Write a sane default tmux config on every minds host. tuple-typed # `extra_provision_command` stacks across templates (see @@ -254,11 +255,11 @@ start_arg__extend = ["--security-opt=no-new-privileges", "--workdir=/", "--resta # Disable idle detection entirely -- the host should never auto-shutdown. # This also prevents shutdown when all agent tmux sessions exit. idle_mode = "disabled" -# Forward Anthropic creds + GH_TOKEN from minds' subprocess env onto the +# Forward Anthropic creds from minds' subprocess env onto the # new container's env. The agent process inherits the container env, so # this is what gives the agent its ANTHROPIC_API_KEY (etc.) in # non-IMBUE_CLOUD compute modes. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Seed /mngr/code from /docker_build_code (baked into the image) onto the # per-host volume. Runs synchronously after the host is online but # before any agent work_dir setup, so the workspace is in place by the @@ -299,7 +300,7 @@ start_arg__extend = ["--cpus=2", "--memory=4", "--disk=20"] # version-check disable is needed. setting__extend = ["providers.lima.is_enabled=true"] # Same forwarding as the docker template; see comment there. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Run the SAME setup as a Dockerfile-built workspace, directly in the VM as root. # These run after the project is synced to /mngr/code (agent-level provisioning, # post work_dir sync) and reuse the exact scripts the Dockerfile RUNs. There is @@ -347,7 +348,7 @@ setting__extend = [ "providers.modal.default_idle_timeout=86400", ] # Same forwarding as the docker/lima templates so the agent inherits its creds. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Modal has no Dockerfile build (like lima): run the same setup a Dockerfile-built # workspace would, directly in the sandbox over SSH. extra_provision_command__extend = [ @@ -382,7 +383,7 @@ idle_mode = "disabled" # agent container back after a VPS reboot; the entrypoint then self-heals sshd. start_arg__extend = ["--security-opt=no-new-privileges", "--workdir=/", "--restart=unless-stopped"] # Same forwarding as the docker template; see comment there. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Same first-boot seed as the docker template; see comment there. post_host_create_command__extend = ["/usr/local/bin/default-workspace-template-seed"] # Install the outer-VM boot unit so the agent recovers after a VPS reboot even @@ -449,7 +450,7 @@ idle_mode = "disabled" # relaunched by the outer boot unit installed below. start_arg__extend = ["--security-opt=no-new-privileges", "--workdir=/", "--restart=unless-stopped"] # Same forwarding as the docker template; see comment there. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Same first-boot seed as the docker template; see comment there. post_host_create_command__extend = ["/usr/local/bin/default-workspace-template-seed"] # Install a systemd unit on the OUTER pool/slice VM that, on every VM boot, @@ -523,7 +524,7 @@ idle_mode = "disabled" # agent container back after an EC2 reboot; the entrypoint then self-heals sshd. start_arg__extend = ["--security-opt=no-new-privileges", "--workdir=/", "--restart=unless-stopped"] # Same forwarding as the docker template; see comment there. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL"] # Same first-boot seed as the docker template; see comment there. post_host_create_command__extend = ["/usr/local/bin/default-workspace-template-seed"] # Install the outer-VM boot unit so the agent recovers after an EC2 reboot even @@ -594,12 +595,12 @@ build_arg__extend = ["--file=Dockerfile", "."] # pre-baked container, which already carries this policy. start_arg__extend = ["--restart=unless-stopped"] # Forward the LiteLLM-minted ANTHROPIC creds + the minds-side MNGR_PREFIX -# + an optional GH_TOKEN from minds' subprocess env onto the host's +# from minds' subprocess env onto the host's # ``/mngr/env``. minds writes these into the subprocess env it hands # ``mngr create`` (see ``run_mngr_create`` in # ``vendor/mngr/apps/minds/imbue/minds/desktop_client/agent_creator.py``) # so the host actually has them at the moment ``--pass-host-env`` reads. -pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "MNGR_PREFIX", "GH_TOKEN"] +pass_host_env__extend = ["ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "MNGR_PREFIX"] # Seed /mngr/code from the baked image on first boot (idempotent warm-boot # fast path). Runs after create_host for both paths; on the fast path the # code is already present so default-workspace-template-seed is a no-op. Same as the vultr/ovh diff --git a/CLAUDE.md b/CLAUDE.md index 2962ea066..7867ad980 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -12,7 +12,7 @@ IF YOU FAIL TO FOLLOW ONE, YOU MUST EXPLICITLY CALL THAT OUT IN YOUR RESPONSE. - NEVER amend commits or rebase--always create new commits. - If you ever need to work with another *git* repo that is *outside* of this monorepo as a read-only dependency, you should do so by adding a git subtree under `vendor/`. - If you need to *actively develop* against an external repo (e.g. `mngr`), check out a standalone clone of it under `.external_worktrees//`. This directory is gitignored so the external clones don't pollute the monorepo. The branch in the external clone should mirror the branch you're on in this monorepo. -- This project uses a CLI ticket system (`tk`) for task management. Run `tk help` when you need to use it. Tickets live under `runtime/tickets/` (the path is set via the `TICKETS_DIR` env var so tickets ride the `mindsbackup/$MNGR_AGENT_ID` runtime-backup branch). +- This project uses a CLI ticket system (`tk`) for task management. Run `tk help` when you need to use it. Tickets live under `runtime/tickets/` (the path is set via the `TICKETS_DIR` env var so tickets sit with the rest of the workspace's runtime state). - All relative paths in this repo assume cwd = repo root (`/code`). Supervisord runs the services from there; any process started elsewhere (manual launch, subprocess from a different cwd) must either set cwd to the repo root or use absolute paths. State directories live under `runtime//`. - When adding a new web app, do NOT edit `libs/web_server/` -- it's an example placeholder. Use the `build-web-service` skill, which sets up a new lib + service entry + `forward_port.py` registration on its own port. @@ -229,7 +229,7 @@ The upstream is defined in `parent.toml`. # Memory Use Claude's built-in memory system. Your memory directory is `runtime/memory/` (configured via `autoMemoryDirectory` in `.claude/settings.json`). -Memory is gitignored from the main branch but is backed up automatically by the runtime-backup service onto the `mindsbackup/$MNGR_AGENT_ID` branch when `GH_TOKEN` is set, so it survives container loss. +Memory is gitignored from the main branch. When the user has enabled GitHub sync (the `github-sync` skill), the github-sync service ships it -- with the rest of `runtime/` -- to the `runtime-sync` branch of the workspace's private sync repo, so it survives container loss. # Services @@ -243,13 +243,11 @@ See the `edit-services` skill for details. Commit your changes locally. `runtime/` is gitignored from the main branch (it includes `runtime/memory/` for Claude memory and other transient state). -Chat file uploads (files a user attaches to a message) are stored in the top-level `uploads/` directory inside the repo working tree -- NOT under `runtime/`. Uploads can be arbitrarily large and any format, so they don't belong in version-controllable content; `uploads/` is gitignored. Being outside `runtime/`, uploads are NOT carried by the `runtime-backup` orphan branch (which ships only `runtime/`), but the host-level `host-backup` service (a restic snapshot of the whole host dir) does capture them, so uploads still survive container loss. See `libs/host_backup/README.md`. +Chat file uploads (files a user attaches to a message) are stored in the top-level `uploads/` directory inside the repo working tree -- NOT under `runtime/`. Uploads can be arbitrarily large and any format, so they don't belong in version-controllable content; `uploads/` is gitignored. Being outside `runtime/`, uploads are NOT carried by the opt-in GitHub sync (which ships only `runtime/`), but the host-level `host-backup` service (a restic snapshot of the whole host dir) does capture them, so uploads still survive container loss. See `libs/host_backup/README.md`. -A `post-commit` hook installed via `core.hooksPath = /mngr/code/scripts/git_hooks` auto-pushes the active branch to `origin` in the background, but only when `GH_TOKEN` is set in the environment. You do not need to push manually. The hook never blocks the commit; output is captured at `/tmp/post-commit-push.log`. +GitHub sync is opt-in via the `github-sync` skill. When the user has enabled it: a `post-commit` hook auto-pushes the active branch of every checkout to `origin` (the workspace's dedicated private repo) in the background -- you do not need to push manually; the hook never blocks the commit, and output is captured at `/tmp/post-commit-push.log`. The `github-sync` service additionally syncs `runtime/` onto a separate orphan branch (`runtime-sync`) on the same `origin`. See `libs/github_sync/README.md`. -`runtime/` is backed up automatically by the `runtime-backup` service onto a separate orphan branch (`mindsbackup/$MNGR_AGENT_ID`) on the same `origin`, also gated on `GH_TOKEN`. See `libs/runtime_backup/README.md`. - -If `GH_TOKEN` is unset, both auto-pushes silently no-op; commits stay local. +When GitHub sync is not enabled, there is no auto-push and no GitHub remote to push to; commits stay local (the restic `host-backup` still protects the whole host dir). - Don't include auto-generated lockfile churn (`uv.lock`, `package-lock.json`, etc.) in commits unless the change intentionally bumps a dependency. diff --git a/Dockerfile b/Dockerfile index 51a124624..c021cd8ed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,7 @@ COPY pyproject.toml uv.lock /mngr/code/ COPY libs/app_watcher/pyproject.toml /mngr/code/libs/app_watcher/pyproject.toml COPY libs/bootstrap/pyproject.toml /mngr/code/libs/bootstrap/pyproject.toml COPY libs/cloudflare_tunnel/pyproject.toml /mngr/code/libs/cloudflare_tunnel/pyproject.toml -COPY libs/runtime_backup/pyproject.toml /mngr/code/libs/runtime_backup/pyproject.toml +COPY libs/github_sync/pyproject.toml /mngr/code/libs/github_sync/pyproject.toml COPY libs/web_server/pyproject.toml /mngr/code/libs/web_server/pyproject.toml COPY apps/system_interface/pyproject.toml /mngr/code/apps/system_interface/pyproject.toml diff --git a/README.md b/README.md index 16952ea53..9e4d04039 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ mngr create my-workspace main -t local \ - `supervisord.conf` - Supervisord config defining the background services - `libs/bootstrap/` - First-boot setup, then launches supervisord to supervise the services - `vendor/mngr/` - A vendored, mutable copy of mngr. Note that making changes here *will* affect the behavior of the `mngr` command -- `vendor/tk/` - A vendored copy of the [tk](https://github.com/wedow/ticket) ticket tracker. The `ticket` script (also callable as `tk`) manages tickets stored as markdown. We point `TICKETS_DIR` at `runtime/tickets/` (set in `.mngr/settings.toml`'s `host_env`) so tickets are backed up alongside the rest of `runtime/` on the `mindsbackup/$MNGR_AGENT_ID` branch. +- `vendor/tk/` - A vendored copy of the [tk](https://github.com/wedow/ticket) ticket tracker. The `ticket` script (also callable as `tk`) manages tickets stored as markdown. We point `TICKETS_DIR` at `runtime/tickets/` (set in `.mngr/settings.toml`'s `host_env`) so tickets live alongside the rest of `runtime/` (and are covered by the opt-in GitHub sync when the `github-sync` skill has enabled it). ## Create templates diff --git a/apps/system_interface/imbue/system_interface/server.py b/apps/system_interface/imbue/system_interface/server.py index 691e82d94..64456df3c 100644 --- a/apps/system_interface/imbue/system_interface/server.py +++ b/apps/system_interface/imbue/system_interface/server.py @@ -419,7 +419,7 @@ def _interrupt_agent_endpoint(agent_id: str) -> Response: Refuses to interrupt agents carrying the ``is_primary=true`` label: that's the services agent for the workspace, and restarting it would stop the - bootstrap, web, cloudflared, and runtime-backup services. The + bootstrap, web, cloudflared, and other supervised services. The frontend already hides ``is_primary=true`` agents from the visible agent list; this is defense-in-depth for callers that hit the endpoint directly (curl, scripted use, etc.). @@ -952,7 +952,7 @@ def _destroy_agent(agent_id: str) -> Response: Refuses to destroy agents carrying the ``is_primary=true`` label: that's the services agent for the workspace, and destroying it would tear down - the bootstrap, web, cloudflared, and runtime-backup services + the bootstrap, web, cloudflared, and other supervised services along with it. The frontend already hides ``is_primary=true`` agents from the visible agent list; this is defense-in-depth for callers that hit the endpoint directly (curl, scripted use, etc.). diff --git a/changelog/mngr-github-backup-skill.md b/changelog/mngr-github-backup-skill.md new file mode 100644 index 000000000..0ab0851ae --- /dev/null +++ b/changelog/mngr-github-backup-skill.md @@ -0,0 +1,13 @@ +GitHub sync is now opt-in via the new `github-sync` skill, replacing the always-on `runtime-backup` service. + +By default, nothing pushes to GitHub anymore: the `runtime-backup` supervisord service, bootstrap's `runtime/` worktree init (the `mindsbackup/$MNGR_AGENT_ID` orphan branch), the always-installed `core.hooksPath` post-commit auto-push, and every use of `GH_TOKEN` have been removed. `runtime/` is a plain gitignored directory until sync is enabled; the restic `host-backup` service remains the default durability story. + +The new `github-sync` skill enables sync on request: it creates a dedicated PRIVATE GitHub repo through latchkey (no token ever enters the container), points `origin` at it, wires plain `git push` through the latchkey gateway for every checkout (with the per-VPS secondary gateway as an offline fallback), installs the post-commit hook so all agent and worker commits auto-push their branch, and adds a `[program:github-sync]` service that syncs `runtime/` to the stable `runtime-sync` orphan branch every 60s. `libs/runtime_backup` was renamed and extended into `libs/github_sync` (`uv run github-sync run|wire-git|unwire-git|setup-worktree|check-visibility|status`). + +Private-only is enforced: the skill refuses public repos at setup and the service re-verifies visibility every 15 minutes, halting all pushes (service and hook) when the repo is public or unverifiable. + +The skill's permission ask was validated against a live workspace and corrected: creating the repo needs `github-write-all` (the narrower `github-write-repos` covers only existing-repo paths, not `POST /user/repos`), and `github-read-user` is requested so the skill can name the owning account and verify the grants landed. Both requests now go out back-to-back before any GitHub call, so the user approves once and setup runs to completion. + +A workspace recreated from a previously-synced repo self-heals: after the user re-grants the GitHub permissions, the service re-wires git and restores `runtime/` (memory, tickets, transcripts) from the `runtime-sync` branch automatically. Disable is supported (full unwind of the service, hook, and git wiring; the user chooses whether to keep the remote repo). + +Existing workspaces that update-self onto this version simply stop git-syncing `runtime/` until they opt in via the skill; legacy `mindsbackup/*` branches are not migrated. diff --git a/libs/bootstrap/README.md b/libs/bootstrap/README.md index 8eea7fef2..3c70b81ff 100644 --- a/libs/bootstrap/README.md +++ b/libs/bootstrap/README.md @@ -14,22 +14,17 @@ service. `uv run bootstrap` runs, in order: -1. **Global git config** - rewrites `git@`/`ssh://` GitHub remotes to `https://` - and points `core.hooksPath` at `scripts/git_hooks`. (This replaces the old - `git_auth_setup` extra_window, minus the retired `gh auth setup-git`.) -2. **runtime/ worktree init** - ensures `runtime/` is a git worktree of the - per-agent backup branch `mindsbackup/$MNGR_AGENT_ID`, so anything written - under `runtime/` (Claude memory, tickets, transcripts, the - `initial_chat_created` signal, etc.) is replicated off-box by the - `runtime-backup` service. Done before supervisord starts so the - runtime-backup / host-backup services find `runtime/` in place. -3. **CLAUDE_CONFIG_DIR host-env write** - records the services agent's per-agent +1. **Global git config** - rewrites `git@`/`ssh://` GitHub remotes to `https://`. + (`core.hooksPath` is deliberately NOT set here: the post-commit auto-push + hook only becomes active when the opt-in github-sync skill wires it up -- + see `libs/github_sync/README.md`.) +2. **CLAUDE_CONFIG_DIR host-env write** - records the services agent's per-agent Claude config dir in `$MNGR_HOST_DIR/env` so every other agent on the host inherits it. -4. **Initial chat agent** - on first boot only (gated by +3. **Initial chat agent** - on first boot only (gated by `runtime/initial_chat_created`), commits the rsynced workspace onto a clean `main` branch and creates the welcome chat agent (`--message /welcome`). -5. **Launch supervisord** - `exec supervisord -n -c supervisord.conf`. Running +4. **Launch supervisord** - `exec supervisord -n -c supervisord.conf`. Running via `exec` keeps the bootstrap tmux window alive as supervisord and lets the supervised services inherit this shell's already-sourced agent environment. diff --git a/libs/bootstrap/src/bootstrap/manager.py b/libs/bootstrap/src/bootstrap/manager.py index 9a8387f97..2de6024d4 100644 --- a/libs/bootstrap/src/bootstrap/manager.py +++ b/libs/bootstrap/src/bootstrap/manager.py @@ -1,12 +1,10 @@ """Bootstrap: first-boot setup, then launch supervisord. `uv run bootstrap` runs once per container boot (from the `bootstrap` -extra_window). It performs first-boot setup -- global git config, ensuring -runtime/ exists as a git worktree of the per-agent backup branch -(mindsbackup/$MNGR_AGENT_ID), writing CLAUDE_CONFIG_DIR into the host env, -and creating the initial chat agent -- and then -`exec`s the system supervisord in the foreground. supervisord (configured by -supervisord.conf) owns every background service from then on. +extra_window). It performs first-boot setup -- global git config, writing +CLAUDE_CONFIG_DIR into the host env, and creating the initial chat agent -- +and then `exec`s the system supervisord in the foreground. supervisord +(configured by supervisord.conf) owns every background service from then on. Running supervisord via exec keeps the bootstrap tmux window alive as supervisord and lets the supervised services inherit this shell's already- @@ -15,7 +13,6 @@ import json import os -import shutil import subprocess from pathlib import Path @@ -29,13 +26,10 @@ SUPERVISOR_LOG_DIR = Path("/var/log/supervisor") RUNTIME_DIR = Path("runtime") -RUNTIME_PREEXISTING_DIR = Path("runtime.preexisting") -RUNTIME_BACKUP_USER_NAME = "runtime-backup" -RUNTIME_BACKUP_USER_EMAIL = "runtime-backup@mindsbackup.local" # Signal file gating exactly-once creation of the initial chat agent. Lives -# under runtime/ so the runtime-backup service replicates it to the -# mindsbackup/$MNGR_AGENT_ID branch (survives container loss). +# under runtime/, which persists with the container volume (and is synced to +# GitHub when the opt-in github-sync skill has been enabled). INITIAL_CHAT_SIGNAL = RUNTIME_DIR / "initial_chat_created" # Basename (under $MNGR_HOST_DIR) of the file holding the initial chat agent's id, # read by system_interface's welcome_resend to address the resend by id. @@ -47,9 +41,14 @@ _HOST_DIR_ENV_VAR = "MNGR_HOST_DIR" _CLAUDE_CONFIG_DIR_ENV_VAR = "CLAUDE_CONFIG_DIR" -# Global git config the old git_auth_setup extra_window used to apply (minus the -# retired `gh auth setup-git`): rewrite git@ / ssh:// GitHub remotes to https and -# point core.hooksPath at the repo's git_hooks. +# Global git config applied on every boot: rewrite git@ / ssh:// GitHub +# remotes to https (there are no SSH credentials in the container). Note that +# git applies at most one insteadOf rewrite per URL, so this rewrite's output +# is NOT further rewritten by github-sync's latchkey gateway wiring: only +# remotes stored as https://github.com/ URLs (the shape the github-sync skill +# always configures) route through the gateway. +# core.hooksPath is deliberately NOT set here -- the post-commit auto-push +# hook only becomes active when the github-sync skill wires it up. _GIT_GLOBAL_CONFIG_ARGVS = ( ( "config", @@ -65,7 +64,6 @@ "url.https://github.com/.insteadOf", "ssh://git@github.com/", ), - ("config", "--global", "core.hooksPath", "/mngr/code/scripts/git_hooks"), ) @@ -412,7 +410,7 @@ def _maybe_create_initial_chat() -> None: Touches the signal file only on a successful create -- a failed create leaves the signal file absent so the next bootstrap run retries. The user's manually-destroyed initial chat agent is *not* recreated, - because the signal file persists in the runtime-backup branch. + because the signal file persists in runtime/. """ if INITIAL_CHAT_SIGNAL.exists(): logger.debug( @@ -448,12 +446,11 @@ def _bootstrap_init_chat_dir() -> None: def _configure_git_global() -> None: - """Apply the global git config the old git_auth_setup extra_window set. + """Apply the boot-time global git config. - Rewrites git@ / ssh:// GitHub remotes to https and points core.hooksPath at - the repo's git_hooks (see _GIT_GLOBAL_CONFIG_ARGVS). The retired - `gh auth setup-git` step is intentionally dropped. Best-effort: a failure - here should not block the supervisord launch. + Rewrites git@ / ssh:// GitHub remotes to https (see + _GIT_GLOBAL_CONFIG_ARGVS). Best-effort: a failure here should not block + the supervisord launch. """ for argv in _GIT_GLOBAL_CONFIG_ARGVS: result = subprocess.run( @@ -494,239 +491,13 @@ def _exec_supervisord() -> None: os.execvp("supervisord", ["supervisord", "-n", "-c", str(SUPERVISORD_CONF)]) -def _git_noninteractive_env() -> dict[str, str]: - """Environment for bootstrap git calls: never prompt for credentials. - - Git prompts for a username/password on the controlling TTY (bypassing our - captured pipes) when a remote needs auth and no credential is available -- - e.g. the "best-effort" fetch in _init_runtime_worktree against a PRIVATE - origin (a mind created from a private inspiration repo) with no GH_TOKEN. - That prompt blocks bootstrap forever, so supervisord never starts and the - workspace sits on "Loading workspace" indefinitely. GIT_TERMINAL_PROMPT=0 - turns the prompt into a fast failure, which every caller here already - handles (all bootstrap git calls are best-effort by design). - """ - return {**os.environ, "GIT_TERMINAL_PROMPT": "0"} - - -def _git_main(*args: str) -> subprocess.CompletedProcess[str]: - """Run a git command in the main checkout, never raising or prompting.""" - return subprocess.run( - ["git", *args], - capture_output=True, - text=True, - check=False, - env=_git_noninteractive_env(), - ) - - -def _git_runtime(*args: str) -> subprocess.CompletedProcess[str]: - """Run a git command inside the runtime worktree, never raising or prompting.""" - return subprocess.run( - ["git", "-C", str(RUNTIME_DIR), *args], - capture_output=True, - text=True, - check=False, - env=_git_noninteractive_env(), - ) - - -def _restore_preexisting_into_worktree() -> None: - """Move any files from runtime.preexisting/ back into runtime/.""" - if not RUNTIME_PREEXISTING_DIR.exists(): - return - for entry in list(RUNTIME_PREEXISTING_DIR.iterdir()): - target = RUNTIME_DIR / entry.name - if target.exists(): - # Don't clobber what the worktree already has (e.g. a fresh - # .gitignore we just wrote). - continue - shutil.move(str(entry), str(target)) - try: - RUNTIME_PREEXISTING_DIR.rmdir() - except OSError: - logger.warning( - "{} not empty after restore; leaving for inspection", - RUNTIME_PREEXISTING_DIR, - ) - - -def _stage_preexisting_aside() -> None: - """Move runtime/'s contents to runtime.preexisting/ so we can add a worktree. - - Only called when runtime/ exists with files but is not yet a git worktree. - """ - if RUNTIME_PREEXISTING_DIR.exists(): - # Stale leftover from a prior failed init -- clear it. - shutil.rmtree(RUNTIME_PREEXISTING_DIR) - shutil.move(str(RUNTIME_DIR), str(RUNTIME_PREEXISTING_DIR)) - - -def _runtime_dir_has_files() -> bool: - """Return True if runtime/ exists and contains anything.""" - if not RUNTIME_DIR.exists(): - return False - return any(RUNTIME_DIR.iterdir()) - - -def _create_orphan_runtime_worktree(branch: str) -> subprocess.CompletedProcess[str]: - """Add runtime/ as a worktree on a fresh orphan branch, git-version-agnostically. - - `git worktree add --orphan` only exists in git >= 2.42, but the Lima - provider's Debian 12 base ships git 2.39. So build the orphan branch with - plumbing that has worked for ages -- a parentless commit on the empty tree -- - then do a normal `git worktree add` for it. Returns the final worktree-add - CompletedProcess; if an earlier plumbing step fails, returns that failing - CompletedProcess so the caller's existing error handling fires. - """ - empty_tree = _git_main("hash-object", "-w", "-t", "tree", "/dev/null") - if empty_tree.returncode != 0: - return empty_tree - # Commit identity is passed via -c because the container may have no global - # git identity yet, and commit-tree refuses to run without one. - orphan_commit = _git_main( - "-c", - f"user.name={RUNTIME_BACKUP_USER_NAME}", - "-c", - f"user.email={RUNTIME_BACKUP_USER_EMAIL}", - "commit-tree", - empty_tree.stdout.strip(), - "-m", - "runtime backup: init", - ) - if orphan_commit.returncode != 0: - return orphan_commit - branch_result = _git_main("branch", branch, orphan_commit.stdout.strip()) - if branch_result.returncode != 0: - return branch_result - return _git_main("worktree", "add", str(RUNTIME_DIR), branch) - - -def _init_runtime_worktree() -> None: - """One-time setup of runtime/ as a worktree of mindsbackup/$MNGR_AGENT_ID. - - Best-effort: logs and returns rather than raising on any failure, so a - transient git problem does not prevent other services from starting. - """ - agent_id = os.environ.get("MNGR_AGENT_ID") - if not agent_id: - logger.warning( - "MNGR_AGENT_ID is unset; skipping runtime worktree init " - "(runtime-backup service will also no-op)" - ) - return - - branch = f"mindsbackup/{agent_id}" - - if (RUNTIME_DIR / ".git").exists(): - logger.info("runtime/ is already a worktree; skipping init") - # A prior init may have staged runtime/ content aside and been killed - # before restoring it (leaving runtime.preexisting/ behind while the - # worktree itself already exists). Recover that content now rather - # than stranding it. _restore_preexisting_into_worktree no-ops when - # runtime.preexisting/ is absent, which is the common case. - _restore_preexisting_into_worktree() - return - - logger.info("Initializing runtime worktree on branch {}", branch) - - # Best-effort fetch so we can detect a pre-existing remote branch (e.g. - # restored after a container restart on the same agent id). - fetch_result = _git_main("fetch", "origin", branch) - remote_ref = f"origin/{branch}" - has_remote = ( - fetch_result.returncode == 0 - and _git_main("rev-parse", "--verify", remote_ref).returncode == 0 - ) - - staged_aside = False - if _runtime_dir_has_files(): - logger.warning( - "runtime/ already has files; staging them aside before adding the worktree" - ) - _stage_preexisting_aside() - staged_aside = True - - if has_remote: - result = _git_main( - "worktree", "add", "-B", branch, str(RUNTIME_DIR), remote_ref - ) - else: - result = _create_orphan_runtime_worktree(branch) - - if result.returncode != 0: - logger.error( - "git worktree add failed (rc={}): {}", - result.returncode, - result.stderr.strip(), - ) - # Restore preexisting files so other services don't lose them. - if staged_aside: - if not RUNTIME_DIR.exists(): - shutil.move(str(RUNTIME_PREEXISTING_DIR), str(RUNTIME_DIR)) - else: - _restore_preexisting_into_worktree() - return - - # Configure bot identity for backup commits inside this worktree only. - _git_runtime("config", "user.name", RUNTIME_BACKUP_USER_NAME) - _git_runtime("config", "user.email", RUNTIME_BACKUP_USER_EMAIL) - - if has_remote: - # Make sure the local branch tracks the remote (some git versions - # don't set this automatically with -B + an explicit ref). - _git_runtime("branch", "--set-upstream-to", remote_ref) - else: - # Fresh orphan branch: write the .gitignore for secrets and make an - # initial empty commit so push has something to push. - gitignore = RUNTIME_DIR / ".gitignore" - gitignore.write_text("secrets\n") - _git_runtime("add", ".gitignore") - commit = _git_runtime("commit", "--allow-empty", "-m", "runtime backup: init") - if commit.returncode != 0: - logger.error( - "initial commit failed (rc={}): {}", - commit.returncode, - commit.stderr.strip(), - ) - - # Restore staged-aside content. Calling unconditionally (rather than - # gating on the `staged_aside` flag) also recovers content left by a - # prior init that staged aside but was killed before it could restore. - _restore_preexisting_into_worktree() - - if os.environ.get("GH_TOKEN"): - if has_remote: - push = _git_runtime("push") - else: - push = _git_runtime("push", "--set-upstream", "origin", branch) - if push.returncode != 0: - logger.warning( - "initial push failed (rc={}): {} (runtime-backup service will retry)", - push.returncode, - push.stderr.strip(), - ) - else: - logger.info("No GH_TOKEN; skipping initial push") - - def main() -> None: logger.info("Bootstrap starting: first-boot setup, then supervisord") - # Apply the global git config (https rewrites + repo hooksPath) before any - # service or agent runs git. Replaces the old git_auth_setup extra_window. + # Apply the global git config (https rewrites) before any service or + # agent runs git. _configure_git_global() - # Restore runtime/ FIRST so the initial_chat_created signal file (which - # lives inside the worktree and is replicated to mindsbackup/$MNGR_AGENT_ID - # by the runtime-backup service) is in place before we decide whether to - # create the initial chat agent. Without this ordering, every container - # restart sees an empty runtime/, treats the boot as first-ever, and - # re-creates the welcome chat agent (and auto-commits any uncommitted - # work_dir state). This must also happen before supervisord starts the - # runtime-backup / host-backup services that write into runtime/. - _init_runtime_worktree() - _bootstrap_init_chat_dir() # Make sure supervisord's log directory exists, then hand off: replace this diff --git a/libs/bootstrap/src/bootstrap/manager_test.py b/libs/bootstrap/src/bootstrap/manager_test.py index 2a2e2991e..6dc14679a 100644 --- a/libs/bootstrap/src/bootstrap/manager_test.py +++ b/libs/bootstrap/src/bootstrap/manager_test.py @@ -16,7 +16,6 @@ INITIAL_CHAT_AGENT_ID_FILENAME, _build_create_chat_command, _configure_git_global, - _create_orphan_runtime_worktree, _ensure_host_claude_config_dir, _format_env_file, _initialize_workspace_main_branch, @@ -32,12 +31,14 @@ # --- _configure_git_global --- -def test_configure_git_global_sets_insteadof_and_hookspath( +def test_configure_git_global_sets_insteadof_but_not_hookspath( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: # Isolate the global git config to a tmp file so the test does not touch the # developer's real ~/.gitconfig. _configure_git_global should set both - # insteadOf rewrites (git@ and ssh://) plus core.hooksPath. + # insteadOf rewrites (git@ and ssh://). core.hooksPath must NOT be set: + # the post-commit auto-push hook only becomes active when the opt-in + # github-sync skill wires it up. gitconfig = tmp_path / ".gitconfig" monkeypatch.setenv("HOME", str(tmp_path)) monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(gitconfig)) @@ -59,7 +60,7 @@ def test_configure_git_global_sets_insteadof_and_hookspath( text=True, check=False, ).stdout.strip() - assert hooks_path == "/mngr/code/scripts/git_hooks" + assert hooks_path == "" # --- Env-file helpers --- @@ -511,35 +512,3 @@ def test_initialize_workspace_main_branch_is_idempotent_on_clean_main( _initialize_workspace_main_branch() branch = _git_in(work_dir, "branch", "--show-current").stdout.strip() assert branch == "main" - - -# --- _create_orphan_runtime_worktree --- - - -def test_create_orphan_runtime_worktree_creates_empty_orphan_branch( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - """The helper must add runtime/ as a worktree on a brand-new orphan branch - (no parent, empty tree) using only plumbing that works on old git -- no - `git worktree add --orphan` (git >= 2.42), which Lima's Debian 12 lacks.""" - repo = tmp_path / "repo" - repo.mkdir() - _git_in(repo, "init", "-q", "--initial-branch=main") - _git_in(repo, "config", "user.email", "seed@test.local") - _git_in(repo, "config", "user.name", "seed") - (repo / "README.md").write_text("seed\n") - _git_in(repo, "add", "-A") - _git_in(repo, "commit", "-qm", "seed") - - monkeypatch.chdir(repo) - result = _create_orphan_runtime_worktree("mindsbackup/agent-test") - assert result.returncode == 0, result.stderr - - runtime = repo / "runtime" - assert (runtime / ".git").exists() - # The branch is an orphan: its tip commit has no parents. - parents = _git_in(runtime, "rev-list", "--parents", "-1", "HEAD").stdout.split() - assert len(parents) == 1 # just the commit sha, no parent sha - # Nothing is tracked yet (empty tree); the worktree has no repo content. - assert _git_in(runtime, "ls-files").stdout.strip() == "" - assert not (runtime / "README.md").exists() diff --git a/libs/browser/README.md b/libs/browser/README.md index b53137c69..226af7163 100644 --- a/libs/browser/README.md +++ b/libs/browser/README.md @@ -38,7 +38,7 @@ agent, identified by its `MNGR_AGENT_ID`, or the human). its own persistent Chromium profile under `$MNGR_HOST_DIR/browser-profiles/` (Tier A -- on the workspace volume), so cookies/logins/history come back; Chromium does this itself, we just point `user_data_dir` at a durable dir. A tiny manifest - (`runtime/browser-fleet.json`, Tier B -- git-backed to the mindsbackup branch) + (`runtime/browser-fleet.json`, Tier B -- covered by the opt-in GitHub sync) records which browsers existed and their tab URLs, so even a full rebuild restores the tab list (logged out, since profiles are volume-only). On daemon startup the fleet is restored **eager-sequentially** (one browser at a time, no cold-boot diff --git a/libs/browser/src/browser/manifest.py b/libs/browser/src/browser/manifest.py index 10eff02fa..34bcfc829 100644 --- a/libs/browser/src/browser/manifest.py +++ b/libs/browser/src/browser/manifest.py @@ -6,7 +6,7 @@ and dies with the old container) and NO Chromium profile bytes (cookies/logins/ history live in each browser's persistent ``user_data_dir`` on the workspace volume; see ``session.py``). Because it's tiny JSON it can live under ``runtime/`` and ride -the mindsbackup branch, so even a full container rebuild restores the tab list. +the opt-in GitHub sync, so even a full container rebuild restores the tab list. Pure synchronous file IO (no asyncio here, on purpose): writes are atomic via a temp file + ``os.replace`` so a reader on the next boot sees either the old or the diff --git a/libs/browser/src/browser/session.py b/libs/browser/src/browser/session.py index dca7b0ffb..3b20d5416 100644 --- a/libs/browser/src/browser/session.py +++ b/libs/browser/src/browser/session.py @@ -208,8 +208,8 @@ def _repo_root() -> Path: # Per-browser persistent Chromium profiles (cookies/logins/history) live here, on the # workspace volume under $MNGR_HOST_DIR -- Tier A durability: they survive stop/start # and restart of a single workspace (lost only on a permanent delete). They are NOT -# under runtime/ (which is git-backed to the mindsbackup branch) -- a fat, churny -# profile would bloat that branch. Override the root for tests / alternate layouts. +# under runtime/ (which the opt-in GitHub sync ships to a git branch) -- a fat, +# churny profile would bloat that branch. Override the root for tests / alternate layouts. _PROFILE_ROOT = Path( os.environ.get( "BROWSER_PROFILE_ROOT", diff --git a/libs/github_sync/README.md b/libs/github_sync/README.md new file mode 100644 index 000000000..38a440934 --- /dev/null +++ b/libs/github_sync/README.md @@ -0,0 +1,76 @@ +# github_sync + +Opt-in GitHub sync for a workspace. Nothing in this library is active by +default: the `github-sync` skill enables it for a workspace whose user asks +for it, and the skill (not this library) creates the dedicated **private** +GitHub repo and points `origin` at it. + +Once enabled, three pieces work together: + +1. **The service** (`uv run github-sync run`, supervised as + `[program:github-sync]`): every 60 seconds it commits the contents of + `runtime/` (Claude memory, ticket state, transcripts, app port registry, + etc.) on the stable orphan branch `runtime-sync` -- checked out as a git + worktree at `runtime/` -- and pushes it to `origin`. +2. **The git wiring** (`uv run github-sync wire-git`): global git config that + rewrites `https://github.com/...` remotes to the latchkey gateway's git + proxy and attaches the gateway auth headers, so a plain `git push` works + for every checkout in the container (main repo, worker worktrees, the + runtime worktree). The GitHub credential is injected server-side by the + gateway; no token ever enters the container. The wiring also points + `core.hooksPath` at `scripts/git_hooks`, activating the post-commit + auto-push hook for every checkout. +3. **The post-commit hook** (`scripts/git_hooks/post-commit`, in the repo but + inert until the hooks path is wired): auto-pushes the active branch of any + checkout after each commit, so both main-agent and worker commits land on + the GitHub remote without manual pushes. + +## Behavior + +- Sync is configured iff `github_sync.toml` exists at the repo root (it holds + `repo_url`); the skill writes it. Without it the service idles. +- Tick interval: 60 seconds. Each tick: `git add -A`, commit (only if dirty) + with message `runtime sync: `, push. All `git` + operations target the runtime worktree. +- Pushes go through the latchkey gateway on the user's machine, falling back + to the per-VPS secondary gateway (remote hosts only) when the user's + machine is offline. A failed push is retried on the next tick; `--force` is + never used (the service is the branch's only writer). +- **Private-only enforcement**: the service re-checks the repo's visibility + through latchkey every 15 minutes; pushes are held until the first + confirmed-private answer and halted whenever the repo is confirmed public. + A re-check that fails outright (e.g. the gateway is offline -- in which + case pushes would fail too) keeps the last confirmed answer and is retried + every tick. The post-commit hook consults the same status and skips pushes + during a halt. +- `runtime/secrets` is excluded via the worktree's own `.gitignore` (written + at init), so e.g. the Cloudflare tunnel token never reaches the remote. +- Each tick first clears a stale `index.lock` from the runtime worktree if + one is present and older than the tick interval (a killed git process never + cleans up its own lock, and without this every later `git add` would fail + identically and syncing would stop permanently and silently). +- On any git failure the service logs to stderr and `/tmp/github-sync.log` + and tries again on the next tick. It mirrors machine-readable status to + `/tmp/github-sync-status.json` (read by the hook and the skill). + +## Restoring on a fresh container + +If a workspace is recreated from a previously-synced repo, the synced-in +supervisord config already contains the `[program:github-sync]` block, but +the latchkey permissions and the container-local git wiring/worktree do not +carry over. The service self-heals: each tick it re-applies the wiring and +retries the worktree init, which fetches origin's `runtime-sync` branch and +materializes the prior `runtime/` state (memory, tickets, transcripts) as +soon as the user re-grants the GitHub permissions (the github-sync skill +walks them through it). + +## CLI + +``` +uv run github-sync run # the service loop (supervisord) +uv run github-sync wire-git # apply gateway git config + hooks path +uv run github-sync unwire-git # remove them (disable path) +uv run github-sync setup-worktree # create/restore the runtime/ worktree +uv run github-sync check-visibility # print private/public/unknown; rc!=0 unless private +uv run github-sync status # config + latest service status as JSON +``` diff --git a/libs/github_sync/pyproject.toml b/libs/github_sync/pyproject.toml new file mode 100644 index 000000000..de2cfd56d --- /dev/null +++ b/libs/github_sync/pyproject.toml @@ -0,0 +1,20 @@ +[project] +name = "github-sync" +version = "0.1.0" +description = "Opt-in service that syncs runtime/ (and, via a post-commit hook, every branch) to a private GitHub repo through the latchkey gateway" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "click>=8.0", + "loguru>=0.7.0", +] + +[project.scripts] +github-sync = "github_sync.cli:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/github_sync"] diff --git a/libs/runtime_backup/src/runtime_backup/__init__.py b/libs/github_sync/src/github_sync/__init__.py similarity index 100% rename from libs/runtime_backup/src/runtime_backup/__init__.py rename to libs/github_sync/src/github_sync/__init__.py diff --git a/libs/github_sync/src/github_sync/cli.py b/libs/github_sync/src/github_sync/cli.py new file mode 100644 index 000000000..67fc67f91 --- /dev/null +++ b/libs/github_sync/src/github_sync/cli.py @@ -0,0 +1,95 @@ +"""CLI for github-sync: the service loop plus the helpers the skill drives. + +`uv run github-sync run` is what the [program:github-sync] supervisord block +executes; the remaining subcommands are one-shot steps invoked by the +github-sync skill during enable / disable / status. +""" + +import json + +import click + +from github_sync import runner +from github_sync.config import GithubSyncConfigError, load_repo_url +from github_sync.visibility import VISIBILITY_PRIVATE, check_repo_visibility +from github_sync.wiring import apply_git_wiring, remove_git_wiring +from github_sync.worktree import init_runtime_worktree, is_runtime_worktree + + +@click.group() +def main() -> None: + """Manage the opt-in GitHub sync for this workspace.""" + + +@main.command() +def run() -> None: + """Run the periodic runtime/ sync service loop (used by supervisord).""" + runner.run_forever() + + +@main.command("wire-git") +def wire_git() -> None: + """Route git's GitHub access through the latchkey gateway (global config).""" + if not apply_git_wiring(): + raise SystemExit(1) + click.echo("wired") + + +@main.command("unwire-git") +def unwire_git() -> None: + """Remove the gateway git config and the hooks path (disable path).""" + remove_git_wiring() + click.echo("unwired") + + +@main.command("setup-worktree") +def setup_worktree() -> None: + """Create runtime/ as a worktree of runtime-sync, restoring from origin if it exists there.""" + if not init_runtime_worktree(): + raise SystemExit(1) + click.echo("ready") + + +@main.command("check-visibility") +def check_visibility() -> None: + """Print the sync repo's visibility; exits nonzero unless confirmed private.""" + try: + repo_url = load_repo_url() + except GithubSyncConfigError as e: + raise click.ClickException(str(e)) from e + if repo_url is None: + raise click.ClickException("sync is not configured (github_sync.toml missing)") + visibility = check_repo_visibility(repo_url) + click.echo(visibility) + if visibility != VISIBILITY_PRIVATE: + raise SystemExit(1) + + +@main.command() +def status() -> None: + """Print the sync configuration and the service's latest status as JSON.""" + try: + repo_url = load_repo_url() + config_error = None + except GithubSyncConfigError as e: + repo_url = None + config_error = str(e) + service_status = None + status_path = runner.status_file_path() + if status_path.exists(): + try: + service_status = json.loads(status_path.read_text()) + except (OSError, json.JSONDecodeError) as e: + service_status = {"error": f"unreadable status file: {e}"} + payload = { + "is_configured": repo_url is not None, + "repo_url": repo_url, + "config_error": config_error, + "is_runtime_worktree": is_runtime_worktree(), + "service": service_status, + } + click.echo(json.dumps(payload, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/libs/github_sync/src/github_sync/config.py b/libs/github_sync/src/github_sync/config.py new file mode 100644 index 000000000..cfbdfd306 --- /dev/null +++ b/libs/github_sync/src/github_sync/config.py @@ -0,0 +1,114 @@ +"""Shared configuration and constants for the opt-in GitHub sync. + +GitHub sync is disabled by default. The github-sync skill enables it by +writing github_sync.toml at the repo root (the presence of that file is the +"sync is configured" marker), wiring git up to push through the latchkey +gateway, and adding the [program:github-sync] supervisord service. +""" + +import os +import tomllib +from pathlib import Path + +# All relative paths assume cwd = repo root (/mngr/code), matching +# supervisord's `directory=` and the other template services. +RUNTIME_DIR = Path("runtime") +CONFIG_PATH = Path("github_sync.toml") + +# The stable branch that runtime/ state is synced to. The sync repo is a +# dedicated private repo per workspace, so no per-agent namespacing is needed. +SYNC_BRANCH = "runtime-sync" + +GITHUB_URL_PREFIX = "https://github.com/" + +# Env vars injected by mngr_latchkey's prepare_agent_latchkey for every agent. +# The secondary gateway (remote VPS hosts only) keeps working when the user's +# own machine -- which runs the primary gateway -- is offline. +ENV_GATEWAY = "LATCHKEY_GATEWAY" +ENV_GATEWAY_SECONDARY = "LATCHKEY_GATEWAY_SECONDARY" +ENV_GATEWAY_PASSWORD = "LATCHKEY_GATEWAY_PASSWORD" +ENV_GATEWAY_PERMISSIONS_OVERRIDE = "LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE" + + +class GithubSyncError(Exception): + """Base error for the github_sync library.""" + + +class GithubSyncConfigError(GithubSyncError, ValueError): + """Raised when github_sync.toml exists but is not a valid sync config.""" + + +def load_repo_url() -> str | None: + """Return the sync repo URL from github_sync.toml, or None when sync is not configured. + + Raises GithubSyncConfigError when the file exists but is malformed: a + skill-authored config problem should be loud, not silently ignored. + """ + if not CONFIG_PATH.exists(): + return None + try: + raw_text = CONFIG_PATH.read_text() + except OSError as e: + raise GithubSyncConfigError(f"Cannot read {CONFIG_PATH}: {e}") from e + try: + data = tomllib.loads(raw_text) + except tomllib.TOMLDecodeError as e: + raise GithubSyncConfigError(f"Cannot parse {CONFIG_PATH}: {e}") from e + repo_url = data.get("repo_url") + if not isinstance(repo_url, str) or not repo_url.startswith(GITHUB_URL_PREFIX): + raise GithubSyncConfigError( + f"{CONFIG_PATH} must set repo_url to an " + f"{GITHUB_URL_PREFIX}/ URL, got {repo_url!r}" + ) + # Strip any trailing slash before the .git suffix so a URL written as + # ".../repo.git/" also normalizes fully (same order as the post-commit + # hook's sed normalization). + normalized_url = repo_url.rstrip("/").removesuffix(".git") + # Validate the full / shape here, at the single load point, + # so a malformed URL surfaces as a GithubSyncConfigError that every caller + # already handles -- rather than escaping later from + # parse_owner_and_name inside the service tick and crash-looping it. + parse_owner_and_name(normalized_url) + return normalized_url + + +def parse_owner_and_name(repo_url: str) -> tuple[str, str]: + """Split a GitHub repo URL into (owner, name). + + Raises GithubSyncConfigError when the URL is not of the form + https://github.com//. + """ + path = repo_url.removeprefix(GITHUB_URL_PREFIX).removesuffix(".git") + parts = [part for part in path.split("/") if part] + if len(parts) != 2: + raise GithubSyncConfigError( + f"Repo URL must be {GITHUB_URL_PREFIX}/, got {repo_url!r}" + ) + return parts[0], parts[1] + + +def get_gateway_url() -> str | None: + """The primary latchkey gateway URL (no trailing slash), or None if unset.""" + value = os.environ.get(ENV_GATEWAY, "").rstrip("/") + return value or None + + +def get_secondary_gateway_url() -> str | None: + """The per-VPS backup gateway URL (no trailing slash), or None if unset.""" + value = os.environ.get(ENV_GATEWAY_SECONDARY, "").rstrip("/") + return value or None + + +def get_gateway_password() -> str | None: + value = os.environ.get(ENV_GATEWAY_PASSWORD, "") + return value or None + + +def get_gateway_permissions_override() -> str | None: + value = os.environ.get(ENV_GATEWAY_PERMISSIONS_OVERRIDE, "") + return value or None + + +def proxied_url(gateway_url: str, target_url: str) -> str: + """The gateway's git smart-HTTP proxy URL for a GitHub target URL.""" + return f"{gateway_url}/gateway/{target_url}" diff --git a/libs/github_sync/src/github_sync/config_test.py b/libs/github_sync/src/github_sync/config_test.py new file mode 100644 index 000000000..d7baf8223 --- /dev/null +++ b/libs/github_sync/src/github_sync/config_test.py @@ -0,0 +1,126 @@ +"""Unit tests for github_sync config parsing and gateway env helpers.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from github_sync.config import ( + GithubSyncConfigError, + get_gateway_password, + get_gateway_url, + get_secondary_gateway_url, + load_repo_url, + parse_owner_and_name, + proxied_url, +) + + +def test_load_repo_url_returns_none_when_unconfigured( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + assert load_repo_url() is None + + +def test_load_repo_url_normalizes_git_suffix_and_trailing_slash( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + config_path = tmp_path / "github_sync.toml" + config_path.write_text( + 'repo_url = "https://github.com/some-user/my-workspace.git"\n' + ) + assert load_repo_url() == "https://github.com/some-user/my-workspace" + # A .git suffix followed by a trailing slash must also normalize fully + # (the slash has to come off before the suffix). + config_path.write_text( + 'repo_url = "https://github.com/some-user/my-workspace.git/"\n' + ) + assert load_repo_url() == "https://github.com/some-user/my-workspace" + + +def test_load_repo_url_raises_on_malformed_toml( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text("repo_url = [not toml") + with pytest.raises(GithubSyncConfigError): + load_repo_url() + + +def test_load_repo_url_raises_on_non_github_url( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text( + 'repo_url = "https://gitlab.com/user/repo"\n' + ) + with pytest.raises(GithubSyncConfigError): + load_repo_url() + + +def test_load_repo_url_raises_when_repo_url_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text('other_key = "value"\n') + with pytest.raises(GithubSyncConfigError): + load_repo_url() + + +def test_load_repo_url_raises_when_owner_or_repo_segment_missing( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # A github.com URL without the full / shape must fail at load + # time (where callers handle GithubSyncConfigError), not later inside the + # service tick via parse_owner_and_name. + monkeypatch.chdir(tmp_path) + config_path = tmp_path / "github_sync.toml" + config_path.write_text('repo_url = "https://github.com/just-an-owner"\n') + with pytest.raises(GithubSyncConfigError): + load_repo_url() + config_path.write_text('repo_url = "https://github.com/owner/repo/extra"\n') + with pytest.raises(GithubSyncConfigError): + load_repo_url() + + +def test_parse_owner_and_name_splits_url() -> None: + assert parse_owner_and_name("https://github.com/some-user/my-repo") == ( + "some-user", + "my-repo", + ) + assert parse_owner_and_name("https://github.com/some-user/my-repo.git") == ( + "some-user", + "my-repo", + ) + + +def test_parse_owner_and_name_rejects_malformed_url() -> None: + with pytest.raises(GithubSyncConfigError): + parse_owner_and_name("https://github.com/just-an-owner") + with pytest.raises(GithubSyncConfigError): + parse_owner_and_name("https://github.com/a/b/c") + + +def test_proxied_url_composes_gateway_git_proxy() -> None: + assert ( + proxied_url("http://127.0.0.1:39000", "https://github.com/u/r.git") + == "http://127.0.0.1:39000/gateway/https://github.com/u/r.git" + ) + + +def test_gateway_env_getters_strip_and_default( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("LATCHKEY_GATEWAY", "http://127.0.0.1:39000/") + assert get_gateway_url() == "http://127.0.0.1:39000" + monkeypatch.delenv("LATCHKEY_GATEWAY", raising=False) + assert get_gateway_url() is None + monkeypatch.setenv("LATCHKEY_GATEWAY_SECONDARY", "") + assert get_secondary_gateway_url() is None + monkeypatch.delenv("LATCHKEY_GATEWAY_PASSWORD", raising=False) + assert get_gateway_password() is None + monkeypatch.setenv("LATCHKEY_GATEWAY_PASSWORD", "pw-123") + assert get_gateway_password() == "pw-123" diff --git a/libs/github_sync/src/github_sync/conftest.py b/libs/github_sync/src/github_sync/conftest.py new file mode 100644 index 000000000..0fa1b425d --- /dev/null +++ b/libs/github_sync/src/github_sync/conftest.py @@ -0,0 +1,44 @@ +"""Fixtures shared by the github_sync test files.""" + +from __future__ import annotations + +import os +from pathlib import Path + +import pytest + + +@pytest.fixture +def isolated_git_and_gateway_env( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> Path: + """Isolate global git config and clear all latchkey/gateway env vars. + + Keeps tests from ever touching the developer's real ~/.gitconfig (wiring + writes global config) and from picking up a real gateway from the + environment. Returns the isolated global gitconfig path. + """ + gitconfig = tmp_path / "gitconfig" + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv("GIT_CONFIG_GLOBAL", str(gitconfig)) + for name in ( + "LATCHKEY_GATEWAY", + "LATCHKEY_GATEWAY_SECONDARY", + "LATCHKEY_GATEWAY_PASSWORD", + "LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE", + ): + monkeypatch.delenv(name, raising=False) + # Steer subprocesses away from any inherited status-file override. + monkeypatch.setenv( + "GITHUB_SYNC_STATUS_FILE", str(tmp_path / "github-sync-status.json") + ) + return gitconfig + + +@pytest.fixture +def fake_latchkey_bin(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path: + """A directory prepended to PATH where tests install a fake `latchkey`.""" + bin_dir = tmp_path / "fakebin" + bin_dir.mkdir(parents=True, exist_ok=True) + monkeypatch.setenv("PATH", f"{bin_dir}{os.pathsep}{os.environ.get('PATH', '')}") + return bin_dir diff --git a/libs/github_sync/src/github_sync/post_commit_hook_test.py b/libs/github_sync/src/github_sync/post_commit_hook_test.py new file mode 100644 index 000000000..b8cff3c74 --- /dev/null +++ b/libs/github_sync/src/github_sync/post_commit_hook_test.py @@ -0,0 +1,128 @@ +"""Tests for the scripts/git_hooks/post-commit auto-push hook. + +The hook is the third piece of the opt-in GitHub sync (see the README): it +auto-pushes the committed branch of any checkout, but only when sync is +configured, and never while the github-sync service has halted pushes because +the sync repo is not confirmed private. The halt is the security-critical +behavior, so it gets a direct regression test (an earlier version used +jq's `//` operator, which turned an explicit ``"is_push_allowed": false`` +into ``true`` and silently disabled the halt). +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import time +from pathlib import Path + +import pytest + +from github_sync.testing import init_repo_with_origin + +_HOOK_DIR = Path(__file__).parents[4] / "scripts" / "git_hooks" + +_PUSH_WAIT_SECONDS = 20.0 + + +def _commit_with_hook( + repo: Path, tmp_path: Path, is_push_allowed: bool | None, is_configured: bool = True +) -> Path: + """Make a hook-firing commit in `repo`; returns the hook log path. + + `is_push_allowed=None` means no status file (service has not reported yet). + """ + config_path = tmp_path / "github_sync.toml" + if is_configured: + config_path.write_text( + 'repo_url = "https://github.com/some-user/my-workspace"\n' + ) + status_path = tmp_path / "github-sync-status.json" + if is_push_allowed is not None: + status_path.write_text(json.dumps({"is_push_allowed": is_push_allowed})) + log_path = tmp_path / "post-commit-push.log" + + subprocess.run( + ["git", "-C", str(repo), "config", "core.hooksPath", str(_HOOK_DIR)], + check=True, + capture_output=True, + ) + (repo / "change.txt").write_text("change\n") + subprocess.run( + ["git", "-C", str(repo), "add", "-A"], check=True, capture_output=True + ) + hook_env = { + **os.environ, + # Never let a push attempt block on a TTY credential prompt. + "GIT_TERMINAL_PROMPT": "0", + "GITHUB_SYNC_CONFIG_FILE": str(config_path), + "GITHUB_SYNC_STATUS_FILE": str(status_path), + "GITHUB_SYNC_HOOK_LOG_FILE": str(log_path), + } + subprocess.run( + ["git", "-C", str(repo), "commit", "-qm", "change"], + check=True, + capture_output=True, + env=hook_env, + ) + return log_path + + +def _origin_has_branch(origin: Path, branch: str) -> bool: + result = subprocess.run( + ["git", "-C", str(origin), "rev-parse", "--verify", branch], + capture_output=True, + text=True, + check=False, + ) + return result.returncode == 0 + + +def test_hook_pushes_committed_branch_when_push_allowed( + tmp_path: Path, isolated_git_and_gateway_env: Path +) -> None: + """With sync configured and the repo confirmed private, a commit's branch + lands on origin (the push runs in the background, so poll for it).""" + main, origin = init_repo_with_origin(tmp_path) + + _commit_with_hook(main, tmp_path, is_push_allowed=True) + + deadline = time.monotonic() + _PUSH_WAIT_SECONDS + while not _origin_has_branch(origin, "main"): + assert time.monotonic() < deadline, "hook never pushed the branch" + time.sleep(0.1) + + +def test_hook_skips_push_during_visibility_halt( + tmp_path: Path, isolated_git_and_gateway_env: Path +) -> None: + """Regression: an explicit `"is_push_allowed": false` must block the push + (jq's `//` operator once turned it into `true`). The halt path is fully + synchronous -- no background push job is spawned -- so asserting right + after the commit is race-free.""" + if shutil.which("jq") is None: + pytest.skip("jq is not installed; the hook's halt check requires it") + main, origin = init_repo_with_origin(tmp_path) + + log_path = _commit_with_hook(main, tmp_path, is_push_allowed=False) + + assert not _origin_has_branch(origin, "main") + assert "skipping push of main: sync repo not confirmed private" in ( + log_path.read_text() + ) + + +def test_hook_does_nothing_when_sync_not_configured( + tmp_path: Path, isolated_git_and_gateway_env: Path +) -> None: + """Without github_sync.toml the hook must exit before doing anything.""" + main, origin = init_repo_with_origin(tmp_path) + + log_path = _commit_with_hook( + main, tmp_path, is_push_allowed=None, is_configured=False + ) + + assert not _origin_has_branch(origin, "main") + assert not log_path.exists() diff --git a/libs/github_sync/src/github_sync/runner.py b/libs/github_sync/src/github_sync/runner.py new file mode 100644 index 000000000..9253ebaf6 --- /dev/null +++ b/libs/github_sync/src/github_sync/runner.py @@ -0,0 +1,380 @@ +"""GitHub sync service loop. + +Polls runtime/ every TICK_INTERVAL_SECONDS; if there are uncommitted changes, +makes a sync commit on the runtime-sync branch checked out at runtime/ and +pushes it to origin through the latchkey gateway. Periodically re-verifies +that the sync repo is still private: pushes are held until visibility is +first confirmed private and halted whenever the repo is confirmed public. A +re-check that fails outright keeps the last confirmed answer and is retried +every tick (see _refresh_visibility). + +The service only exists when the github-sync skill has enabled sync (it adds +the [program:github-sync] block to supervisord.conf). The skill normally also +creates the runtime/ worktree; if it is missing (e.g. a workspace recreated +from a previously-synced repo), each tick retries the worktree init, which +restores runtime/ from origin once the latchkey GitHub permissions have been +re-granted. +""" + +import json +import os +import time +from datetime import datetime, timezone +from pathlib import Path + +from loguru import logger + +from github_sync.config import ( + SYNC_BRANCH, + GithubSyncConfigError, + get_gateway_password, + get_secondary_gateway_url, + load_repo_url, + proxied_url, +) +from github_sync.visibility import ( + VISIBILITY_PRIVATE, + VISIBILITY_PUBLIC, + VISIBILITY_UNKNOWN, + check_repo_visibility, +) +from github_sync.wiring import PASSWORD_HEADER, apply_git_wiring +from github_sync.worktree import ( + git_runtime, + init_runtime_worktree, + is_runtime_worktree, +) + +TICK_INTERVAL_SECONDS = 60 +# How long a confirmed visibility answer stays fresh before it is re-checked. +# Failed checks are retried every tick instead. +VISIBILITY_CHECK_INTERVAL_SECONDS = 900 +LOG_FILE = Path("/tmp/github-sync.log") +# Machine-readable status mirror, read by the post-commit hook (to respect a +# visibility halt) and by the github-sync skill's status report. Lives in /tmp +# deliberately: it is per-boot state, not something to sync. The default path +# is what scripts/git_hooks/post-commit reads. +DEFAULT_STATUS_FILE = Path("/tmp/github-sync-status.json") + +# Minimum age before an index.lock is treated as stale and removed. A real git +# operation on the small runtime/ tree holds the lock for well under a second, +# so a lock older than this cannot belong to a live operation. Set to the tick +# interval so a lock skipped as possibly-live on one tick is guaranteed old +# enough to clear on the next. +STALE_LOCK_MIN_AGE_SECONDS = TICK_INTERVAL_SECONDS + + +class _SyncState: + """Mutable sync status carried across ticks and mirrored to the status file.""" + + def __init__(self) -> None: + self.repo_url: str | None = None + self.visibility: str = VISIBILITY_UNKNOWN + self.visibility_checked_at: datetime | None = None + self.last_push_ok: bool | None = None + self.last_push_at: datetime | None = None + self.last_error: str | None = None + + @property + def is_push_allowed(self) -> bool: + return self.visibility == VISIBILITY_PRIVATE + + +def status_file_path() -> Path: + """The status-mirror path; overridable via env so tests stay isolated.""" + override = os.environ.get("GITHUB_SYNC_STATUS_FILE", "") + return Path(override) if override else DEFAULT_STATUS_FILE + + +def _now_utc() -> datetime: + return datetime.now(timezone.utc) + + +def _iso_or_none(moment: datetime | None) -> str | None: + """ISO-8601 with a trailing Z (e.g. 2026-05-06T17:42:13Z), or None.""" + if moment is None: + return None + return moment.isoformat(timespec="seconds").replace("+00:00", "Z") + + +def _clear_stale_index_lock() -> None: + """Remove a stale ``index.lock`` from the runtime worktree, if present. + + github-sync is the only writer of this worktree's git index and its ticks + run strictly sequentially, so a lock here is normally stale -- left behind + by a previous tick's git process that was killed before it could release + the lock (whenever something kills the process mid-commit). Git never + clears such a lock itself, so without this every subsequent ``git add`` + fails identically and syncing stops forever. + + The removal is deliberately *not* unconditional: to stay safe even if the + single-writer assumption is ever violated (a concurrent git process in the + worktree), the lock is only removed once it is older than + ``STALE_LOCK_MIN_AGE_SECONDS``. A live git operation on the small runtime/ + tree holds the lock for well under a second, so an older lock cannot be + live; a genuinely in-progress operation is left untouched and reconsidered + on the next tick. + + The lock path is resolved via ``--absolute-git-dir`` so it is correct + whether runtime/ is a normal repo or (as in production) a linked worktree + with a per-worktree git dir. + """ + git_dir_result = git_runtime("rev-parse", "--absolute-git-dir") + if git_dir_result.returncode != 0: + # runtime/ is not a git repo yet; nothing to clear. + return + lock_path = Path(git_dir_result.stdout.strip()) / "index.lock" + try: + lock_age_seconds = time.time() - lock_path.stat().st_mtime + except OSError: + # No lock present (the common case), or it vanished underneath us. + return + if lock_age_seconds < STALE_LOCK_MIN_AGE_SECONDS: + logger.warning( + "git index.lock at {} is only {:.0f}s old; leaving it in case a " + "git operation is in progress (will reconsider next tick)", + lock_path, + lock_age_seconds, + ) + return + logger.warning( + "Removing stale git index lock at {} ({:.0f}s old)", + lock_path, + lock_age_seconds, + ) + try: + lock_path.unlink() + except OSError as e: + logger.warning("Failed to remove stale index lock at {}: {}", lock_path, e) + + +def _has_uncommitted_changes() -> bool: + """Return True if runtime/ has anything to commit.""" + result = git_runtime("status", "--porcelain") + if result.returncode != 0: + logger.warning( + "git status failed (rc={}): {}", result.returncode, result.stderr.strip() + ) + return False + return bool(result.stdout.strip()) + + +def _commit_runtime_changes() -> None: + """Stage and commit anything new in runtime/ (no-op when clean).""" + add_result = git_runtime("add", "-A") + if add_result.returncode != 0: + logger.warning( + "git add failed (rc={}): {}", + add_result.returncode, + add_result.stderr.strip(), + ) + return + if _has_uncommitted_changes(): + commit_result = git_runtime( + "commit", "-m", f"runtime sync: {_iso_or_none(_now_utc())}" + ) + if commit_result.returncode != 0: + logger.warning( + "git commit failed (rc={}): {}", + commit_result.returncode, + commit_result.stderr.strip(), + ) + + +def _refresh_visibility(state: _SyncState, repo_url: str) -> None: + """Re-check repo visibility when the last confirmed answer has gone stale. + + A completed check updates the state; a failed check (UNKNOWN result) + leaves the previous answer in place and is retried next tick. Transitions + are logged loudly since a repo flipping public is a security condition. + """ + if ( + state.visibility_checked_at is not None + and state.visibility != VISIBILITY_UNKNOWN + ): + age_seconds = (_now_utc() - state.visibility_checked_at).total_seconds() + if age_seconds < VISIBILITY_CHECK_INTERVAL_SECONDS: + return + visibility = check_repo_visibility(repo_url) + if visibility == VISIBILITY_UNKNOWN: + logger.debug("Could not check visibility of {}; will retry", repo_url) + return + if visibility != state.visibility: + if visibility == VISIBILITY_PRIVATE: + logger.info("Sync repo {} confirmed private; pushes enabled", repo_url) + else: + logger.error( + "Sync repo {} is PUBLIC; halting all sync pushes until it is " + "made private again", + repo_url, + ) + state.visibility = visibility + state.visibility_checked_at = _now_utc() + + +def _push_via_secondary_gateway(repo_url: str) -> bool: + """Push runtime-sync through the per-VPS backup gateway, if one exists. + + Used when the primary gateway (on the user's machine) is unreachable. The + secondary takes only the gateway password header -- no permissions + override -- per the latchkey contract. The proxied URL is passed + explicitly so the global insteadOf rewrite (which targets the primary + gateway) does not apply. + """ + secondary_url = get_secondary_gateway_url() + password = get_gateway_password() + if secondary_url is None or password is None: + return False + push_url = proxied_url(secondary_url, f"{repo_url}.git") + result = git_runtime( + "-c", + f"http.extraHeader={PASSWORD_HEADER}: {password}", + "push", + push_url, + f"{SYNC_BRANCH}:{SYNC_BRANCH}", + ) + if result.returncode != 0: + logger.debug( + "secondary-gateway push failed (rc={}): {}", + result.returncode, + result.stderr.strip(), + ) + return result.returncode == 0 + + +def _push_runtime(state: _SyncState, repo_url: str) -> None: + """Push the runtime-sync branch, self-healing wiring drift along the way. + + Always attempts a push (even on a clean tick) so commits whose push failed + earlier still get shipped. The ladder: plain push; re-apply gateway wiring + (the gateway URL embeds a port that can change across restarts) and retry; + set-upstream (heals a lost upstream); secondary gateway. + """ + push_result = git_runtime("push") + if push_result.returncode != 0: + apply_git_wiring() + push_result = git_runtime("push") + if push_result.returncode != 0: + push_result = git_runtime("push", "--set-upstream", "origin", SYNC_BRANCH) + is_pushed = push_result.returncode == 0 + if not is_pushed: + is_pushed = _push_via_secondary_gateway(repo_url) + state.last_push_ok = is_pushed + state.last_push_at = _now_utc() + if is_pushed: + state.last_error = None + else: + state.last_error = push_result.stderr.strip() + logger.warning( + "git push failed (rc={}): {}", + push_result.returncode, + push_result.stderr.strip(), + ) + + +def _write_status(state: _SyncState) -> None: + """Mirror the sync state to the status file for the hook and the skill. + + Written atomically (temp file in the same directory + ``os.replace``) so + the post-commit hook -- which reads this file concurrently to decide + whether a visibility halt is in effect -- can never observe a truncated + file. A truncated read would make jq fail and the hook would default to + allowing the push, silently defeating the halt. + """ + payload = { + "timestamp": _iso_or_none(_now_utc()), + "repo_url": state.repo_url, + "visibility": state.visibility, + "visibility_checked_at": _iso_or_none(state.visibility_checked_at), + "is_push_allowed": state.is_push_allowed, + "last_push_ok": state.last_push_ok, + "last_push_at": _iso_or_none(state.last_push_at), + "last_error": state.last_error, + } + status_path = status_file_path() + tmp_path = status_path.with_name(status_path.name + ".tmp") + try: + tmp_path.write_text(json.dumps(payload, indent=2) + "\n") + os.replace(tmp_path, status_path) + except OSError as e: + logger.warning("Failed to write status file {}: {}", status_path, e) + + +def _do_tick(state: _SyncState) -> None: + """Run one sync tick: commit runtime/ changes, verify visibility, push.""" + try: + repo_url = load_repo_url() + except GithubSyncConfigError as e: + logger.error("Invalid sync config: {}", e) + state.last_error = str(e) + _write_status(state) + return + if repo_url is None: + logger.warning( + "github_sync.toml is missing; sync is not configured (run the " + "github-sync skill), idling" + ) + _write_status(state) + return + if repo_url != state.repo_url: + if state.repo_url is not None: + # The confirmed visibility answer belongs to the previously + # configured repo; a swapped-in repo must earn its own + # confirmed-private answer before any push. + logger.info( + "Sync repo changed from {} to {}; holding pushes until the " + "new repo is confirmed private", + state.repo_url, + repo_url, + ) + state.visibility = VISIBILITY_UNKNOWN + state.visibility_checked_at = None + state.repo_url = repo_url + + if not is_runtime_worktree(): + # Self-healing path for a workspace recreated from a synced repo: the + # wiring and worktree are container-local, so recreate them here once + # the gateway/permissions allow it. + apply_git_wiring() + if not init_runtime_worktree(): + state.last_error = "runtime/ worktree init deferred (origin unreachable)" + _write_status(state) + return + + # Clear any stale index.lock first, so a commit interrupted by something + # killing the process cannot wedge every future tick. + _clear_stale_index_lock() + _commit_runtime_changes() + + _refresh_visibility(state, repo_url) + if state.is_push_allowed: + _push_runtime(state, repo_url) + elif state.visibility == VISIBILITY_PUBLIC: + logger.error( + "Sync repo {} is PUBLIC; refusing to push (make it private again " + "to resume syncing)", + repo_url, + ) + else: + logger.warning( + "Sync repo {} visibility not confirmed yet; holding pushes", repo_url + ) + _write_status(state) + + +def run_forever() -> None: + """Main loop: poll runtime/ on a fixed interval and sync changes.""" + # Tee stderr-bound logs into LOG_FILE so operators can `tail` the file + # across restarts of just this service window. /tmp wipes on container + # restart, which is the intended scope for the debug log. Set up here + # rather than at module import so that merely importing this module + # (e.g. from tests) does not start writing to the log file. + logger.add(LOG_FILE, level="INFO") + + logger.info("Starting github-sync (interval={}s)", TICK_INTERVAL_SECONDS) + apply_git_wiring() + + state = _SyncState() + while True: + _do_tick(state) + time.sleep(TICK_INTERVAL_SECONDS) diff --git a/libs/github_sync/src/github_sync/runner_test.py b/libs/github_sync/src/github_sync/runner_test.py new file mode 100644 index 000000000..98a9fae90 --- /dev/null +++ b/libs/github_sync/src/github_sync/runner_test.py @@ -0,0 +1,410 @@ +"""Unit tests for the github-sync runner. + +Covers the stale-index-lock recovery (an interrupted commit must not wedge +every future tick), the visibility re-check policy (a confirmed answer is +cached for the check interval; a failed re-check keeps the last confirmed +answer and retries next tick), and full ticks against a local bare origin: +commit+push when the repo is confirmed private, and the push halt when it is +public or unverifiable. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import time +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from github_sync.runner import ( + STALE_LOCK_MIN_AGE_SECONDS, + VISIBILITY_CHECK_INTERVAL_SECONDS, + _clear_stale_index_lock, + _do_tick, + _refresh_visibility, + _SyncState, + status_file_path, +) +from github_sync.testing import ( + init_repo, + init_repo_with_origin, + install_fake_latchkey, + run_git, +) +from github_sync.visibility import VISIBILITY_PRIVATE, VISIBILITY_PUBLIC +from github_sync.worktree import init_runtime_worktree, is_runtime_worktree + +_REPO_URL = "https://github.com/some-user/my-workspace" + + +def _age_lock(lock_path: Path) -> None: + """Backdate a lock file's mtime so it counts as stale (past the age guard).""" + old = time.time() - STALE_LOCK_MIN_AGE_SECONDS - 60 + os.utime(lock_path, (old, old)) + + +def _git_out(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True, check=False + ) + + +def test_clear_stale_index_lock_removes_aged_lock_in_linked_worktree( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Faithful to production: runtime/ is a linked worktree, so the lock + lives in the per-worktree git dir, not in a top-level .git/.""" + monkeypatch.chdir(tmp_path) + main = tmp_path / "main" + init_repo(main) + (main / "seed.txt").write_text("seed\n") + _git_out(main, "add", "-A") + _git_out(main, "commit", "-qm", "seed") + # The runner resolves runtime/ relative to its cwd, so the worktree must + # be named exactly "runtime" and sit in tmp_path. + _git_out(main, "worktree", "add", str(tmp_path / "runtime"), "-b", "sync") + + lock_path = main / ".git" / "worktrees" / "runtime" / "index.lock" + lock_path.write_text("") + _age_lock(lock_path) + assert lock_path.exists() + + _clear_stale_index_lock() + + assert not lock_path.exists() + + +def test_clear_stale_index_lock_keeps_fresh_lock( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A recently-created lock might belong to a live git operation, so it + must be left alone rather than yanked out from under that operation.""" + monkeypatch.chdir(tmp_path) + runtime = tmp_path / "runtime" + init_repo(runtime) + lock_path = runtime / ".git" / "index.lock" + lock_path.write_text("") # fresh: mtime is now + + _clear_stale_index_lock() + + assert lock_path.exists() + + +def test_clear_stale_index_lock_noop_when_no_lock_present( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + init_repo(tmp_path / "runtime") + # No lock and no git repo problems -- must simply not raise. + _clear_stale_index_lock() + + +def test_clear_stale_index_lock_noop_when_runtime_not_a_repo( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "runtime").mkdir() + # `git rev-parse` fails; the function must silently return. + _clear_stale_index_lock() + + +def _set_up_synced_workspace( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> tuple[Path, Path]: + """A workspace with a local bare origin, a runtime worktree, and sync config.""" + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + (main / "github_sync.toml").write_text( + 'repo_url = "https://github.com/some-user/my-workspace"\n' + ) + assert init_runtime_worktree() is True + return main, origin + + +def test_do_tick_commits_and_pushes_when_repo_private( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """End to end against a local bare origin: new runtime state is committed + on runtime-sync and pushed once the repo is confirmed private.""" + main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": true}'") + (main / "runtime" / "memory.txt").write_text("important state\n") + state = _SyncState() + + _do_tick(state) + + remote_log = _git_out(origin, "log", "--oneline", "runtime-sync") + assert remote_log.returncode == 0 + assert "runtime sync:" in remote_log.stdout + assert state.last_push_ok is True + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is True + assert status["visibility"] == "private" + assert status["last_push_ok"] is True + + +def test_do_tick_self_heals_stale_index_lock( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """A stale index.lock from a killed prior tick must not wedge syncing: + the next tick clears it and commits the pending runtime state.""" + main, _ = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": true}'") + lock_path = main / ".git" / "worktrees" / "runtime" / "index.lock" + lock_path.write_text("") + _age_lock(lock_path) + (main / "runtime" / "memory.txt").write_text("important state\n") + + _do_tick(_SyncState()) + + log = _git_out(main / "runtime", "log", "--oneline") + assert "runtime sync:" in log.stdout + + +def test_do_tick_halts_pushes_when_repo_public( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """Public repo => commit locally (nothing is lost) but never push.""" + main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": false}'") + (main / "runtime" / "memory.txt").write_text("important state\n") + state = _SyncState() + + _do_tick(state) + + # Committed locally... + local_log = _git_out(main / "runtime", "log", "--oneline") + assert "runtime sync:" in local_log.stdout + # ...but the branch never reached origin. + remote_branch = _git_out(origin, "rev-parse", "--verify", "runtime-sync") + assert remote_branch.returncode != 0 + assert state.is_push_allowed is False + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is False + assert status["visibility"] == "public" + + +def test_do_tick_holds_pushes_while_visibility_unconfirmed( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """An unreachable visibility check (gateway offline) must fail closed.""" + main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, "exit 7") + (main / "runtime" / "memory.txt").write_text("important state\n") + + _do_tick(_SyncState()) + + remote_branch = _git_out(origin, "rev-parse", "--verify", "runtime-sync") + assert remote_branch.returncode != 0 + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is False + assert status["visibility"] == "unknown" + + +def _confirmed_private_state(age_seconds: float) -> _SyncState: + """A state whose private confirmation is `age_seconds` old.""" + state = _SyncState() + state.repo_url = _REPO_URL + state.visibility = VISIBILITY_PRIVATE + state.visibility_checked_at = datetime.now(timezone.utc) - timedelta( + seconds=age_seconds + ) + return state + + +def test_refresh_visibility_skips_recheck_while_confirmed_answer_is_fresh( + isolated_git_and_gateway_env: Path, fake_latchkey_bin: Path +) -> None: + """A confirmed answer younger than the check interval must be trusted + without re-asking GitHub (the fake would answer public, so any re-check + would flip the state and fail the assertion).""" + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": false}'") + state = _confirmed_private_state(age_seconds=1) + checked_at_before = state.visibility_checked_at + + _refresh_visibility(state, _REPO_URL) + + assert state.visibility == VISIBILITY_PRIVATE + assert state.is_push_allowed is True + assert state.visibility_checked_at == checked_at_before + + +def test_refresh_visibility_keeps_last_confirmed_answer_when_recheck_fails( + isolated_git_and_gateway_env: Path, fake_latchkey_bin: Path +) -> None: + """A stale confirmation whose re-check fails outright (gateway offline) + must keep the last confirmed answer -- pushes would fail too in that + state, so halting adds nothing -- and leave checked_at unchanged so the + re-check is retried on the next tick rather than in 15 minutes.""" + install_fake_latchkey(fake_latchkey_bin, "exit 7") + state = _confirmed_private_state(age_seconds=VISIBILITY_CHECK_INTERVAL_SECONDS + 60) + checked_at_before = state.visibility_checked_at + + _refresh_visibility(state, _REPO_URL) + + assert state.visibility == VISIBILITY_PRIVATE + assert state.is_push_allowed is True + assert state.visibility_checked_at == checked_at_before + + +def test_refresh_visibility_rechecks_stale_answer_and_halts_on_public( + isolated_git_and_gateway_env: Path, fake_latchkey_bin: Path +) -> None: + """Once the confirmed answer goes stale, a re-check happens and a repo + that flipped public halts pushes.""" + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": false}'") + state = _confirmed_private_state(age_seconds=VISIBILITY_CHECK_INTERVAL_SECONDS + 60) + checked_at_before = state.visibility_checked_at + assert checked_at_before is not None + + _refresh_visibility(state, _REPO_URL) + + assert state.visibility == VISIBILITY_PUBLIC + assert state.is_push_allowed is False + assert state.visibility_checked_at is not None + assert state.visibility_checked_at > checked_at_before + + +def test_do_tick_rechecks_visibility_when_repo_url_changes( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """A repointed github_sync.toml must not inherit the previous repo's + confirmed-private answer: the new repo has to earn its own confirmation + before any push (here the re-check fails, so pushes stay held).""" + main, origin = _set_up_synced_workspace(tmp_path, monkeypatch) + install_fake_latchkey(fake_latchkey_bin, "exit 7") + (main / "runtime" / "memory.txt").write_text("important state\n") + # State carried over from ticks against a different, previously-configured + # repo, with a still-fresh private confirmation. + state = _confirmed_private_state(age_seconds=1) + state.repo_url = "https://github.com/some-user/previous-repo" + + _do_tick(state) + + assert state.repo_url == _REPO_URL + remote_branch = _git_out(origin, "rev-parse", "--verify", "runtime-sync") + assert remote_branch.returncode != 0 + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is False + assert status["visibility"] == "unknown" + + +def test_do_tick_restores_missing_worktree_from_origin( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, + fake_latchkey_bin: Path, +) -> None: + """The recreated-workspace self-heal path: when runtime/ is not yet a + worktree, a tick restores it from origin's runtime-sync branch (prior + memory/tickets come back) and then syncs as usual.""" + # A first workspace creates runtime state and pushes it to the origin. + first_base = tmp_path / "first" + first_base.mkdir() + first_main, origin = init_repo_with_origin(first_base) + monkeypatch.chdir(first_main) + assert init_runtime_worktree() is True + (first_main / "runtime" / "memory.md").write_text("remember me\n") + run_git(first_main / "runtime", "add", "-A") + run_git(first_main / "runtime", "commit", "-qm", "state") + run_git(first_main / "runtime", "push", "--set-upstream", "origin", "runtime-sync") + + # A workspace recreated from the synced repo: github_sync.toml came along + # with the checkout, but the container-local runtime worktree did not. + second_main = tmp_path / "second" + init_repo(second_main) + (second_main / "seed.txt").write_text("seed\n") + run_git(second_main, "add", "-A") + run_git(second_main, "commit", "-qm", "seed") + run_git(second_main, "remote", "add", "origin", str(origin)) + (second_main / "github_sync.toml").write_text( + 'repo_url = "https://github.com/some-user/my-workspace"\n' + ) + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": true}'") + monkeypatch.chdir(second_main) + state = _SyncState() + + _do_tick(state) + + assert is_runtime_worktree() + assert (second_main / "runtime" / "memory.md").read_text() == "remember me\n" + status = json.loads(status_file_path().read_text()) + assert status["is_push_allowed"] is True + assert status["last_push_ok"] is True + + +def test_do_tick_defers_when_worktree_missing_and_origin_unreachable( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, +) -> None: + """A recreated workspace whose origin cannot be reached yet (e.g. the + GitHub permissions have not been re-granted) must defer the worktree init + and report that in the status, not create a fresh orphan branch.""" + main = tmp_path / "main" + init_repo(main) + (main / "seed.txt").write_text("seed\n") + run_git(main, "add", "-A") + run_git(main, "commit", "-qm", "seed") + run_git(main, "remote", "add", "origin", str(tmp_path / "does-not-exist.git")) + (main / "github_sync.toml").write_text( + 'repo_url = "https://github.com/some-user/my-workspace"\n' + ) + monkeypatch.chdir(main) + state = _SyncState() + + _do_tick(state) + + assert not is_runtime_worktree() + assert state.last_error is not None + assert "deferred" in state.last_error + status = json.loads(status_file_path().read_text()) + assert "deferred" in status["last_error"] + + +def test_do_tick_idles_when_sync_not_configured( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, +) -> None: + monkeypatch.chdir(tmp_path) + + state = _SyncState() + _do_tick(state) + + assert state.repo_url is None + status = json.loads(status_file_path().read_text()) + assert status["repo_url"] is None + + +def test_do_tick_reports_malformed_config( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + isolated_git_and_gateway_env: Path, +) -> None: + monkeypatch.chdir(tmp_path) + (tmp_path / "github_sync.toml").write_text("repo_url = [broken") + + state = _SyncState() + _do_tick(state) + + assert state.last_error is not None + assert "github_sync.toml" in state.last_error diff --git a/libs/github_sync/src/github_sync/testing.py b/libs/github_sync/src/github_sync/testing.py new file mode 100644 index 000000000..1cceca1d0 --- /dev/null +++ b/libs/github_sync/src/github_sync/testing.py @@ -0,0 +1,52 @@ +"""Shared test helpers for the github_sync test files.""" + +from __future__ import annotations + +import stat +import subprocess +from pathlib import Path + + +def run_git(repo: Path, *args: str) -> None: + """Run a git command in `repo`, raising on failure.""" + subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True) + + +def init_repo(repo: Path) -> None: + """Create a git repo at `repo` with a committer identity configured.""" + repo.mkdir(parents=True, exist_ok=True) + subprocess.run( + ["git", "init", "-q", "--initial-branch=main", str(repo)], + check=True, + capture_output=True, + ) + run_git(repo, "config", "user.email", "test@test.local") + run_git(repo, "config", "user.name", "test") + + +def init_repo_with_origin(base: Path) -> tuple[Path, Path]: + """Create a seeded main repo at base/main with a bare origin at base/origin.git.""" + main = base / "main" + init_repo(main) + (main / "seed.txt").write_text("seed\n") + run_git(main, "add", "-A") + run_git(main, "commit", "-qm", "seed") + origin = base / "origin.git" + subprocess.run( + ["git", "init", "-q", "--bare", str(origin)], check=True, capture_output=True + ) + run_git(main, "remote", "add", "origin", str(origin)) + return main, origin + + +def install_fake_latchkey(bin_dir: Path, script_body: str) -> None: + """Install an executable `latchkey` shell script into `bin_dir`. + + The caller prepends `bin_dir` to PATH (via monkeypatch.setenv) so + check_repo_visibility exercises its real subprocess path against a + controllable stand-in instead of the real latchkey CLI. + """ + bin_dir.mkdir(parents=True, exist_ok=True) + script = bin_dir / "latchkey" + script.write_text(f"#!/usr/bin/env bash\n{script_body}\n") + script.chmod(script.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) diff --git a/libs/github_sync/src/github_sync/visibility.py b/libs/github_sync/src/github_sync/visibility.py new file mode 100644 index 000000000..f4edec5bb --- /dev/null +++ b/libs/github_sync/src/github_sync/visibility.py @@ -0,0 +1,107 @@ +"""Sync-repo visibility checking through latchkey. + +Public repos must never be synced to: agents might push secrets or other +sensitive data without thinking about it. The skill verifies the repo is +private at enable time, and the service re-checks periodically: pushes are +held until visibility is first confirmed private and halted whenever the +repo is confirmed public, while a re-check that fails outright keeps the +last confirmed answer (see runner._refresh_visibility for that policy). +""" + +import json +import os +import subprocess + +from loguru import logger + +from github_sync.config import ( + ENV_GATEWAY, + ENV_GATEWAY_PERMISSIONS_OVERRIDE, + get_secondary_gateway_url, + parse_owner_and_name, +) + +VISIBILITY_PRIVATE = "private" +VISIBILITY_PUBLIC = "public" +VISIBILITY_UNKNOWN = "unknown" + +_LATCHKEY_CURL_TIMEOUT_SECONDS = 60 + + +def parse_visibility_response(body: str) -> str: + """Map a GitHub `GET /repos//` response body to a visibility. + + Anything that is not an explicit `"private": true/false` (error bodies, + truncated output, a 404 for a deleted repo) is UNKNOWN, which callers + treat as push-blocking. + """ + try: + data = json.loads(body) + except json.JSONDecodeError: + return VISIBILITY_UNKNOWN + if not isinstance(data, dict): + return VISIBILITY_UNKNOWN + is_private = data.get("private") + if is_private is True: + return VISIBILITY_PRIVATE + elif is_private is False: + return VISIBILITY_PUBLIC + else: + return VISIBILITY_UNKNOWN + + +def _latchkey_curl_env(is_secondary: bool) -> dict[str, str] | None: + """Env for a `latchkey curl` call; None means inherit (primary gateway).""" + if not is_secondary: + return None + secondary_url = get_secondary_gateway_url() + if secondary_url is None: + return None + # Per the latchkey skill: the secondary gateway takes no permissions + # override, so it must be cleared alongside the gateway swap. + return { + **os.environ, + ENV_GATEWAY: secondary_url, + ENV_GATEWAY_PERMISSIONS_OVERRIDE: "", + } + + +def check_repo_visibility(repo_url: str) -> str: + """Ask GitHub (via latchkey) whether the sync repo is private. + + Tries the primary gateway first, then the secondary (per-VPS) gateway so + the check keeps working when the user's machine is offline. Returns + UNKNOWN when neither gateway produces a parseable answer. + """ + owner, name = parse_owner_and_name(repo_url) + api_url = f"https://api.github.com/repos/{owner}/{name}" + gateway_attempts = [False] + if get_secondary_gateway_url() is not None: + gateway_attempts.append(True) + for is_secondary in gateway_attempts: + # latchkey curl injects the GitHub credential server-side; -s keeps + # stdout parseable. + try: + result = subprocess.run( + ["latchkey", "curl", "-s", api_url], + capture_output=True, + text=True, + check=False, + env=_latchkey_curl_env(is_secondary), + timeout=_LATCHKEY_CURL_TIMEOUT_SECONDS, + ) + except (OSError, subprocess.TimeoutExpired) as e: + logger.debug("latchkey curl failed (secondary={}): {}", is_secondary, e) + continue + if result.returncode != 0: + logger.debug( + "latchkey curl exited {} (secondary={}): {}", + result.returncode, + is_secondary, + result.stderr.strip(), + ) + continue + visibility = parse_visibility_response(result.stdout) + if visibility != VISIBILITY_UNKNOWN: + return visibility + return VISIBILITY_UNKNOWN diff --git a/libs/github_sync/src/github_sync/visibility_test.py b/libs/github_sync/src/github_sync/visibility_test.py new file mode 100644 index 000000000..d8466c827 --- /dev/null +++ b/libs/github_sync/src/github_sync/visibility_test.py @@ -0,0 +1,74 @@ +"""Unit tests for repo-visibility checking (the private-only enforcement).""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from github_sync.testing import install_fake_latchkey +from github_sync.visibility import ( + VISIBILITY_PRIVATE, + VISIBILITY_PUBLIC, + VISIBILITY_UNKNOWN, + check_repo_visibility, + parse_visibility_response, +) + +_REPO_URL = "https://github.com/some-user/my-workspace" + + +def test_parse_visibility_response_private() -> None: + assert parse_visibility_response('{"private": true}') == VISIBILITY_PRIVATE + + +def test_parse_visibility_response_public() -> None: + assert parse_visibility_response('{"private": false}') == VISIBILITY_PUBLIC + + +def test_parse_visibility_response_unknown_on_error_bodies() -> None: + # A 404 body (deleted repo), non-JSON output, and non-object JSON must all + # come back UNKNOWN, which blocks pushes. + assert parse_visibility_response('{"message": "Not Found"}') == VISIBILITY_UNKNOWN + assert parse_visibility_response("curl: connection refused") == VISIBILITY_UNKNOWN + assert parse_visibility_response("[1, 2]") == VISIBILITY_UNKNOWN + assert parse_visibility_response("") == VISIBILITY_UNKNOWN + + +def test_check_repo_visibility_reads_private_from_latchkey( + fake_latchkey_bin: Path, +) -> None: + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": true}'") + assert check_repo_visibility(_REPO_URL) == VISIBILITY_PRIVATE + + +def test_check_repo_visibility_detects_public_repo(fake_latchkey_bin: Path) -> None: + install_fake_latchkey(fake_latchkey_bin, "echo '{\"private\": false}'") + assert check_repo_visibility(_REPO_URL) == VISIBILITY_PUBLIC + + +def test_check_repo_visibility_unknown_when_latchkey_fails( + fake_latchkey_bin: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # Exit code 7 is what latchkey curl produces when the gateway is offline. + monkeypatch.delenv("LATCHKEY_GATEWAY_SECONDARY", raising=False) + install_fake_latchkey(fake_latchkey_bin, "exit 7") + assert check_repo_visibility(_REPO_URL) == VISIBILITY_UNKNOWN + + +def test_check_repo_visibility_falls_back_to_secondary_gateway( + fake_latchkey_bin: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The fake gateway only answers when called with the secondary gateway's + # env override (primary attempt fails), proving the fallback fires with + # the documented env shape (secondary URL + cleared permissions override). + monkeypatch.setenv("LATCHKEY_GATEWAY", "http://primary.invalid:1") + monkeypatch.setenv("LATCHKEY_GATEWAY_SECONDARY", "http://127.0.0.1:46123") + monkeypatch.setenv("LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE", "some-jwt") + install_fake_latchkey( + fake_latchkey_bin, + 'if [ "$LATCHKEY_GATEWAY" = "http://127.0.0.1:46123" ] ' + '&& [ -z "$LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE" ]; then ' + "echo '{\"private\": true}'; else exit 7; fi", + ) + assert check_repo_visibility(_REPO_URL) == VISIBILITY_PRIVATE diff --git a/libs/github_sync/src/github_sync/wiring.py b/libs/github_sync/src/github_sync/wiring.py new file mode 100644 index 000000000..5ccbc3f14 --- /dev/null +++ b/libs/github_sync/src/github_sync/wiring.py @@ -0,0 +1,136 @@ +"""Global git config wiring that routes GitHub traffic through the latchkey gateway. + +Applied by the github-sync skill at enable time and re-applied by the service +when a push fails (self-healing a gateway URL that changed across container +restarts). With the wiring in place, plain `git push` / `git fetch` against any +https://github.com/... remote is transparently rewritten to the gateway's git +proxy and authenticated with the gateway headers -- for every checkout in the +container (main repo, worker worktrees, the runtime/ worktree). The GitHub +credential itself is injected server-side by the gateway; no token ever enters +the container. + +The wiring also points core.hooksPath at the repo's git_hooks so the +post-commit auto-push hook applies everywhere. +""" + +import subprocess + +from loguru import logger + +from github_sync.config import ( + GITHUB_URL_PREFIX, + get_gateway_password, + get_gateway_permissions_override, + get_gateway_url, + proxied_url, +) + +HOOKS_PATH = "/mngr/code/scripts/git_hooks" +PASSWORD_HEADER = "X-Latchkey-Gateway-Password" +PERMISSIONS_OVERRIDE_HEADER = "X-Latchkey-Gateway-Permissions-Override" + + +def _git_config(*args: str) -> subprocess.CompletedProcess[str]: + """Run a `git config --global` command, never raising.""" + return subprocess.run( + ["git", "config", "--global", *args], + capture_output=True, + text=True, + check=False, + ) + + +def _list_global_config(key_regexp: str) -> list[tuple[str, str]]: + """List (key, value) pairs of global git config entries matching a key regexp.""" + result = _git_config("--get-regexp", key_regexp) + if result.returncode != 0: + # rc=1 simply means no matching entries. + return [] + entries: list[tuple[str, str]] = [] + for line in result.stdout.splitlines(): + key, _, value = line.partition(" ") + if key: + entries.append((key, value)) + return entries + + +def _remove_gateway_entries(kept_gateway_url: str | None) -> None: + """Remove latchkey-gateway git config entries for every other gateway URL. + + With `kept_gateway_url=None` this removes all of them (the disable path). + Gateway URLs embed a reverse-tunneled local port that can change across + container restarts, so re-wiring must clean up entries pointing at a stale + port -- two insteadOf rewrites for the same prefix would be ambiguous. + """ + kept_insteadof_key = ( + f"url.{proxied_url(kept_gateway_url, GITHUB_URL_PREFIX)}.insteadof" + if kept_gateway_url is not None + else None + ) + for key, value in _list_global_config(r"url\..*\.insteadof"): + is_gateway_rewrite = value == GITHUB_URL_PREFIX and "/gateway/" in key + if is_gateway_rewrite and key.lower() != (kept_insteadof_key or "").lower(): + _git_config("--unset-all", key) + + kept_header_key = ( + f"http.{kept_gateway_url}/.extraheader" + if kept_gateway_url is not None + else None + ) + for key, value in _list_global_config(r"http\..*\.extraheader"): + is_gateway_header = value.startswith( + (PASSWORD_HEADER, PERMISSIONS_OVERRIDE_HEADER) + ) + if is_gateway_header and key.lower() != (kept_header_key or "").lower(): + _git_config("--unset-all", key) + + +def apply_git_wiring() -> bool: + """Install the gateway rewrite, auth headers, and hooks path in global git config. + + Idempotent; safe to re-run on every service self-heal. Returns False (with + a logged warning) when the latchkey gateway env vars are not all present. + """ + gateway_url = get_gateway_url() + password = get_gateway_password() + permissions_override = get_gateway_permissions_override() + if not gateway_url or not password or not permissions_override: + logger.warning( + "Latchkey gateway env is incomplete (need LATCHKEY_GATEWAY, " + "LATCHKEY_GATEWAY_PASSWORD, LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE); " + "cannot wire git for GitHub sync" + ) + return False + + _remove_gateway_entries(gateway_url) + + header_key = f"http.{gateway_url}/.extraHeader" + config_steps = ( + ( + "--replace-all", + f"url.{proxied_url(gateway_url, GITHUB_URL_PREFIX)}.insteadOf", + GITHUB_URL_PREFIX, + ), + ("--replace-all", header_key, f"{PASSWORD_HEADER}: {password}"), + ("--add", header_key, f"{PERMISSIONS_OVERRIDE_HEADER}: {permissions_override}"), + ("--replace-all", "core.hooksPath", HOOKS_PATH), + ) + for argv in config_steps: + result = _git_config(*argv) + if result.returncode != 0: + logger.warning( + "git config --global {} failed (rc={}): {}", + " ".join(argv), + result.returncode, + result.stderr.strip(), + ) + return False + logger.debug("Wired git GitHub access through gateway {}", gateway_url) + return True + + +def remove_git_wiring() -> None: + """Remove every gateway git config entry and the hooks path (disable path).""" + _remove_gateway_entries(None) + _git_config("--unset-all", "core.hooksPath") + logger.info("Removed GitHub sync git wiring from global git config") diff --git a/libs/github_sync/src/github_sync/wiring_test.py b/libs/github_sync/src/github_sync/wiring_test.py new file mode 100644 index 000000000..27d9150df --- /dev/null +++ b/libs/github_sync/src/github_sync/wiring_test.py @@ -0,0 +1,98 @@ +"""Unit tests for the latchkey-gateway git config wiring.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from github_sync.wiring import HOOKS_PATH, apply_git_wiring, remove_git_wiring + + +def _get_all_global(key: str) -> list[str]: + result = subprocess.run( + ["git", "config", "--global", "--get-all", key], + capture_output=True, + text=True, + check=False, + ) + return result.stdout.splitlines() + + +def _set_gateway_env(monkeypatch: pytest.MonkeyPatch, gateway_url: str) -> None: + monkeypatch.setenv("LATCHKEY_GATEWAY", gateway_url) + monkeypatch.setenv("LATCHKEY_GATEWAY_PASSWORD", "pw-abc") + monkeypatch.setenv("LATCHKEY_GATEWAY_PERMISSIONS_OVERRIDE", "jwt-xyz") + + +def test_apply_git_wiring_writes_rewrite_headers_and_hookspath( + isolated_git_and_gateway_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + _set_gateway_env(monkeypatch, "http://127.0.0.1:41234") + + assert apply_git_wiring() is True + + insteadof_key = "url.http://127.0.0.1:41234/gateway/https://github.com/.insteadOf" + assert _get_all_global(insteadof_key) == ["https://github.com/"] + headers = _get_all_global("http.http://127.0.0.1:41234/.extraHeader") + assert headers == [ + "X-Latchkey-Gateway-Password: pw-abc", + "X-Latchkey-Gateway-Permissions-Override: jwt-xyz", + ] + assert _get_all_global("core.hooksPath") == [HOOKS_PATH] + + +def test_apply_git_wiring_fails_without_gateway_env( + isolated_git_and_gateway_env: Path, +) -> None: + assert apply_git_wiring() is False + assert _get_all_global("core.hooksPath") == [] + + +def test_apply_git_wiring_replaces_stale_gateway_entries( + isolated_git_and_gateway_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A gateway URL embeds a port that changes across restarts; re-wiring must + remove the previous port's entries or git would see two ambiguous + rewrites for the same prefix.""" + _set_gateway_env(monkeypatch, "http://127.0.0.1:41234") + assert apply_git_wiring() is True + _set_gateway_env(monkeypatch, "http://127.0.0.1:59999") + + assert apply_git_wiring() is True + + stale_key = "url.http://127.0.0.1:41234/gateway/https://github.com/.insteadOf" + fresh_key = "url.http://127.0.0.1:59999/gateway/https://github.com/.insteadOf" + assert _get_all_global(stale_key) == [] + assert _get_all_global(fresh_key) == ["https://github.com/"] + assert _get_all_global("http.http://127.0.0.1:41234/.extraHeader") == [] + assert len(_get_all_global("http.http://127.0.0.1:59999/.extraHeader")) == 2 + + +def test_remove_git_wiring_clears_everything_but_keeps_ssh_rewrites( + isolated_git_and_gateway_env: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + # The bootstrap-owned ssh->https rewrite shares the value shape but is not + # gateway wiring and must survive a disable. + subprocess.run( + [ + "git", + "config", + "--global", + "url.https://github.com/.insteadOf", + "git@github.com:", + ], + check=True, + capture_output=True, + ) + _set_gateway_env(monkeypatch, "http://127.0.0.1:41234") + assert apply_git_wiring() is True + + remove_git_wiring() + + gateway_key = "url.http://127.0.0.1:41234/gateway/https://github.com/.insteadOf" + assert _get_all_global(gateway_key) == [] + assert _get_all_global("http.http://127.0.0.1:41234/.extraHeader") == [] + assert _get_all_global("core.hooksPath") == [] + assert _get_all_global("url.https://github.com/.insteadOf") == ["git@github.com:"] diff --git a/libs/github_sync/src/github_sync/worktree.py b/libs/github_sync/src/github_sync/worktree.py new file mode 100644 index 000000000..00a0f3f11 --- /dev/null +++ b/libs/github_sync/src/github_sync/worktree.py @@ -0,0 +1,264 @@ +"""Setup and restore of runtime/ as a git worktree on the runtime-sync branch. + +Run by the github-sync skill at enable time, and retried by the service on +every tick while runtime/ is not yet a worktree (the self-healing path for a +workspace recreated from a previously-synced repo: once the user re-grants +the latchkey GitHub permissions, the next tick fetches origin's runtime-sync +branch and materializes the prior runtime/ state -- memory, tickets, +transcripts -- automatically). + +Any files already sitting in runtime/ (a fresh boot writes signal files and +tickets there before sync is enabled) are staged aside during worktree +creation and restored on top afterwards, never clobbering restored state. +""" + +import os +import shutil +import subprocess + +from loguru import logger + +from github_sync.config import RUNTIME_DIR, SYNC_BRANCH + +SYNC_USER_NAME = "github-sync" +SYNC_USER_EMAIL = "github-sync@minds.local" + +_RUNTIME_PREEXISTING_DIR = RUNTIME_DIR.with_name(RUNTIME_DIR.name + ".preexisting") + +# `git ls-remote --exit-code` exits 2 when the remote is reachable but has no +# matching refs. +_LS_REMOTE_NO_MATCHING_REFS = 2 + + +def _git_noninteractive_env() -> dict[str, str]: + """Environment for sync git calls: never prompt for credentials. + + GIT_TERMINAL_PROMPT=0 turns any credential prompt (e.g. a fetch through a + gateway whose permission grant lapsed) into a fast failure instead of a + TTY prompt that would wedge the caller forever; every caller here already + logs-and-continues on a nonzero exit. + """ + return {**os.environ, "GIT_TERMINAL_PROMPT": "0"} + + +def git_main(*args: str) -> subprocess.CompletedProcess[str]: + """Run a git command in the main checkout, never raising or prompting.""" + return subprocess.run( + ["git", *args], + capture_output=True, + text=True, + check=False, + env=_git_noninteractive_env(), + ) + + +def git_runtime(*args: str) -> subprocess.CompletedProcess[str]: + """Run a git command inside the runtime worktree, never raising or prompting.""" + return subprocess.run( + ["git", "-C", str(RUNTIME_DIR), *args], + capture_output=True, + text=True, + check=False, + env=_git_noninteractive_env(), + ) + + +def is_runtime_worktree() -> bool: + """Return True when runtime/ is already a git worktree.""" + return (RUNTIME_DIR / ".git").exists() + + +def _restore_preexisting_into_worktree() -> None: + """Move any files from runtime.preexisting/ back into runtime/.""" + if not _RUNTIME_PREEXISTING_DIR.exists(): + return + for entry in list(_RUNTIME_PREEXISTING_DIR.iterdir()): + target = RUNTIME_DIR / entry.name + if target.exists(): + # Don't clobber what the worktree already has (e.g. restored + # state fetched from origin, or a fresh .gitignore). + continue + shutil.move(str(entry), str(target)) + try: + _RUNTIME_PREEXISTING_DIR.rmdir() + except OSError: + logger.warning( + "{} not empty after restore; leaving for inspection", + _RUNTIME_PREEXISTING_DIR, + ) + + +def _stage_preexisting_aside() -> None: + """Move runtime/'s contents to runtime.preexisting/ so we can add a worktree. + + Only called when runtime/ exists with files but is not yet a git worktree. + """ + if _RUNTIME_PREEXISTING_DIR.exists(): + # Stale leftover from a prior failed init -- clear it. + shutil.rmtree(_RUNTIME_PREEXISTING_DIR) + shutil.move(str(RUNTIME_DIR), str(_RUNTIME_PREEXISTING_DIR)) + + +def _runtime_dir_has_files() -> bool: + """Return True if runtime/ exists and contains anything.""" + if not RUNTIME_DIR.exists(): + return False + return any(RUNTIME_DIR.iterdir()) + + +def _create_orphan_runtime_worktree(branch: str) -> subprocess.CompletedProcess[str]: + """Add runtime/ as a worktree on a fresh orphan branch, git-version-agnostically. + + `git worktree add --orphan` only exists in git >= 2.42, but the Lima + provider's Debian 12 base ships git 2.39. So build the orphan branch with + plumbing that has worked for ages -- a parentless commit on the empty tree -- + then do a normal `git worktree add` for it. Returns the final worktree-add + CompletedProcess; if an earlier plumbing step fails, returns that failing + CompletedProcess so the caller's existing error handling fires. + """ + empty_tree = git_main("hash-object", "-w", "-t", "tree", "/dev/null") + if empty_tree.returncode != 0: + return empty_tree + # Commit identity is passed via -c because the container may have no global + # git identity yet, and commit-tree refuses to run without one. + orphan_commit = git_main( + "-c", + f"user.name={SYNC_USER_NAME}", + "-c", + f"user.email={SYNC_USER_EMAIL}", + "commit-tree", + empty_tree.stdout.strip(), + "-m", + "runtime sync: init", + ) + if orphan_commit.returncode != 0: + return orphan_commit + branch_result = git_main("branch", branch, orphan_commit.stdout.strip()) + if branch_result.returncode != 0: + return branch_result + return git_main("worktree", "add", str(RUNTIME_DIR), branch) + + +def init_runtime_worktree() -> bool: + """One-time setup of runtime/ as a worktree of the runtime-sync branch. + + Returns True when runtime/ is a worktree afterwards. Never raises: a + failure (most commonly the origin remote being unreachable because the + latchkey GitHub permission has not been granted yet) is logged and the + caller retries later. Deliberately refuses to create a *fresh* orphan + branch while origin is unreachable -- a remote runtime-sync branch might + exist (workspace recreated from a synced repo), and starting an unrelated + local history would leave the two permanently diverged. + """ + if is_runtime_worktree(): + # A prior init may have staged runtime/ content aside and been killed + # before restoring it. Recover that content now rather than stranding + # it; this no-ops when runtime.preexisting/ is absent (the common case). + _restore_preexisting_into_worktree() + return True + + has_local_branch = ( + git_main("rev-parse", "--verify", "--quiet", SYNC_BRANCH).returncode == 0 + ) + + # Classify the remote state: branch exists (restore it), remote reachable + # but branch absent (fresh orphan), or remote unreachable (retry later). + ls_remote = git_main("ls-remote", "--exit-code", "--heads", "origin", SYNC_BRANCH) + if ls_remote.returncode == 0: + has_remote_branch = True + elif ls_remote.returncode == _LS_REMOTE_NO_MATCHING_REFS: + has_remote_branch = False + elif has_local_branch: + # Offline, but a local runtime-sync branch already exists (e.g. a + # prior init whose worktree was removed) -- reattaching to it cannot + # diverge from origin any further than it already has. + has_remote_branch = False + else: + logger.warning( + "origin is unreachable (rc={}): {}; deferring runtime worktree init", + ls_remote.returncode, + ls_remote.stderr.strip(), + ) + return False + + logger.info("Initializing runtime worktree on branch {}", SYNC_BRANCH) + + remote_ref = f"origin/{SYNC_BRANCH}" + if has_remote_branch: + fetch_result = git_main("fetch", "origin", SYNC_BRANCH) + if fetch_result.returncode != 0: + logger.warning( + "git fetch origin {} failed (rc={}): {}; deferring init", + SYNC_BRANCH, + fetch_result.returncode, + fetch_result.stderr.strip(), + ) + return False + + staged_aside = False + if _runtime_dir_has_files(): + logger.info( + "runtime/ already has files; staging them aside before adding the worktree" + ) + _stage_preexisting_aside() + staged_aside = True + + if has_remote_branch: + result = git_main( + "worktree", "add", "-B", SYNC_BRANCH, str(RUNTIME_DIR), remote_ref + ) + elif has_local_branch: + result = git_main("worktree", "add", str(RUNTIME_DIR), SYNC_BRANCH) + else: + result = _create_orphan_runtime_worktree(SYNC_BRANCH) + + if result.returncode != 0: + logger.error( + "git worktree add failed (rc={}): {}", + result.returncode, + result.stderr.strip(), + ) + # Restore preexisting files so other services don't lose them. + if staged_aside: + if not RUNTIME_DIR.exists(): + shutil.move(str(_RUNTIME_PREEXISTING_DIR), str(RUNTIME_DIR)) + else: + _restore_preexisting_into_worktree() + return False + + # Configure bot identity for sync commits inside this worktree only. + git_runtime("config", "user.name", SYNC_USER_NAME) + git_runtime("config", "user.email", SYNC_USER_EMAIL) + + if has_remote_branch: + # Make sure the local branch tracks the remote (some git versions + # don't set this automatically with -B + an explicit ref). + git_runtime("branch", "--set-upstream-to", remote_ref) + + # Every path must end with the secrets-excluding .gitignore in place: + # runtime/secrets holds e.g. the Cloudflare tunnel token, which must never + # reach the remote. A restored or reattached branch normally already + # carries the file in its history, but a branch left behind by an + # interrupted prior init (created just before its `worktree add` failed) + # does not -- and without it the next sync tick's `git add -A` would ship + # the secrets. So write and commit it whenever it is missing, never + # clobbering an existing (possibly customized) one. On the fresh-orphan + # path this doubles as the initial commit that gives push something to + # push. + gitignore = RUNTIME_DIR / ".gitignore" + if not gitignore.exists(): + gitignore.write_text("secrets\n") + git_runtime("add", ".gitignore") + commit = git_runtime("commit", "-m", "runtime sync: init") + if commit.returncode != 0: + logger.error( + "initial commit failed (rc={}): {}", + commit.returncode, + commit.stderr.strip(), + ) + + # Restore staged-aside content. Calling unconditionally (rather than + # gating on the `staged_aside` flag) also recovers content left by a + # prior init that staged aside but was killed before it could restore. + _restore_preexisting_into_worktree() + return True diff --git a/libs/github_sync/src/github_sync/worktree_test.py b/libs/github_sync/src/github_sync/worktree_test.py new file mode 100644 index 000000000..9cf94c839 --- /dev/null +++ b/libs/github_sync/src/github_sync/worktree_test.py @@ -0,0 +1,153 @@ +"""Unit tests for runtime/ worktree init and restore-from-origin.""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +import pytest + +from github_sync.testing import init_repo, init_repo_with_origin, run_git +from github_sync.worktree import init_runtime_worktree, is_runtime_worktree + + +def _git_out(repo: Path, *args: str) -> str: + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True, check=False + ).stdout + + +def test_init_creates_orphan_worktree_when_origin_has_no_sync_branch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + + assert init_runtime_worktree() is True + + runtime = main / "runtime" + assert is_runtime_worktree() + # The branch is an orphan: a single parentless root, sharing no history + # with main (whose seed commit must not be reachable). + roots = _git_out(runtime, "rev-list", "--max-parents=0", "HEAD").split() + assert len(roots) == 1 + assert "seed" not in _git_out(runtime, "log", "--format=%s") + assert _git_out(runtime, "branch", "--show-current").strip() == "runtime-sync" + # Secrets are excluded from the sync branch. + assert (runtime / ".gitignore").read_text() == "secrets\n" + + +def test_init_stages_aside_and_restores_preexisting_runtime_files( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + runtime = main / "runtime" + runtime.mkdir() + (runtime / "initial_chat_created").write_text("") + + assert init_runtime_worktree() is True + + assert (runtime / "initial_chat_created").exists() + assert not (main / "runtime.preexisting").exists() + + +def test_init_restores_runtime_state_from_origin_sync_branch( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The recreated-workspace path: origin already has runtime-sync, so init + materializes the prior runtime/ state instead of starting fresh.""" + # A first workspace creates and pushes runtime state. + first_base = tmp_path / "first" + first_base.mkdir() + first_main, origin = init_repo_with_origin(first_base) + monkeypatch.chdir(first_main) + assert init_runtime_worktree() is True + (first_main / "runtime" / "memory.md").write_text("remember me\n") + run_git(first_main / "runtime", "add", "-A") + run_git(first_main / "runtime", "commit", "-qm", "state") + run_git(first_main / "runtime", "push", "--set-upstream", "origin", "runtime-sync") + + # A second workspace pointing at the same origin restores that state. + second_main = tmp_path / "second" + init_repo(second_main) + (second_main / "seed.txt").write_text("seed\n") + run_git(second_main, "add", "-A") + run_git(second_main, "commit", "-qm", "seed") + run_git(second_main, "remote", "add", "origin", str(origin)) + # Fresh-boot files already in runtime/ must survive the restore. + runtime = second_main / "runtime" + runtime.mkdir() + (runtime / "initial_chat_created").write_text("") + monkeypatch.chdir(second_main) + + assert init_runtime_worktree() is True + + assert (runtime / "memory.md").read_text() == "remember me\n" + assert (runtime / "initial_chat_created").exists() + upstream = _git_out( + runtime, "rev-parse", "--abbrev-ref", "runtime-sync@{upstream}" + ).strip() + assert upstream == "origin/runtime-sync" + + +def test_init_defers_when_origin_unreachable( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """With origin unreachable we cannot know whether a remote runtime-sync + branch exists, so init must NOT create a fresh orphan branch (it could + permanently diverge from a restorable remote); it defers instead.""" + main = tmp_path / "main" + init_repo(main) + (main / "seed.txt").write_text("seed\n") + run_git(main, "add", "-A") + run_git(main, "commit", "-qm", "seed") + run_git(main, "remote", "add", "origin", str(tmp_path / "does-not-exist.git")) + runtime = main / "runtime" + runtime.mkdir() + (runtime / "precious.txt").write_text("keep\n") + monkeypatch.chdir(main) + + assert init_runtime_worktree() is False + + assert not is_runtime_worktree() + assert (runtime / "precious.txt").read_text() == "keep\n" + + +def test_init_reattach_writes_missing_secrets_gitignore( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A runtime-sync branch left behind by an interrupted prior init (the + branch is created just before the `worktree add` that failed) has no + .gitignore in its history. Reattaching to it must still put the secrets + exclusion in place, or the next sync tick's `git add -A` would push + runtime/secrets to the remote.""" + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + # Mirror what the interrupted init leaves: a parentless empty-tree commit + # on runtime-sync and no worktree. + empty_tree = _git_out(main, "hash-object", "-w", "-t", "tree", "/dev/null").strip() + orphan_commit = _git_out( + main, "commit-tree", empty_tree, "-m", "runtime sync: init" + ).strip() + run_git(main, "branch", "runtime-sync", orphan_commit) + + assert init_runtime_worktree() is True + + runtime = main / "runtime" + assert (runtime / ".gitignore").read_text() == "secrets\n" + # Committed, not just written: the worktree is clean afterwards. + assert _git_out(runtime, "status", "--porcelain").strip() == "" + + +def test_init_noops_when_already_a_worktree( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + main, origin = init_repo_with_origin(tmp_path) + monkeypatch.chdir(main) + assert init_runtime_worktree() is True + head_before = _git_out(main / "runtime", "rev-parse", "HEAD") + + assert init_runtime_worktree() is True + + assert _git_out(main / "runtime", "rev-parse", "HEAD") == head_before diff --git a/libs/runtime_backup/test_runtime_backup_ratchets.py b/libs/github_sync/test_github_sync_ratchets.py similarity index 84% rename from libs/runtime_backup/test_runtime_backup_ratchets.py rename to libs/github_sync/test_github_sync_ratchets.py index 30889dc13..548576101 100644 --- a/libs/runtime_backup/test_runtime_backup_ratchets.py +++ b/libs/github_sync/test_github_sync_ratchets.py @@ -26,7 +26,10 @@ def test_prevent_while_true() -> None: def test_prevent_time_sleep() -> None: - rc.check_time_sleep(_DIR, snapshot(1)) + # 1 = the service loop's tick interval; 1 = the bounded condition-poll in + # post_commit_hook_test.py waiting for the hook's intentionally-backgrounded + # push (the polling pattern the rule_description itself recommends). + rc.check_time_sleep(_DIR, snapshot(2)) def test_prevent_global_keyword() -> None: @@ -72,4 +75,3 @@ def test_prevent_asyncio_import() -> None: def test_prevent_dataclasses_import() -> None: rc.check_dataclasses_import(_DIR, snapshot(0)) - diff --git a/libs/host_backup/README.md b/libs/host_backup/README.md index 282624d28..7246de6f1 100644 --- a/libs/host_backup/README.md +++ b/libs/host_backup/README.md @@ -3,8 +3,8 @@ Background service that continuously backs up the agent's full `host_dir` (`/mngr/`) to a remote restic repository (Cloudflare R2 by default). -Distinct from `runtime_backup`, which only ships `runtime/` to a GitHub -orphan branch as a fine-grained checkpoint. `host_backup` covers the whole +Distinct from the opt-in `github_sync` service, which only ships `runtime/` +to a GitHub orphan branch as a fine-grained checkpoint. `host_backup` covers the whole host_dir (code, worktrees, agent state, chat sessions, logs) and pushes to an encrypted restic repo on cheaper object storage. @@ -31,7 +31,8 @@ an encrypted restic repo on cheaper object storage. `AWS_SECRET_ACCESS_KEY` for an S3/R2 backend). Written only by the minds app (injected whole); a missing file means backups are not configured. `restic.env` is gitignored (rides nothing). `backup.toml` is *not* - gitignored so it survives container loss via runtime-backup. + gitignored, so it also rides the opt-in GitHub sync of runtime/ when + that is enabled. - A tick only runs once both `RESTIC_REPOSITORY` and `RESTIC_PASSWORD` are set in `restic.env`. Backend credentials are not gated by host_backup -- restic reports its own error if the chosen backend needs one that is @@ -58,8 +59,8 @@ an encrypted restic repo on cheaper object storage. M --keep-weekly W --keep-monthly O` runs (cheap, index-only). At most once per `prune_interval_hours` (default 24) we additionally run `restic prune` (the slow data deletion step); gated by - `runtime/last-restic-prune` (a timestamp file that rides runtime-backup - so it survives container loss). + `runtime/last-restic-prune` (a timestamp file under runtime/, covered by + the opt-in GitHub sync when enabled). - The outer loop never exits. Every exception is logged with full traceback to loguru and as a `tick_error` event in the jsonl stream; the loop continues to the next tick. diff --git a/libs/host_backup/src/host_backup/config.py b/libs/host_backup/src/host_backup/config.py index 109a6877f..7aed4926c 100644 --- a/libs/host_backup/src/host_backup/config.py +++ b/libs/host_backup/src/host_backup/config.py @@ -8,7 +8,7 @@ `[snapshot]` section old bootstraps keep writing forever) and malformed values produce log warnings and fall back to defaults -- they never crash the service and never block the remaining valid settings from applying. - Rides the runtime-backup git push. + Rides the opt-in GitHub sync of runtime/ when that is enabled. - `runtime/secrets/restic.env` -- restic's repository address + all secrets (`RESTIC_REPOSITORY`, `RESTIC_PASSWORD`, and any backend credentials restic reads from the environment, e.g. `AWS_ACCESS_KEY_ID` / diff --git a/libs/runtime_backup/README.md b/libs/runtime_backup/README.md deleted file mode 100644 index 1db7c063b..000000000 --- a/libs/runtime_backup/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# runtime_backup - -Background service that periodically commits and pushes the contents of -`runtime/` (which holds Claude memory, ticket state, transcripts, app port -registry, etc.) to a per-agent orphan branch named -`mindsbackup/$MNGR_AGENT_ID` on the same `origin` as the main checkout. - -The branch and the worktree at `runtime/` itself are created by -`libs/bootstrap` during its pre-services init step. This service assumes -that has already happened and just polls. - -## Behavior - -- Tick interval: 60 seconds. -- Each tick: `git add -A`, commit (only if dirty) with message - `runtime backup: `, push (only when `GH_TOKEN` - is set in env). All `git` operations target the runtime worktree. -- Push first tries default args (no `--force`). Bootstrap normally sets - the upstream during init, but if that best-effort push failed, the - next tick's plain `git push` fails with "no upstream" -- so on push - failure the runner retries once with `git push --set-upstream origin - ` to self-heal that case (mirroring the post-commit hook). - `--force` is never used; the per-agent branch model means there is - only ever one writer, so non-fast-forwards should not happen. -- `runtime/secrets` is excluded via the worktree's own `.gitignore` - (written by bootstrap during init), so the Cloudflare tunnel token - never reaches the remote. -- Each tick first clears a stale `index.lock` from the runtime worktree if - one is present. This service is the worktree's only git writer and its - ticks are sequential, so a lock here was normally left by a prior tick's - git process that was killed before releasing it (if something kills the - process mid-commit). Git never clears such a lock itself, so without this - every later `git add` would fail identically and backups would stop - permanently and silently. As a safeguard against the unlikely case of a - concurrent writer, the lock is only removed once it is older than the tick - interval -- a live git operation on the small `runtime/` tree holds the - lock far more briefly than that, so an in-progress operation is never - disturbed. -- On any git failure the service logs to stderr and `/tmp/runtime-backup.log` - and tries again on the next tick. It does not exit, so bootstrap's - `restart = "on-failure"` policy is only triggered on a hard crash. -- Without `GH_TOKEN` the service still commits locally; pushes are - skipped until a token appears (typically on container restart). - -## Restoring on a fresh container - -If `mindsbackup/$MNGR_AGENT_ID` already exists on origin (e.g. the same -agent is recreated), bootstrap fetches and materializes it into `runtime/` -on first boot, so prior memory and runtime state come back automatically. - -Migration to a *different* `MNGR_AGENT_ID` is intentionally manual. diff --git a/libs/runtime_backup/pyproject.toml b/libs/runtime_backup/pyproject.toml deleted file mode 100644 index 0cf347675..000000000 --- a/libs/runtime_backup/pyproject.toml +++ /dev/null @@ -1,19 +0,0 @@ -[project] -name = "runtime-backup" -version = "0.1.0" -description = "Service that periodically commits + pushes runtime/ to a per-agent orphan branch (mindsbackup/$MNGR_AGENT_ID)" -readme = "README.md" -requires-python = ">=3.11" -dependencies = [ - "loguru>=0.7.0", -] - -[project.scripts] -runtime-backup = "runtime_backup.runner:main" - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/runtime_backup"] diff --git a/libs/runtime_backup/src/runtime_backup/runner.py b/libs/runtime_backup/src/runtime_backup/runner.py deleted file mode 100644 index a145ecd2e..000000000 --- a/libs/runtime_backup/src/runtime_backup/runner.py +++ /dev/null @@ -1,197 +0,0 @@ -"""Runtime backup service. - -Polls runtime/ every TICK_INTERVAL_SECONDS; if there are uncommitted changes, -makes a backup commit on the orphan branch checked out at runtime/ and (when -GH_TOKEN is set) pushes it to origin. - -The orphan branch (mindsbackup/$MNGR_AGENT_ID) and the worktree at runtime/ -are created by libs/bootstrap during its pre-services init step, so this -service can assume runtime/ is already a git worktree on that branch. -""" - -import os -import subprocess -import time -from datetime import datetime, timezone -from pathlib import Path - -from loguru import logger - -RUNTIME_DIR = Path("runtime") -TICK_INTERVAL_SECONDS = 60 -LOG_FILE = Path("/tmp/runtime-backup.log") - -# Minimum age before an index.lock is treated as stale and removed. A real git -# operation on the small runtime/ tree holds the lock for well under a second, -# so a lock older than this cannot belong to a live operation. Set to the tick -# interval so a lock skipped as possibly-live on one tick is guaranteed old -# enough to clear on the next. -STALE_LOCK_MIN_AGE_SECONDS = TICK_INTERVAL_SECONDS - - -def _git_noninteractive_env() -> dict[str, str]: - """Environment for backup git calls: never prompt for credentials. - - GIT_TERMINAL_PROMPT=0 turns any credential prompt (e.g. a push to a - private origin whose token went stale) into a fast failure instead of a - TTY prompt that would wedge the backup loop forever; every caller already - logs-and-continues on a nonzero exit. Mirrors the helper of the same name - in bootstrap.manager (kept local so this lib stays dependency-free). - """ - return {**os.environ, "GIT_TERMINAL_PROMPT": "0"} - - -def _git(*args: str) -> subprocess.CompletedProcess[str]: - """Run a git command inside the runtime worktree, never raising or prompting.""" - return subprocess.run( - ["git", "-C", str(RUNTIME_DIR), *args], - capture_output=True, - text=True, - check=False, - env=_git_noninteractive_env(), - ) - - -def _clear_stale_index_lock() -> None: - """Remove a stale ``index.lock`` from the runtime worktree, if present. - - runtime-backup is the only writer of this worktree's git index and its - ticks run strictly sequentially, so a lock here is normally stale -- left - behind by a previous tick's git process that was killed before it could - release the lock (whenever something kills the process mid-commit). Git - never clears such a lock itself, so without this every subsequent - ``git add`` fails identically and backups stop forever. - - The removal is deliberately *not* unconditional: to stay safe even if the - single-writer assumption is ever violated (a concurrent git process in the - worktree), the lock is only removed once it is older than - ``STALE_LOCK_MIN_AGE_SECONDS``. A live git operation on the small runtime/ - tree holds the lock for well under a second, so an older lock cannot be - live; a genuinely in-progress operation is left untouched and reconsidered - on the next tick. - - The lock path is resolved via ``--absolute-git-dir`` so it is correct - whether runtime/ is a normal repo or (as in production) a linked worktree - with a per-worktree git dir. - """ - git_dir_result = _git("rev-parse", "--absolute-git-dir") - if git_dir_result.returncode != 0: - # runtime/ is not a git repo yet; nothing to clear. - return - lock_path = Path(git_dir_result.stdout.strip()) / "index.lock" - try: - lock_age_seconds = time.time() - lock_path.stat().st_mtime - except OSError: - # No lock present (the common case), or it vanished underneath us. - return - if lock_age_seconds < STALE_LOCK_MIN_AGE_SECONDS: - logger.warning( - "git index.lock at {} is only {:.0f}s old; leaving it in case a " - "git operation is in progress (will reconsider next tick)", - lock_path, - lock_age_seconds, - ) - return - logger.warning( - "Removing stale git index lock at {} ({:.0f}s old)", - lock_path, - lock_age_seconds, - ) - try: - lock_path.unlink() - except OSError as e: - logger.warning("Failed to remove stale index lock at {}: {}", lock_path, e) - - -def _has_uncommitted_changes() -> bool: - """Return True if runtime/ has anything to commit.""" - result = _git("status", "--porcelain") - if result.returncode != 0: - logger.warning( - "git status failed (rc={}): {}", result.returncode, result.stderr.strip() - ) - return False - return bool(result.stdout.strip()) - - -def _now_iso_utc() -> str: - """Current UTC time as ISO-8601 with a trailing Z (e.g. 2026-05-06T17:42:13Z).""" - return ( - datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") - ) - - -def _do_tick(should_push: bool) -> None: - """Run one backup tick: add, commit-if-dirty, push-if-token.""" - # Clear any stale index.lock first, so a commit interrupted by something - # killing the process cannot wedge every future tick. - _clear_stale_index_lock() - - add_result = _git("add", "-A") - if add_result.returncode != 0: - logger.warning( - "git add failed (rc={}): {}", - add_result.returncode, - add_result.stderr.strip(), - ) - return - - if _has_uncommitted_changes(): - commit_result = _git("commit", "-m", f"runtime backup: {_now_iso_utc()}") - if commit_result.returncode != 0: - logger.warning( - "git commit failed (rc={}): {}", - commit_result.returncode, - commit_result.stderr.strip(), - ) - # Fall through to push: any prior unpushed commits should still - # be shipped even if this tick's commit failed. - - if should_push: - # Always attempt push: covers the case where a prior tick committed but - # failed to push, so the next tick still ships the unpushed commit. - # Bootstrap's initial `--set-upstream` push is best-effort; if it - # failed, plain `git push` will fail forever with "no upstream". Fall - # back to `--set-upstream origin ` to self-heal that case - # (mirrors the post-commit hook's chain). - push_result = _git("push") - if push_result.returncode != 0: - branch_result = _git("symbolic-ref", "--short", "HEAD") - branch = branch_result.stdout.strip() - if branch_result.returncode == 0 and branch: - push_result = _git("push", "--set-upstream", "origin", branch) - if push_result.returncode != 0: - logger.warning( - "git push failed (rc={}): {}", - push_result.returncode, - push_result.stderr.strip(), - ) - - -def main() -> None: - """Main loop: poll runtime/ on a fixed interval and back up changes.""" - # Tee stderr-bound logs into LOG_FILE so operators can `tail` the file - # across restarts of just this service window. /tmp wipes on container - # restart, which is the intended scope for the debug log. Set up here - # rather than at module import so that merely importing this module - # (e.g. from tests) does not start writing to the log file. - logger.add(LOG_FILE, level="INFO") - - logger.info("Starting runtime-backup (interval={}s)", TICK_INTERVAL_SECONDS) - - if not (RUNTIME_DIR / ".git").exists(): - logger.warning( - "runtime/ is not a git worktree; bootstrap should have created it" - ) - - has_token = bool(os.environ.get("GH_TOKEN")) - if not has_token: - logger.info("No GH_TOKEN; will commit locally but skip push") - - while True: - _do_tick(should_push=has_token) - time.sleep(TICK_INTERVAL_SECONDS) - - -if __name__ == "__main__": - main() diff --git a/libs/runtime_backup/src/runtime_backup/runner_test.py b/libs/runtime_backup/src/runtime_backup/runner_test.py deleted file mode 100644 index 1286b4ce6..000000000 --- a/libs/runtime_backup/src/runtime_backup/runner_test.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Unit tests for the runtime-backup runner. - -Focus: the stale-index-lock recovery that keeps an interrupted commit from -permanently wedging every future backup tick, and the age guard that keeps it -from disturbing a genuinely in-progress git operation. -""" - -from __future__ import annotations - -import os -import subprocess -import time -from pathlib import Path - -import pytest - -from runtime_backup.runner import ( - STALE_LOCK_MIN_AGE_SECONDS, - _clear_stale_index_lock, - _do_tick, -) - - -def _git(repo: Path, *args: str) -> None: - """Run a git command in `repo`, raising on failure.""" - subprocess.run(["git", "-C", str(repo), *args], check=True, capture_output=True) - - -def _init_repo(repo: Path) -> None: - """Create a git repo at `repo` with a committer identity configured.""" - repo.mkdir(parents=True, exist_ok=True) - subprocess.run(["git", "init", "-q", str(repo)], check=True, capture_output=True) - _git(repo, "config", "user.email", "test@test.local") - _git(repo, "config", "user.name", "test") - - -def _age_lock(lock_path: Path) -> None: - """Backdate a lock file's mtime so it counts as stale (past the age guard).""" - old = time.time() - STALE_LOCK_MIN_AGE_SECONDS - 60 - os.utime(lock_path, (old, old)) - - -def test_clear_stale_index_lock_removes_aged_lock_in_linked_worktree( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """Faithful to production: runtime/ is a linked worktree, so the lock - lives in the per-worktree git dir, not in a top-level .git/.""" - monkeypatch.chdir(tmp_path) - main = tmp_path / "main" - _init_repo(main) - (main / "seed.txt").write_text("seed\n") - _git(main, "add", "-A") - _git(main, "commit", "-qm", "seed") - # The runner resolves runtime/ relative to its cwd, so the worktree must - # be named exactly "runtime" and sit in tmp_path. - _git(main, "worktree", "add", str(tmp_path / "runtime"), "-b", "backup") - - lock_path = main / ".git" / "worktrees" / "runtime" / "index.lock" - lock_path.write_text("") - _age_lock(lock_path) - assert lock_path.exists() - - _clear_stale_index_lock() - - assert not lock_path.exists() - - -def test_clear_stale_index_lock_keeps_fresh_lock( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A recently-created lock might belong to a live git operation, so it - must be left alone rather than yanked out from under that operation.""" - monkeypatch.chdir(tmp_path) - runtime = tmp_path / "runtime" - _init_repo(runtime) - lock_path = runtime / ".git" / "index.lock" - lock_path.write_text("") # fresh: mtime is now - - _clear_stale_index_lock() - - assert lock_path.exists() - - -def test_clear_stale_index_lock_noop_when_no_lock_present( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - _init_repo(tmp_path / "runtime") - # No lock and no git repo problems -- must simply not raise. - _clear_stale_index_lock() - - -def test_clear_stale_index_lock_noop_when_runtime_not_a_repo( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.chdir(tmp_path) - (tmp_path / "runtime").mkdir() - # `git rev-parse` fails; the function must silently return. - _clear_stale_index_lock() - - -def test_do_tick_self_heals_stale_index_lock( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - """A stale index.lock from a killed prior tick must not wedge backups: - the next tick clears it and commits the pending runtime state.""" - monkeypatch.chdir(tmp_path) - runtime = tmp_path / "runtime" - _init_repo(runtime) - # A leftover lock from a prior tick's git process that was killed; aged - # past the guard so it is recognized as stale. - lock_path = runtime / ".git" / "index.lock" - lock_path.write_text("") - _age_lock(lock_path) - # New runtime state waiting to be backed up. - (runtime / "memory.txt").write_text("important state\n") - - _do_tick(should_push=False) - - log = subprocess.run( - ["git", "-C", str(runtime), "log", "--oneline"], - check=True, - capture_output=True, - text=True, - ) - assert "runtime backup:" in log.stdout diff --git a/pyproject.toml b/pyproject.toml index c63e4b581..2ce47d968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ dependencies = [ "cloudflare-tunnel", "app-watcher", "web-server", - "runtime-backup", + "github-sync", "host-backup", "tk-command-parsing", "imbue-common", @@ -55,14 +55,14 @@ package = false override-dependencies = ["openai>=2.20.0"] [tool.uv.workspace] -members = ["libs/bootstrap", "libs/cloudflare_tunnel", "libs/app_watcher", "libs/web_server", "libs/runtime_backup", "libs/host_backup", "libs/mngr_cli_contract", "libs/tk_command_parsing", "apps/system_interface", "libs/browser"] +members = ["libs/bootstrap", "libs/cloudflare_tunnel", "libs/app_watcher", "libs/web_server", "libs/github_sync", "libs/host_backup", "libs/mngr_cli_contract", "libs/tk_command_parsing", "apps/system_interface", "libs/browser"] [tool.uv.sources] bootstrap = { workspace = true } cloudflare-tunnel = { workspace = true } app-watcher = { workspace = true } web-server = { workspace = true } -runtime-backup = { workspace = true } +github-sync = { workspace = true } host-backup = { workspace = true } mngr-cli-contract = { workspace = true } tk-command-parsing = { workspace = true } diff --git a/scripts/claude_open_tickets_reminder.sh b/scripts/claude_open_tickets_reminder.sh index bcb3e38e1..734ee2633 100755 --- a/scripts/claude_open_tickets_reminder.sh +++ b/scripts/claude_open_tickets_reminder.sh @@ -14,7 +14,7 @@ set -euo pipefail repo_root="${MNGR_AGENT_WORK_DIR:-$(pwd)}" # Honor any externally-set TICKETS_DIR (the agent's env normally pins it # via .mngr/settings.toml -- e.g. /code/runtime/tickets -- so the tk -# tickets live alongside the runtime-backup branch). Fall back to tk's +# tickets live under runtime/ with the rest of the synced state). Fall back to tk's # unset-default of /.tickets when nothing is set. tickets_dir="${TICKETS_DIR:-${repo_root}/.tickets}" diff --git a/scripts/claude_open_tickets_stop_nudge.sh b/scripts/claude_open_tickets_stop_nudge.sh index 63075d5d0..0cbcd3ca3 100755 --- a/scripts/claude_open_tickets_stop_nudge.sh +++ b/scripts/claude_open_tickets_stop_nudge.sh @@ -9,7 +9,7 @@ set -euo pipefail repo_root="${MNGR_AGENT_WORK_DIR:-$(pwd)}" # Honor any externally-set TICKETS_DIR (the agent's env normally pins it # via .mngr/settings.toml -- e.g. /code/runtime/tickets -- so the tk -# tickets live alongside the runtime-backup branch). Fall back to tk's +# tickets live under runtime/ with the rest of the synced state). Fall back to tk's # unset-default of /.tickets when nothing is set. tickets_dir="${TICKETS_DIR:-${repo_root}/.tickets}" diff --git a/scripts/git_hooks/post-commit b/scripts/git_hooks/post-commit index 324a9c01d..dcd232378 100755 --- a/scripts/git_hooks/post-commit +++ b/scripts/git_hooks/post-commit @@ -1,16 +1,25 @@ #!/usr/bin/env bash -# Auto-push the active branch after every commit, gated on GH_TOKEN being set. +# Auto-push the active branch after every commit, part of the opt-in GitHub +# sync (see libs/github_sync/README.md and the github-sync skill). # -# Installed for every checkout in the container via: -# git config --global core.hooksPath /mngr/code/scripts/git_hooks -# (set up by the git_auth_setup extra_window in .mngr/settings.toml). +# This hook is inert by default: it only takes effect once the github-sync +# skill points `git config --global core.hooksPath` at this directory (done +# by `uv run github-sync wire-git`). The same wiring makes plain `git push` +# authenticate through the latchkey gateway, so this hook needs no +# credentials of its own -- no token ever enters the container. # -# Skips silently in three cases that are normal, not errors: -# 1. GH_TOKEN is unset (e.g. local dev, or worker sub-agents that should -# never push). +# Because core.hooksPath is global, the hook fires for every checkout in the +# container: the main /mngr/code repo and every worker worktree, each pushing +# its own current branch. +# +# Skips the push in these cases -- 1-3 are normal, not errors, and skip +# silently; 4 is a security halt and logs a line to the log file: +# 1. GitHub sync is not configured (github_sync.toml absent). # 2. HEAD is detached (no branch to push to). -# 3. The current branch is under mindsbackup/ -- those branches belong to -# the runtime-backup polling service, which handles its own pushes. +# 3. The current branch is runtime-sync -- that branch belongs to the +# github-sync polling service, which handles its own pushes. +# 4. The github-sync service has halted pushes because the sync repo is +# public or its visibility could not be confirmed. # # Runs the push in the background so the commit never blocks. All output # (success or failure) is appended to /tmp/post-commit-push.log for debugging; @@ -18,10 +27,23 @@ set -u -LOG_FILE=/tmp/post-commit-push.log +# Never prompt for credentials on the controlling TTY: the gateway wiring +# authenticates pushes via injected headers, so a prompt (e.g. after a 401 +# from a lapsed gateway permission) can never succeed -- it would only write +# a stray username prompt into the committing agent's tmux session and hang +# the background push forever. With prompts disabled the push fails fast and +# the failure lands in the log file. +export GIT_TERMINAL_PROMPT=0 + +# Env overrides exist for tests; production always uses the defaults. +# GITHUB_SYNC_STATUS_FILE is the same override the github-sync service honors +# when writing the file this hook reads. +CONFIG_FILE="${GITHUB_SYNC_CONFIG_FILE:-/mngr/code/github_sync.toml}" +STATUS_FILE="${GITHUB_SYNC_STATUS_FILE:-/tmp/github-sync-status.json}" +LOG_FILE="${GITHUB_SYNC_HOOK_LOG_FILE:-/tmp/post-commit-push.log}" -# 1. No token, nothing to do. -if [ -z "${GH_TOKEN:-}" ]; then +# 1. Sync not configured, nothing to do. +if [ ! -f "$CONFIG_FILE" ]; then exit 0 fi @@ -31,17 +53,45 @@ if [ -z "$branch" ]; then exit 0 fi -# 3. Runtime backup branches push themselves via the runtime-backup service. +# 3. The runtime-sync branch pushes itself via the github-sync service. case "$branch" in - mindsbackup/*) exit 0 ;; + runtime-sync) exit 0 ;; esac +# 4. Respect a visibility halt: the service refuses to push while the sync +# repo is public (or unconfirmed), and so must we. A missing status file or +# field reads as "null"/empty, which is not "false", so the default is to +# allow (the service simply hasn't reported yet). Note: jq's `//` operator +# must NOT be used here -- `.is_push_allowed // true` would turn an +# explicit `false` into `true` and defeat the halt. +if [ -f "$STATUS_FILE" ] && command -v jq >/dev/null 2>&1; then + allowed="$(jq -r '.is_push_allowed' "$STATUS_FILE" 2>/dev/null)" + if [ "$allowed" = "false" ]; then + echo "[$(date -Is)] skipping push of $branch: sync repo not confirmed private" >> "$LOG_FILE" + exit 0 + fi +fi + # Background push, never blocking the commit. First try a plain push; if the -# branch has no upstream yet, fall back to setting upstream as we push. +# branch has no upstream yet, fall back to setting upstream as we push. If the +# primary latchkey gateway is unreachable and a secondary (per-VPS) gateway +# exists, push the proxied URL directly through it (password header only -- +# the secondary takes no permissions override). { echo "[$(date -Is)] post-commit push for $branch" if ! git push 2>&1; then - git push --set-upstream origin "$branch" 2>&1 + if ! git push --set-upstream origin "$branch" 2>&1; then + repo_url="$(sed -n 's/^repo_url *= *"\(.*\)".*/\1/p' "$CONFIG_FILE")" + # Normalize like github_sync.config.load_repo_url does, so a + # repo_url written with a trailing slash or .git suffix still + # composes a valid proxied URL below. + repo_url="${repo_url%/}" + repo_url="${repo_url%.git}" + if [ -n "${LATCHKEY_GATEWAY_SECONDARY:-}" ] && [ -n "${LATCHKEY_GATEWAY_PASSWORD:-}" ] && [ -n "$repo_url" ]; then + git -c "http.extraHeader=X-Latchkey-Gateway-Password: $LATCHKEY_GATEWAY_PASSWORD" \ + push "${LATCHKEY_GATEWAY_SECONDARY%/}/gateway/$repo_url.git" "$branch" 2>&1 + fi + fi fi } >> "$LOG_FILE" 2>&1 & diff --git a/supervisord.conf b/supervisord.conf index f8a6f9ed3..e814507d5 100644 --- a/supervisord.conf +++ b/supervisord.conf @@ -6,8 +6,8 @@ # # Conventions: # - Services inherit the agent environment (MNGR_AGENT_STATE_DIR, -# CLAUDE_CONFIG_DIR, MNGR_HOST_DIR, MNGR_AGENT_ID, GH_TOKEN, ...) from the -# bootstrap shell that exec'd supervisord -- there is no per-service +# CLAUDE_CONFIG_DIR, MNGR_HOST_DIR, MNGR_AGENT_ID, LATCHKEY_*, ...) from +# the bootstrap shell that exec'd supervisord -- there is no per-service # `environment=` enumeration. # - All commands run from the repo root (directory=/mngr/code). # - Commands that chain with `&&` (or set inline env) are wrapped in @@ -108,24 +108,10 @@ stderr_logfile_maxbytes=10MB stdout_logfile_backups=3 stderr_logfile_backups=3 -# Periodically commits + pushes runtime/ to mindsbackup/$MNGR_AGENT_ID so that -# Claude memory, ticket state, transcripts, etc. survive container loss. The -# runtime/ worktree is created by bootstrap before supervisord starts, so this -# service always finds it in place. See libs/runtime_backup/README.md. -[program:runtime-backup] -command=uv run runtime-backup -directory=/mngr/code -autostart=true -autorestart=true -startretries=1000000 -stopasgroup=true -killasgroup=true -stdout_logfile=/var/log/supervisor/runtime-backup-stdout.log -stderr_logfile=/var/log/supervisor/runtime-backup-stderr.log -stdout_logfile_maxbytes=10MB -stderr_logfile_maxbytes=10MB -stdout_logfile_backups=3 -stderr_logfile_backups=3 +# NOTE: GitHub sync ([program:github-sync]) is opt-in and deliberately NOT in +# this default config. The github-sync skill appends its program block (and +# sets up the private repo, git wiring, and runtime/ worktree) when the user +# asks for it. See libs/github_sync/README.md. # Continuous restic-based backup of the full host_dir (/mngr/) to a remote # repository. runtime/secrets/restic.env is injected by the minds app; diff --git a/uv.lock b/uv.lock index 4bb33df13..b9f134622 100644 --- a/uv.lock +++ b/uv.lock @@ -29,9 +29,9 @@ members = [ "browser", "cloudflare-tunnel", "default-workspace-template", + "github-sync", "host-backup", "mngr-cli-contract", - "runtime-backup", "system-interface", "tk-command-parsing", "web-server", @@ -942,11 +942,11 @@ dependencies = [ { name = "bootstrap" }, { name = "browser" }, { name = "cloudflare-tunnel" }, + { name = "github-sync" }, { name = "host-backup" }, { name = "imbue-common" }, { name = "litellm" }, { name = "playwright" }, - { name = "runtime-backup" }, { name = "tk-command-parsing" }, { name = "tomlkit" }, { name = "web-server" }, @@ -973,11 +973,11 @@ requires-dist = [ { name = "bootstrap", editable = "libs/bootstrap" }, { name = "browser", editable = "libs/browser" }, { name = "cloudflare-tunnel", editable = "libs/cloudflare_tunnel" }, + { name = "github-sync", editable = "libs/github_sync" }, { name = "host-backup", editable = "libs/host_backup" }, { name = "imbue-common", editable = "vendor/mngr/libs/imbue_common" }, { name = "litellm", specifier = ">=1.88.1" }, { name = "playwright", specifier = "==1.58.0" }, - { name = "runtime-backup", editable = "libs/runtime_backup" }, { name = "tk-command-parsing", editable = "libs/tk_command_parsing" }, { name = "tomlkit", specifier = ">=0.12" }, { name = "web-server", editable = "libs/web_server" }, @@ -1264,6 +1264,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8e/b9/7dd37b6001d16f692b1bfb6e68cad642beb38b34a753c29bbff312f46e4b/gevent-26.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:d1c08bc9bb6bd79732a26710a99588b5e9b67b668e165dd609704b876f41baab", size = 1703189, upload-time = "2026-04-08T22:48:31.713Z" }, ] +[[package]] +name = "github-sync" +version = "0.1.0" +source = { editable = "libs/github_sync" } +dependencies = [ + { name = "click" }, + { name = "loguru" }, +] + +[package.metadata] +requires-dist = [ + { name = "click", specifier = ">=8.0" }, + { name = "loguru", specifier = ">=0.7.0" }, +] + [[package]] name = "google-api-core" version = "2.29.0" @@ -6375,17 +6390,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/28/7e/61c42657f6e4614a4258f1c3b0c5b93adc4d1f8575f5229d1906b483099b/ruff-0.12.12-py3-none-win_arm64.whl", hash = "sha256:2a8199cab4ce4d72d158319b63370abf60991495fb733db96cd923a34c52d093", size = 12256762, upload-time = "2025-09-04T16:50:15.737Z" }, ] -[[package]] -name = "runtime-backup" -version = "0.1.0" -source = { editable = "libs/runtime_backup" } -dependencies = [ - { name = "loguru" }, -] - -[package.metadata] -requires-dist = [{ name = "loguru", specifier = ">=0.7.0" }] - [[package]] name = "s3transfer" version = "0.19.1"