Workspace sync: E2E-encrypted cross-device sync of workspace metadata and secrets

.github/workflows/ci.yml

@@ -753,7 +753,21 @@ jobs:
       # The Anthropic key (for live checks) is always needed; masked in the logs
       # and forwarded into the offload sandbox by the just recipe below. Read via
       # the minds-scoped minds_ci_test_gh role.
+      # HCP Vault's performance replicas intermittently 412 ("required index
+      # state not present"); one spaced retry absorbs the burst.
       - name: Fetch Anthropic key from Vault
+        id: anthropic_key_fetch
+        continue-on-error: true
+        uses: imbue-ai/use-vault-secrets@main
+        with:
+          role: minds_ci_test_gh
+          secrets: |
+            minds/ci/litellm/ANTHROPIC_API_KEY
+      - name: Pause before the Vault retry
+        if: ${{ steps.anthropic_key_fetch.outcome == 'failure' }}
+        run: sleep 15
+      - name: Fetch Anthropic key from Vault (retry)
+        if: ${{ steps.anthropic_key_fetch.outcome == 'failure' }}
         uses: imbue-ai/use-vault-secrets@main
         with:
           role: minds_ci_test_gh
@@ -807,6 +821,67 @@ jobs:
             cargo install offload@0.9.10 --force
           fi
 
+      # -- Sync e2e env plumbing (opt-in, same gate as build-minds-ci-env) --
+      # The workspace-sync e2e tests (apps/minds/test_sync_e2e.py) run INSIDE
+      # the offload sandbox but against the per-run ci env's real connector.
+      # On opt-in runs, resolve the env's coordinates from the env-config
+      # artifact and its SuperTokens admin secrets from Vault, and export them
+      # as MINDS_SYNC_E2E_* so the just recipe forwards them into the sandbox
+      # (the tests skip when the vars are absent). Secret values are masked by
+      # the use-vault-secrets action even though offload --trace echoes args.
+      # `!cancelled()` keeps the artifact available to the later
+      # minds_services step even when an unrelated earlier step failed
+      # (matching that step's own `!cancelled()` gate).
+      - name: Download deployment env config for the sync e2e tests
+        if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }}
+        uses: actions/download-artifact@v7
+        with:
+          name: minds-ci-env-config
+          path: .minds
+      - name: Resolve sync e2e env coordinates
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }}
+        id: sync_e2e_env
+        run: |
+          set -euo pipefail
+          CONFIG=.minds/iterate-default.json
+          {
+            echo "env_name=$(jq -er '.shared_envs.default.env_name' "$CONFIG")"
+            echo "connector_url=$(jq -er '.shared_envs.default.connector_url' "$CONFIG")"
+            echo "litellm_url=$(jq -er '.shared_envs.default.litellm_proxy_url' "$CONFIG")"
+          } >> "$GITHUB_OUTPUT"
+      - name: Fetch sync e2e env secrets from Vault
+        id: sync_e2e_secrets
+        continue-on-error: true
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }}
+        uses: imbue-ai/use-vault-secrets@main
+        with:
+          role: minds_ci_test_gh
+          secrets: |
+            minds/ci/runs/${{ steps.sync_e2e_env.outputs.env_name }}/shared-default/SUPERTOKENS_CONNECTION_URI
+            minds/ci/runs/${{ steps.sync_e2e_env.outputs.env_name }}/shared-default/SUPERTOKENS_API_KEY
+      - name: Pause before the sync e2e Vault retry
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests && steps.sync_e2e_secrets.outcome == 'failure' }}
+        run: sleep 15
+      - name: Fetch sync e2e env secrets from Vault (retry)
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests && steps.sync_e2e_secrets.outcome == 'failure' }}
+        uses: imbue-ai/use-vault-secrets@main
+        with:
+          role: minds_ci_test_gh
+          secrets: |
+            minds/ci/runs/${{ steps.sync_e2e_env.outputs.env_name }}/shared-default/SUPERTOKENS_CONNECTION_URI
+            minds/ci/runs/${{ steps.sync_e2e_env.outputs.env_name }}/shared-default/SUPERTOKENS_API_KEY
+      - name: Export sync e2e env vars for the offload stage
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }}
+        run: |
+          set -euo pipefail
+          test -n "${SUPERTOKENS_CONNECTION_URI:-}" || { echo "Vault secrets missing after retry" >&2; exit 1; }
+          {
+            echo "MINDS_SYNC_E2E_CONNECTOR_URL=${{ steps.sync_e2e_env.outputs.connector_url }}"
+            echo "MINDS_SYNC_E2E_LITELLM_URL=${{ steps.sync_e2e_env.outputs.litellm_url }}"
+            echo "MINDS_SYNC_E2E_SUPERTOKENS_CONNECTION_URI=${SUPERTOKENS_CONNECTION_URI}"
+            echo "MINDS_SYNC_E2E_SUPERTOKENS_API_KEY=${SUPERTOKENS_API_KEY}"
+          } >> "$GITHUB_ENV"
+
       - name: Run minds snapshot-resume tests via offload
         id: tests
         run: just test-offload-minds-snapshot "${{ needs.build-minds-snapshot.outputs.image_id }}"
@@ -835,12 +910,8 @@ jobs:
           VAULT_TOKEN=$(vault write -field=token auth/jwt_github_actions/login role=minds_ci_test_gh jwt="$OIDC")
           echo "::add-mask::$VAULT_TOKEN"
           printf '%s' "$VAULT_TOKEN" > "$HOME/.vault-token"
-      - name: Download deployment env config
-        if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }}
-        uses: actions/download-artifact@v7
-        with:
-          name: minds-ci-env-config
-          path: .minds
+      # The env config was already downloaded to .minds/ by the sync-e2e
+      # plumbing step before the offload run (same opt-in gate).
       - name: Run minds_services tests against the per-run ci env
         id: services_tests
         if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }}

apps/minds/changelog/mngr-account-association.md

@@ -0,0 +1,13 @@
+Workspace/account state now syncs across a user's devices, end-to-end encrypted. Each associated workspace has a record on the Imbue Cloud connector: plaintext metadata (name, color, provider, which device hosts it) plus a secrets blob (the workspace's SSH key material and its backup `restic.env`) encrypted under a per-account data key that only the user's master password can unwrap.
+
+The master password's only role is now wrapping that per-account key: new backup repositories are initialized with the workspace's own random password as their single key, the per-workspace "backup encryption method" / master-password inputs are gone from the create form and workspace settings, and the Settings page's password change rewraps the key (and pushes/clears synced secrets) instead of rekeying every repository. Clearing the password scrubs all synced secrets server-side; while the password is empty, only metadata syncs.
+
+The landing page and sidebar list workspaces that live on the user's other devices as greyed rows with an "on <device>" badge and a remove-from-list action; their backup status and download still work here (the backup credentials are materialized from the synced record once the account is unlocked). A new-device unlock banner asks for the master password once and installs the account key.
+
+Reconcile also heals a missing server-side key bundle: if this device holds the password-wrapped account key but the connector has none (the one-shot legacy conversion wraps the carried-over password locally, and only the settings password-change flow used to push bundles), the bundle is uploaded. Without this, a converted legacy install would sync its encrypted secrets while no other device -- and no reinstall after a machine loss -- could ever unlock them. An existing server bundle always wins, so a rewrap done on another device is never clobbered.
+
+Legacy local state (`workspace_associations.json` -- or the older `sessions.json` layout when that file never existed -- plus `backup_password_hash` and `backup_password`) is converted once at startup and renamed aside with a `.pre-sync` suffix. Destroying a workspace now tombstones its record (metadata and secrets kept) so its backups remain reachable from any device.
+
+New Electron-driven release tests (`apps/minds/test_sync_e2e.py`, run in the minds-snapshot sandbox against the per-run CI connector env on `run_minds_release_tests` runs): a full machine-loss-and-recover lifecycle (real imbue-cloud backups to R2, master password, wipe everything, sign back in, unlock, download and byte-verify the backup), the legacy-file one-shot migration, and the master-password lifecycle (rewrap-only change, clear-scrubs, re-set re-pushes).
+
+CI cleanup now sweeps the R2 buckets imbue-cloud backups leave behind. Nothing ever deleted them (env destroy tears down Modal, Neon, SuperTokens and Cloudflare tunnels, but not buckets), so every CI run that exercised backups leaked one permanently. The `cleanup-minds-ci-envs` stage now also deletes buckets whose owning account no longer exists -- emptying each one first, since Cloudflare refuses to delete a non-empty bucket. Because the dev and CI tiers share a Cloudflare account, the rule is deliberately positive rather than heuristic: a bucket is only swept when its owner prefix matches no user in any SuperTokens app on the shared core, and the sweep fails closed (an unreachable SuperTokens, or an empty live-owner set, aborts before deleting anything) so a developer's own backups can never be caught in it.

apps/minds/conftest.py

@@ -22,14 +22,29 @@
 import subprocess
 from collections.abc import Iterator
 from pathlib import Path
+from uuid import uuid4
 
+import httpx
 import pytest
+from loguru import logger
+from pydantic import AnyUrl
+from pydantic import SecretStr
 
 from imbue.imbue_common.conftest_hooks import register_conftest_hooks
 from imbue.imbue_common.conftest_hooks import register_marker
+from imbue.imbue_common.primitives import NonEmptyStr
+from imbue.minds.deployment_tests.helpers import create_verified_user_via_admin_api
+from imbue.minds.deployment_tests.helpers import delete_user_via_admin_api
+from imbue.minds.testing import SYNC_E2E_CONNECTOR_URL_ENV
+from imbue.minds.testing import SYNC_E2E_LITELLM_URL_ENV
+from imbue.minds.testing import SYNC_E2E_SUPERTOKENS_API_KEY_ENV
+from imbue.minds.testing import SYNC_E2E_SUPERTOKENS_URI_ENV
+from imbue.minds.testing import SyncE2EAccount
+from imbue.minds.testing import SyncE2EEnv
 from imbue.mngr.utils.logging import suppress_warnings
 from imbue.mngr.utils.plugin_testing import register_plugin_test_fixtures
 from imbue.mngr.utils.testing import generate_test_environment_name
+from imbue.mngr.utils.testing import get_short_random_string
 
 # Point ``MINDS_RESTIC_BINARY`` at the bundled ``resources/restic/restic``
 # binary so restic_cli tests don't require a system-wide restic install.
@@ -92,6 +107,44 @@ def mngr_test_prefix() -> str:
     return f"{generate_test_environment_name()}-"
 
 
+_SNAPSHOT_START_DOCKERD_SCRIPT = Path("/code/mngr/libs/mngr/imbue/mngr/resources/start-dockerd.sh")
+_SNAPSHOT_DOCKERD_STARTUP_TIMEOUT_SECONDS = 180
+
+
+@pytest.fixture(scope="session")
+def snapshot_sandbox_dockerd() -> None:
+    """Bring ``dockerd`` back up in a snapshot-resumed sandbox.
+
+    ``sandbox.snapshot_filesystem`` only captures the disk, not running
+    processes -- so a sandbox booted from the minds-workspace snapshot has
+    ``/var/lib/docker`` populated (with the stopped workspace container's
+    image layers + on-disk state) but no ``dockerd`` running. Re-run the same
+    script the snapshot itself used to bring dockerd up, so ``docker``
+    invocations in the requesting tests can talk to the daemon.
+
+    Shared by ``test_snapshot_resume.py`` (via its module-autouse wrapper) and
+    ``test_sync_e2e.py`` (requested explicitly). Mirrors
+    ``_ensure_dockerd_for_release`` in ``libs/mngr/imbue/mngr/conftest.py``
+    but for the snapshot's in-tree script location.
+    """
+    docker_info = subprocess.run(["docker", "info"], capture_output=True)
+    if docker_info.returncode == 0:
+        return
+
+    if not _SNAPSHOT_START_DOCKERD_SCRIPT.is_file():
+        raise FileNotFoundError(
+            f"start-dockerd.sh not found at {_SNAPSHOT_START_DOCKERD_SCRIPT}; this fixture is "
+            "only useful inside a sandbox booted from scripts/snapshot_minds_e2e_state.py."
+        )
+
+    subprocess.run(["chmod", "+x", str(_SNAPSHOT_START_DOCKERD_SCRIPT)], check=True, timeout=5)
+    subprocess.run(
+        [str(_SNAPSHOT_START_DOCKERD_SCRIPT)],
+        check=True,
+        timeout=_SNAPSHOT_DOCKERD_STARTUP_TIMEOUT_SECONDS,
+    )
+
+
 class _XvfbStartupError(RuntimeError):
     """Raised when the Xvfb display server fails to start for an Electron test."""
 
@@ -148,3 +201,62 @@ def xvfb_display() -> Iterator[str]:
             process.wait(timeout=10)
         except subprocess.TimeoutExpired:
             process.kill()
+
+
+@pytest.fixture
+def sync_e2e_env() -> SyncE2EEnv:
+    """The real connector env the workspace-sync e2e tests target, or skip.
+
+    The coordinates are forwarded into the snapshot offload sandbox only on
+    ``run_minds_release_tests`` CI runs (and can be exported by an operator
+    pointing at a dev env for local iteration); on every other run the vars
+    are absent and the sync e2e tests skip.
+    """
+    values: dict[str, str] = {}
+    for env_var in (
+        SYNC_E2E_CONNECTOR_URL_ENV,
+        SYNC_E2E_LITELLM_URL_ENV,
+        SYNC_E2E_SUPERTOKENS_URI_ENV,
+        SYNC_E2E_SUPERTOKENS_API_KEY_ENV,
+    ):
+        value = os.environ.get(env_var)
+        if not value:
+            pytest.skip(f"{env_var} is not set; the sync e2e tests need a real connector env")
+        values[env_var] = value
+    return SyncE2EEnv(
+        connector_url=values[SYNC_E2E_CONNECTOR_URL_ENV],
+        litellm_proxy_url=values[SYNC_E2E_LITELLM_URL_ENV],
+        supertokens_connection_uri=SecretStr(values[SYNC_E2E_SUPERTOKENS_URI_ENV]),
+        supertokens_api_key=SecretStr(values[SYNC_E2E_SUPERTOKENS_API_KEY_ENV]),
+    )
+
+
+@pytest.fixture
+def sync_e2e_account(sync_e2e_env: SyncE2EEnv) -> Iterator[SyncE2EAccount]:
+    """A unique, pre-verified, paid account on the sync e2e env; deleted on teardown.
+
+    The address lives under ``imbue.com`` because the ci/dev deploy tiers seed
+    that domain into ``paid_domains`` -- imbue-cloud backups (R2 bucket
+    provisioning) are paid-gated, and these tests exercise them for real. The
+    account is provisioned through the SuperTokens admin API (setup machinery,
+    not part of the user journey under test); the tests then sign in through
+    the real UI with the returned email + password.
+    """
+    email = f"sync-e2e-{get_short_random_string()}@imbue.com"
+    password = SecretStr(f"pw-{uuid4().hex}")
+    user_id, access_token = create_verified_user_via_admin_api(
+        connection_uri=sync_e2e_env.supertokens_connection_uri,
+        api_key=sync_e2e_env.supertokens_api_key,
+        connector_url=AnyUrl(sync_e2e_env.connector_url),
+        email=NonEmptyStr(email),
+        password=password,
+    )
+    yield SyncE2EAccount(email=email, password=password, user_id=str(user_id), access_token=access_token)
+    try:
+        delete_user_via_admin_api(
+            connection_uri=sync_e2e_env.supertokens_connection_uri,
+            api_key=sync_e2e_env.supertokens_api_key,
+            user_id=NonEmptyStr(str(user_id)),
+        )
+    except httpx.HTTPError as e:
+        logger.warning("Could not delete sync e2e account {}: {}", email, e)

apps/minds/imbue/minds/cli/run.py

@@ -80,11 +80,15 @@
 from imbue.minds.desktop_client.server import serve_desktop_client
 from imbue.minds.desktop_client.session_store import MultiAccountSessionStore
 from imbue.minds.desktop_client.state import get_state
+from imbue.minds.desktop_client.sync_scheduler import WorkspaceSyncScheduler
 from imbue.minds.desktop_client.system_interface_health import SystemInterfaceHealthTracker
 from imbue.minds.desktop_client.system_interface_health import should_enroll_suspect_for_backend_failure
 from imbue.minds.desktop_client.templates import DEFAULT_WORKSPACE_TEMPLATE_GIT_URL
 from imbue.minds.desktop_client.templates import FALLBACK_BRANCH
 from imbue.minds.desktop_client.templates import is_local_workspace_defaults_opt_in
+from imbue.minds.desktop_client.workspace_record_store import WorkspaceRecordStore
+from imbue.minds.desktop_client.workspace_record_store import read_device_id
+from imbue.minds.desktop_client.workspace_record_store import read_device_label
 from imbue.minds.envs.docker_cleanup import DockerCleanupError
 from imbue.minds.envs.docker_cleanup import start_active_env_state_container
 from imbue.minds.primitives import OneTimeCode
@@ -383,7 +387,22 @@ def run(
         mngr_caller=mngr_caller,
         connector_url=client_env_config.connector_url,
     )
-    session_store = MultiAccountSessionStore(data_dir=data_directory, cli=imbue_cloud_cli)
+    workspace_record_store = WorkspaceRecordStore(
+        paths=paths,
+        mngr_host_dir=mngr_host_dir,
+        cli=imbue_cloud_cli,
+        device_id=read_device_id(mngr_host_dir),
+        device_label=read_device_label(),
+    )
+    session_store = MultiAccountSessionStore(
+        data_dir=data_directory, cli=imbue_cloud_cli, record_store=workspace_record_store
+    )
+    sync_scheduler = WorkspaceSyncScheduler(
+        record_store=workspace_record_store,
+        session_store=session_store,
+        resolver=backend_resolver,
+    )
+    sync_scheduler.start(root_concurrency_group)
     response_events = load_response_events(data_directory)
     request_inbox = RequestInbox()
     for resp in response_events:
@@ -563,6 +582,7 @@ def run(
         minds_api_key=minds_api_key,
         latchkey_forward_supervisor=latchkey_forward_supervisor,
         discovery_health_watchdog=discovery_health_watchdog,
+        sync_scheduler=sync_scheduler,
     )
 
     # Background loop driving the discovery-pipeline watchdog: polls snapshot

apps/minds/imbue/minds/desktop_client/agent_creator.py

@@ -1404,7 +1404,7 @@ def start_creation(
         branch_or_tag: str = "",
         region: str = "",
         anthropic_api_key: str = "",
-        on_created: Callable[[AgentId], None] | None = None,
+        on_created: Callable[[AgentId, HostId], None] | None = None,
         backup_request: BackupSetupRequest | None = None,
         color: str | None = None,
         docker_runtime: DockerRuntime = DockerRuntime.RUNC,
@@ -1437,10 +1437,12 @@ def start_creation(
         ask for.
 
         When ``on_created`` is provided, it is called with the canonical
-        ``AgentId`` once ``mngr create`` returns (immediately before the
-        status flips to ``DONE``). The id is parsed from the inner
-        ``mngr create``'s JSONL ``"event": "created"`` line, not pre-generated;
-        for imbue_cloud agents it's the leased pool host's pre-baked id.
+        ``AgentId`` and ``HostId`` once ``mngr create`` returns (immediately
+        after the status flips to ``DONE``, so consumers can rely on the
+        published canonical id). Both ids are parsed from the
+        inner ``mngr create``'s JSONL ``"event": "created"`` line, not
+        pre-generated; for imbue_cloud agents they are the leased pool
+        host's pre-baked ids.
 
         Returns a ``CreationId`` immediately for tracking the in-flight
         creation. Use ``get_creation_info()`` to poll status (and read
@@ -1550,7 +1552,7 @@ def _create_agent_background(
         branch_or_tag: str = "",
         region: str = "",
         anthropic_api_key: str = "",
-        on_created: Callable[[AgentId], None] | None = None,
+        on_created: Callable[[AgentId, HostId], None] | None = None,
         backup_request: BackupSetupRequest | None = None,
         color: str | None = None,
         docker_runtime: DockerRuntime = DockerRuntime.RUNC,
@@ -1889,7 +1891,7 @@ def _create_agent_background(
                     self._redirect_urls[cid_str] = redirect_url
 
                 if on_created is not None:
-                    on_created(canonical_id)
+                    on_created(canonical_id, canonical_host_id)
 
                 # Configure restic backups asynchronously on a detached
                 # thread (mirrors the Cloudflare tunnel-token path): bucket