diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c80f537657..49b7c083ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -753,7 +753,21 @@ jobs: # The Anthropic key (for live checks) is always needed; masked in the logs # and forwarded into the offload sandbox by the just recipe below. Read via # the minds-scoped minds_ci_test_gh role. + # HCP Vault's performance replicas intermittently 412 ("required index + # state not present"); one spaced retry absorbs the burst. - name: Fetch Anthropic key from Vault + id: anthropic_key_fetch + continue-on-error: true + uses: imbue-ai/use-vault-secrets@main + with: + role: minds_ci_test_gh + secrets: | + minds/ci/litellm/ANTHROPIC_API_KEY + - name: Pause before the Vault retry + if: ${{ steps.anthropic_key_fetch.outcome == 'failure' }} + run: sleep 15 + - name: Fetch Anthropic key from Vault (retry) + if: ${{ steps.anthropic_key_fetch.outcome == 'failure' }} uses: imbue-ai/use-vault-secrets@main with: role: minds_ci_test_gh @@ -807,6 +821,67 @@ jobs: cargo install offload@0.9.10 --force fi + # -- Sync e2e env plumbing (opt-in, same gate as build-minds-ci-env) -- + # The workspace-sync e2e tests (apps/minds/test_sync_e2e.py) run INSIDE + # the offload sandbox but against the per-run ci env's real connector. + # On opt-in runs, resolve the env's coordinates from the env-config + # artifact and its SuperTokens admin secrets from Vault, and export them + # as MINDS_SYNC_E2E_* so the just recipe forwards them into the sandbox + # (the tests skip when the vars are absent). Secret values are masked by + # the use-vault-secrets action even though offload --trace echoes args. + # `!cancelled()` keeps the artifact available to the later + # minds_services step even when an unrelated earlier step failed + # (matching that step's own `!cancelled()` gate). + - name: Download deployment env config for the sync e2e tests + if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }} + uses: actions/download-artifact@v7 + with: + name: minds-ci-env-config + path: .minds + - name: Resolve sync e2e env coordinates + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }} + id: sync_e2e_env + run: | + set -euo pipefail + CONFIG=.minds/iterate-default.json + { + echo "env_name=$(jq -er '.shared_envs.default.env_name' "$CONFIG")" + echo "connector_url=$(jq -er '.shared_envs.default.connector_url' "$CONFIG")" + echo "litellm_url=$(jq -er '.shared_envs.default.litellm_proxy_url' "$CONFIG")" + } >> "$GITHUB_OUTPUT" + - name: Fetch sync e2e env secrets from Vault + id: sync_e2e_secrets + continue-on-error: true + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }} + uses: imbue-ai/use-vault-secrets@main + with: + role: minds_ci_test_gh + secrets: | + minds/ci/runs/${{ steps.sync_e2e_env.outputs.env_name }}/shared-default/SUPERTOKENS_CONNECTION_URI + minds/ci/runs/${{ steps.sync_e2e_env.outputs.env_name }}/shared-default/SUPERTOKENS_API_KEY + - name: Pause before the sync e2e Vault retry + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests && steps.sync_e2e_secrets.outcome == 'failure' }} + run: sleep 15 + - name: Fetch sync e2e env secrets from Vault (retry) + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests && steps.sync_e2e_secrets.outcome == 'failure' }} + uses: imbue-ai/use-vault-secrets@main + with: + role: minds_ci_test_gh + secrets: | + minds/ci/runs/${{ steps.sync_e2e_env.outputs.env_name }}/shared-default/SUPERTOKENS_CONNECTION_URI + minds/ci/runs/${{ steps.sync_e2e_env.outputs.env_name }}/shared-default/SUPERTOKENS_API_KEY + - name: Export sync e2e env vars for the offload stage + if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }} + run: | + set -euo pipefail + test -n "${SUPERTOKENS_CONNECTION_URI:-}" || { echo "Vault secrets missing after retry" >&2; exit 1; } + { + echo "MINDS_SYNC_E2E_CONNECTOR_URL=${{ steps.sync_e2e_env.outputs.connector_url }}" + echo "MINDS_SYNC_E2E_LITELLM_URL=${{ steps.sync_e2e_env.outputs.litellm_url }}" + echo "MINDS_SYNC_E2E_SUPERTOKENS_CONNECTION_URI=${SUPERTOKENS_CONNECTION_URI}" + echo "MINDS_SYNC_E2E_SUPERTOKENS_API_KEY=${SUPERTOKENS_API_KEY}" + } >> "$GITHUB_ENV" + - name: Run minds snapshot-resume tests via offload id: tests run: just test-offload-minds-snapshot "${{ needs.build-minds-snapshot.outputs.image_id }}" @@ -835,12 +910,8 @@ jobs: VAULT_TOKEN=$(vault write -field=token auth/jwt_github_actions/login role=minds_ci_test_gh jwt="$OIDC") echo "::add-mask::$VAULT_TOKEN" printf '%s' "$VAULT_TOKEN" > "$HOME/.vault-token" - - name: Download deployment env config - if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }} - uses: actions/download-artifact@v7 - with: - name: minds-ci-env-config - path: .minds + # The env config was already downloaded to .minds/ by the sync-e2e + # plumbing step before the offload run (same opt-in gate). - name: Run minds_services tests against the per-run ci env id: services_tests if: ${{ !cancelled() && github.event_name == 'workflow_dispatch' && inputs.run_minds_release_tests }} diff --git a/apps/minds/changelog/mngr-account-association.md b/apps/minds/changelog/mngr-account-association.md new file mode 100644 index 0000000000..2454c29e4b --- /dev/null +++ b/apps/minds/changelog/mngr-account-association.md @@ -0,0 +1,13 @@ +Workspace/account state now syncs across a user's devices, end-to-end encrypted. Each associated workspace has a record on the Imbue Cloud connector: plaintext metadata (name, color, provider, which device hosts it) plus a secrets blob (the workspace's SSH key material and its backup `restic.env`) encrypted under a per-account data key that only the user's master password can unwrap. + +The master password's only role is now wrapping that per-account key: new backup repositories are initialized with the workspace's own random password as their single key, the per-workspace "backup encryption method" / master-password inputs are gone from the create form and workspace settings, and the Settings page's password change rewraps the key (and pushes/clears synced secrets) instead of rekeying every repository. Clearing the password scrubs all synced secrets server-side; while the password is empty, only metadata syncs. + +The landing page and sidebar list workspaces that live on the user's other devices as greyed rows with an "on " badge and a remove-from-list action; their backup status and download still work here (the backup credentials are materialized from the synced record once the account is unlocked). A new-device unlock banner asks for the master password once and installs the account key. + +Reconcile also heals a missing server-side key bundle: if this device holds the password-wrapped account key but the connector has none (the one-shot legacy conversion wraps the carried-over password locally, and only the settings password-change flow used to push bundles), the bundle is uploaded. Without this, a converted legacy install would sync its encrypted secrets while no other device -- and no reinstall after a machine loss -- could ever unlock them. An existing server bundle always wins, so a rewrap done on another device is never clobbered. + +Legacy local state (`workspace_associations.json` -- or the older `sessions.json` layout when that file never existed -- plus `backup_password_hash` and `backup_password`) is converted once at startup and renamed aside with a `.pre-sync` suffix. Destroying a workspace now tombstones its record (metadata and secrets kept) so its backups remain reachable from any device. + +New Electron-driven release tests (`apps/minds/test_sync_e2e.py`, run in the minds-snapshot sandbox against the per-run CI connector env on `run_minds_release_tests` runs): a full machine-loss-and-recover lifecycle (real imbue-cloud backups to R2, master password, wipe everything, sign back in, unlock, download and byte-verify the backup), the legacy-file one-shot migration, and the master-password lifecycle (rewrap-only change, clear-scrubs, re-set re-pushes). + +CI cleanup now sweeps the R2 buckets imbue-cloud backups leave behind. Nothing ever deleted them (env destroy tears down Modal, Neon, SuperTokens and Cloudflare tunnels, but not buckets), so every CI run that exercised backups leaked one permanently. The `cleanup-minds-ci-envs` stage now also deletes buckets whose owning account no longer exists -- emptying each one first, since Cloudflare refuses to delete a non-empty bucket. Because the dev and CI tiers share a Cloudflare account, the rule is deliberately positive rather than heuristic: a bucket is only swept when its owner prefix matches no user in any SuperTokens app on the shared core, and the sweep fails closed (an unreachable SuperTokens, or an empty live-owner set, aborts before deleting anything) so a developer's own backups can never be caught in it. diff --git a/apps/minds/conftest.py b/apps/minds/conftest.py index 9bc97a2589..275999e9a8 100644 --- a/apps/minds/conftest.py +++ b/apps/minds/conftest.py @@ -22,14 +22,29 @@ import subprocess from collections.abc import Iterator from pathlib import Path +from uuid import uuid4 +import httpx import pytest +from loguru import logger +from pydantic import AnyUrl +from pydantic import SecretStr from imbue.imbue_common.conftest_hooks import register_conftest_hooks from imbue.imbue_common.conftest_hooks import register_marker +from imbue.imbue_common.primitives import NonEmptyStr +from imbue.minds.deployment_tests.helpers import create_verified_user_via_admin_api +from imbue.minds.deployment_tests.helpers import delete_user_via_admin_api +from imbue.minds.testing import SYNC_E2E_CONNECTOR_URL_ENV +from imbue.minds.testing import SYNC_E2E_LITELLM_URL_ENV +from imbue.minds.testing import SYNC_E2E_SUPERTOKENS_API_KEY_ENV +from imbue.minds.testing import SYNC_E2E_SUPERTOKENS_URI_ENV +from imbue.minds.testing import SyncE2EAccount +from imbue.minds.testing import SyncE2EEnv from imbue.mngr.utils.logging import suppress_warnings from imbue.mngr.utils.plugin_testing import register_plugin_test_fixtures from imbue.mngr.utils.testing import generate_test_environment_name +from imbue.mngr.utils.testing import get_short_random_string # Point ``MINDS_RESTIC_BINARY`` at the bundled ``resources/restic/restic`` # binary so restic_cli tests don't require a system-wide restic install. @@ -92,6 +107,44 @@ def mngr_test_prefix() -> str: return f"{generate_test_environment_name()}-" +_SNAPSHOT_START_DOCKERD_SCRIPT = Path("/code/mngr/libs/mngr/imbue/mngr/resources/start-dockerd.sh") +_SNAPSHOT_DOCKERD_STARTUP_TIMEOUT_SECONDS = 180 + + +@pytest.fixture(scope="session") +def snapshot_sandbox_dockerd() -> None: + """Bring ``dockerd`` back up in a snapshot-resumed sandbox. + + ``sandbox.snapshot_filesystem`` only captures the disk, not running + processes -- so a sandbox booted from the minds-workspace snapshot has + ``/var/lib/docker`` populated (with the stopped workspace container's + image layers + on-disk state) but no ``dockerd`` running. Re-run the same + script the snapshot itself used to bring dockerd up, so ``docker`` + invocations in the requesting tests can talk to the daemon. + + Shared by ``test_snapshot_resume.py`` (via its module-autouse wrapper) and + ``test_sync_e2e.py`` (requested explicitly). Mirrors + ``_ensure_dockerd_for_release`` in ``libs/mngr/imbue/mngr/conftest.py`` + but for the snapshot's in-tree script location. + """ + docker_info = subprocess.run(["docker", "info"], capture_output=True) + if docker_info.returncode == 0: + return + + if not _SNAPSHOT_START_DOCKERD_SCRIPT.is_file(): + raise FileNotFoundError( + f"start-dockerd.sh not found at {_SNAPSHOT_START_DOCKERD_SCRIPT}; this fixture is " + "only useful inside a sandbox booted from scripts/snapshot_minds_e2e_state.py." + ) + + subprocess.run(["chmod", "+x", str(_SNAPSHOT_START_DOCKERD_SCRIPT)], check=True, timeout=5) + subprocess.run( + [str(_SNAPSHOT_START_DOCKERD_SCRIPT)], + check=True, + timeout=_SNAPSHOT_DOCKERD_STARTUP_TIMEOUT_SECONDS, + ) + + class _XvfbStartupError(RuntimeError): """Raised when the Xvfb display server fails to start for an Electron test.""" @@ -148,3 +201,62 @@ def xvfb_display() -> Iterator[str]: process.wait(timeout=10) except subprocess.TimeoutExpired: process.kill() + + +@pytest.fixture +def sync_e2e_env() -> SyncE2EEnv: + """The real connector env the workspace-sync e2e tests target, or skip. + + The coordinates are forwarded into the snapshot offload sandbox only on + ``run_minds_release_tests`` CI runs (and can be exported by an operator + pointing at a dev env for local iteration); on every other run the vars + are absent and the sync e2e tests skip. + """ + values: dict[str, str] = {} + for env_var in ( + SYNC_E2E_CONNECTOR_URL_ENV, + SYNC_E2E_LITELLM_URL_ENV, + SYNC_E2E_SUPERTOKENS_URI_ENV, + SYNC_E2E_SUPERTOKENS_API_KEY_ENV, + ): + value = os.environ.get(env_var) + if not value: + pytest.skip(f"{env_var} is not set; the sync e2e tests need a real connector env") + values[env_var] = value + return SyncE2EEnv( + connector_url=values[SYNC_E2E_CONNECTOR_URL_ENV], + litellm_proxy_url=values[SYNC_E2E_LITELLM_URL_ENV], + supertokens_connection_uri=SecretStr(values[SYNC_E2E_SUPERTOKENS_URI_ENV]), + supertokens_api_key=SecretStr(values[SYNC_E2E_SUPERTOKENS_API_KEY_ENV]), + ) + + +@pytest.fixture +def sync_e2e_account(sync_e2e_env: SyncE2EEnv) -> Iterator[SyncE2EAccount]: + """A unique, pre-verified, paid account on the sync e2e env; deleted on teardown. + + The address lives under ``imbue.com`` because the ci/dev deploy tiers seed + that domain into ``paid_domains`` -- imbue-cloud backups (R2 bucket + provisioning) are paid-gated, and these tests exercise them for real. The + account is provisioned through the SuperTokens admin API (setup machinery, + not part of the user journey under test); the tests then sign in through + the real UI with the returned email + password. + """ + email = f"sync-e2e-{get_short_random_string()}@imbue.com" + password = SecretStr(f"pw-{uuid4().hex}") + user_id, access_token = create_verified_user_via_admin_api( + connection_uri=sync_e2e_env.supertokens_connection_uri, + api_key=sync_e2e_env.supertokens_api_key, + connector_url=AnyUrl(sync_e2e_env.connector_url), + email=NonEmptyStr(email), + password=password, + ) + yield SyncE2EAccount(email=email, password=password, user_id=str(user_id), access_token=access_token) + try: + delete_user_via_admin_api( + connection_uri=sync_e2e_env.supertokens_connection_uri, + api_key=sync_e2e_env.supertokens_api_key, + user_id=NonEmptyStr(str(user_id)), + ) + except httpx.HTTPError as e: + logger.warning("Could not delete sync e2e account {}: {}", email, e) diff --git a/apps/minds/imbue/minds/cli/run.py b/apps/minds/imbue/minds/cli/run.py index 418518a343..15b6fdea25 100644 --- a/apps/minds/imbue/minds/cli/run.py +++ b/apps/minds/imbue/minds/cli/run.py @@ -80,11 +80,15 @@ from imbue.minds.desktop_client.server import serve_desktop_client from imbue.minds.desktop_client.session_store import MultiAccountSessionStore from imbue.minds.desktop_client.state import get_state +from imbue.minds.desktop_client.sync_scheduler import WorkspaceSyncScheduler from imbue.minds.desktop_client.system_interface_health import SystemInterfaceHealthTracker from imbue.minds.desktop_client.system_interface_health import should_enroll_suspect_for_backend_failure from imbue.minds.desktop_client.templates import DEFAULT_WORKSPACE_TEMPLATE_GIT_URL from imbue.minds.desktop_client.templates import FALLBACK_BRANCH from imbue.minds.desktop_client.templates import is_local_workspace_defaults_opt_in +from imbue.minds.desktop_client.workspace_record_store import WorkspaceRecordStore +from imbue.minds.desktop_client.workspace_record_store import read_device_id +from imbue.minds.desktop_client.workspace_record_store import read_device_label from imbue.minds.envs.docker_cleanup import DockerCleanupError from imbue.minds.envs.docker_cleanup import start_active_env_state_container from imbue.minds.primitives import OneTimeCode @@ -383,7 +387,22 @@ def run( mngr_caller=mngr_caller, connector_url=client_env_config.connector_url, ) - session_store = MultiAccountSessionStore(data_dir=data_directory, cli=imbue_cloud_cli) + workspace_record_store = WorkspaceRecordStore( + paths=paths, + mngr_host_dir=mngr_host_dir, + cli=imbue_cloud_cli, + device_id=read_device_id(mngr_host_dir), + device_label=read_device_label(), + ) + session_store = MultiAccountSessionStore( + data_dir=data_directory, cli=imbue_cloud_cli, record_store=workspace_record_store + ) + sync_scheduler = WorkspaceSyncScheduler( + record_store=workspace_record_store, + session_store=session_store, + resolver=backend_resolver, + ) + sync_scheduler.start(root_concurrency_group) response_events = load_response_events(data_directory) request_inbox = RequestInbox() for resp in response_events: @@ -563,6 +582,7 @@ def run( minds_api_key=minds_api_key, latchkey_forward_supervisor=latchkey_forward_supervisor, discovery_health_watchdog=discovery_health_watchdog, + sync_scheduler=sync_scheduler, ) # Background loop driving the discovery-pipeline watchdog: polls snapshot diff --git a/apps/minds/imbue/minds/desktop_client/agent_creator.py b/apps/minds/imbue/minds/desktop_client/agent_creator.py index abc93331e4..5c54d1cc99 100644 --- a/apps/minds/imbue/minds/desktop_client/agent_creator.py +++ b/apps/minds/imbue/minds/desktop_client/agent_creator.py @@ -1404,7 +1404,7 @@ def start_creation( branch_or_tag: str = "", region: str = "", anthropic_api_key: str = "", - on_created: Callable[[AgentId], None] | None = None, + on_created: Callable[[AgentId, HostId], None] | None = None, backup_request: BackupSetupRequest | None = None, color: str | None = None, docker_runtime: DockerRuntime = DockerRuntime.RUNC, @@ -1437,10 +1437,12 @@ def start_creation( ask for. When ``on_created`` is provided, it is called with the canonical - ``AgentId`` once ``mngr create`` returns (immediately before the - status flips to ``DONE``). The id is parsed from the inner - ``mngr create``'s JSONL ``"event": "created"`` line, not pre-generated; - for imbue_cloud agents it's the leased pool host's pre-baked id. + ``AgentId`` and ``HostId`` once ``mngr create`` returns (immediately + after the status flips to ``DONE``, so consumers can rely on the + published canonical id). Both ids are parsed from the + inner ``mngr create``'s JSONL ``"event": "created"`` line, not + pre-generated; for imbue_cloud agents they are the leased pool + host's pre-baked ids. Returns a ``CreationId`` immediately for tracking the in-flight creation. Use ``get_creation_info()`` to poll status (and read @@ -1550,7 +1552,7 @@ def _create_agent_background( branch_or_tag: str = "", region: str = "", anthropic_api_key: str = "", - on_created: Callable[[AgentId], None] | None = None, + on_created: Callable[[AgentId, HostId], None] | None = None, backup_request: BackupSetupRequest | None = None, color: str | None = None, docker_runtime: DockerRuntime = DockerRuntime.RUNC, @@ -1889,7 +1891,7 @@ def _create_agent_background( self._redirect_urls[cid_str] = redirect_url if on_created is not None: - on_created(canonical_id) + on_created(canonical_id, canonical_host_id) # Configure restic backups asynchronously on a detached # thread (mirrors the Cloudflare tunnel-token path): bucket diff --git a/apps/minds/imbue/minds/desktop_client/api_models.py b/apps/minds/imbue/minds/desktop_client/api_models.py index 773bd67f0d..1585ee449e 100644 --- a/apps/minds/imbue/minds/desktop_client/api_models.py +++ b/apps/minds/imbue/minds/desktop_client/api_models.py @@ -15,7 +15,6 @@ from pydantic import ConfigDict from pydantic import Field -from pydantic import SecretStr from pydantic import StrictBool from imbue.imbue_common.frozen_model import FrozenModel @@ -128,20 +127,6 @@ class BackupServiceConfigureRequest(ApiRequestModel): """Body for enabling backups or changing a workspace's backup destination.""" backup_provider: str = Field(description="'IMBUE_CLOUD' or 'API_KEY'") - master_password: SecretStr = Field( - default=SecretStr(""), - description=( - "The master password, validated against the stored hash. Blank falls back to the saved " - "plaintext copy when one exists, else means the empty password." - ), - ) - save_password: bool = Field( - default=False, - description=( - "Persist the typed (and just-validated) master password locally so later flows don't require retyping. " - "Never establishes or changes the master password." - ), - ) api_key_env: str = Field(default="", description="For API_KEY: KEY=VALUE block (RESTIC_REPOSITORY + creds)") @@ -206,17 +191,6 @@ class CreateWorkspaceRequest(ApiRequestModel): backup_provider: BackupProvider | None = Field( default=None, description="Restic backup provider (default CONFIGURE_LATER)" ) - backup_master_password: SecretStr | None = Field( - default=None, - description=( - "Master/recovery passphrase, validated against the stored hash. Blank/absent falls back to the " - "saved copy when one exists, else means the empty password. Agents should always create with " - "backups unconfigured and never ask the user for this." - ), - ) - backup_save_password: bool | None = Field( - default=None, description="Persist the typed (just-validated) master password locally for later flows" - ) backup_api_key_env: str | None = Field(default=None, description="KEY=VALUE block for an API_KEY backup provider") diff --git a/apps/minds/imbue/minds/desktop_client/api_v1.py b/apps/minds/imbue/minds/desktop_client/api_v1.py index e87061d3fa..567544123d 100644 --- a/apps/minds/imbue/minds/desktop_client/api_v1.py +++ b/apps/minds/imbue/minds/desktop_client/api_v1.py @@ -38,7 +38,6 @@ from flask import request from loguru import logger from pydantic import Field -from pydantic import SecretStr from imbue.concurrency_group.concurrency_group import ConcurrencyGroup from imbue.concurrency_group.concurrency_group import ConcurrencyGroupError @@ -411,6 +410,22 @@ def _check_backup_service_safely( return backup_verification.BackupServiceCheck(state=backup_verification.BackupServiceCheckState.UNKNOWN) +def _materialize_env_from_record_if_missing(paths: WorkspacePaths, parsed_id: AgentId) -> None: + """Best-effort: write the backup env from the workspace's synced record. + + Lets backup status / export work for workspaces this device never + provisioned (hosted on another device, or destroyed elsewhere), provided + the account is unlocked here. A miss is fine -- the caller degrades to + the ordinary not-configured behavior. + """ + session_store = get_state().session_store + if session_store is None or session_store.record_store is None: + return + if has_canonical_env(paths, parsed_id): + return + session_store.record_store.materialize_env_from_record(str(parsed_id)) + + @require_api_or_cookie_auth @API_SPEC.validate(resp=json_response_model(WorkspaceBackupsResponse)) def _handle_workspace_backups(agent_id: str) -> WorkspaceBackupsResponse | Response: @@ -427,6 +442,7 @@ def _handle_workspace_backups(agent_id: str) -> WorkspaceBackupsResponse | Respo paths: WorkspacePaths | None = state.api_v1_paths if paths is None: return _json_error("Backups are not configured", 501) + _materialize_env_from_record_if_missing(paths, parsed_id) check_results: list[backup_verification.BackupServiceCheck] = [] resolver = state.backend_resolver @@ -478,6 +494,7 @@ def _handle_workspace_backup_export(agent_id: str, snapshot_id: str) -> Response paths: WorkspacePaths | None = get_state().api_v1_paths if paths is None: return _json_error("Backups are not configured", 501) + _materialize_env_from_record_if_missing(paths, parsed_id) backend_resolver = get_state().backend_resolver info = backend_resolver.get_agent_display_info(parsed_id) host_id = info.host_id if info is not None else str(parsed_id) @@ -527,7 +544,6 @@ def _handle_create_workspace() -> tuple[OperationHandleResponse, int] | Response Backup provisioning and Cloudflare tunnel injection match the desktop UI's create flow: the optional ``backup_*`` fields (``backup_provider``, - ``backup_master_password``, ``backup_save_password``, ``backup_api_key_env``) build the same restic setup request, and -- when an ``account_id`` is given -- the same post-creation callback associates the peer with the account and injects a @@ -566,10 +582,6 @@ def _handle_create_workspace() -> tuple[OperationHandleResponse, int] | Response backup_provider = BackupProvider(str(body.get("backup_provider", BackupProvider.CONFIGURE_LATER.value))) except ValueError: return _json_error(f"Invalid backup_provider: {body.get('backup_provider')!r}", 400) - # Wrapped in SecretStr immediately so the plaintext never rides a local - # that could end up in a log or an error message. - backup_master_password = SecretStr(str(body.get("backup_master_password") or "")) - is_save_backup_password = bool(body.get("backup_save_password", False)) backup_api_key_env = str(body.get("backup_api_key_env", "")) account_id = str(body.get("account_id", "")).strip() anthropic_api_key = str(body.get("anthropic_api_key", "")).strip() @@ -621,19 +633,16 @@ def _handle_create_workspace() -> tuple[OperationHandleResponse, int] | Response if account_id and session_store is not None: account_email = session_store.get_account_email(account_id) or "" - # Build the same restic setup request the create form builds (validates the - # master password against the stored hash; optionally saves the plaintext - # convenience copy). Fail fast on a bad config. + # Build the same restic setup request the create form builds. Fail fast on + # a bad config. No password is involved: repositories are keyed by each + # workspace's own random password. backup_request, backup_error = build_backup_request_or_error( backup_provider=backup_provider, - typed_master_password=backup_master_password, - is_save_password=is_save_backup_password, api_key_env=backup_api_key_env, account_email=account_email, - paths=agent_creator.paths, ) if backup_error is not None: - return _json_field_error(backup_error, "backup_master_password") + return _json_field_error(backup_error, "backup_api_key_env") # For imbue_cloud compute the lease needs the resolved template version # (the latest semver tag when no branch was given), matching the form path. @@ -647,7 +656,9 @@ def _handle_create_workspace() -> tuple[OperationHandleResponse, int] | Response # and persists the chosen region -- exactly as the create form does. minds_config = get_state().minds_config region = resolve_effective_region(launch_mode, submitted_region, minds_config, get_state().geo_location_cache) - on_created = build_create_on_created_callback(account_id, minds_config, launch_mode, region) + on_created = build_create_on_created_callback( + account_id, minds_config, launch_mode, region, display_name=host_name or resolved_host_name, color=color + ) creation_id = agent_creator.start_creation( git_url, @@ -1188,11 +1199,8 @@ def _handle_backup_service_configure(agent_id: str) -> tuple[OperationHandleResp backup_request, error_message = build_backup_request_or_error( backup_provider=backup_provider, - typed_master_password=SecretStr(str(body.get("master_password") or "")), - is_save_password=bool(body.get("save_password", False)), api_key_env=str(body.get("api_key_env", "")), account_email=account_email, - paths=paths, ) if backup_request is None or error_message is not None: return _json_error(error_message or "Invalid backup configuration", 400) diff --git a/apps/minds/imbue/minds/desktop_client/api_v1_test.py b/apps/minds/imbue/minds/desktop_client/api_v1_test.py index 1b944baaaf..b5b1cba65d 100644 --- a/apps/minds/imbue/minds/desktop_client/api_v1_test.py +++ b/apps/minds/imbue/minds/desktop_client/api_v1_test.py @@ -56,6 +56,7 @@ from imbue.minds.testing import stub_mngr_host_dir from imbue.minds.utils.testing import RecordingMngrCaller from imbue.mngr.primitives import AgentId +from imbue.mngr.primitives import HostId from imbue.mngr_forward.ssh_tunnel import RemoteSSHInfo _TEST_KEY = "test-minds-api-key" @@ -110,7 +111,7 @@ def start_creation( branch_or_tag: str = "", region: str = "", anthropic_api_key: str = "", - on_created: Callable[[AgentId], None] | None = None, + on_created: Callable[[AgentId, HostId], None] | None = None, backup_request: BackupSetupRequest | None = None, color: str | None = None, docker_runtime: DockerRuntime = DockerRuntime.RUNC, @@ -914,7 +915,14 @@ def _associated_session_store( """Build a session store with one signed-in account that owns ``agent_id``.""" cli.add_account(user_id=user_id, email=email) store = make_session_store_for_test(tmp_path / "sessions", cli=cli) - store.associate_workspace(user_id, str(agent_id)) + store.associate_created_workspace( + user_id=user_id, + agent_id=str(agent_id), + host_id=str(HostId.generate()), + display_name="", + color=None, + is_cloud_row=False, + ) return store diff --git a/apps/minds/imbue/minds/desktop_client/app.py b/apps/minds/imbue/minds/desktop_client/app.py index 6aa8fd2956..7e1546ce73 100644 --- a/apps/minds/imbue/minds/desktop_client/app.py +++ b/apps/minds/imbue/minds/desktop_client/app.py @@ -42,13 +42,11 @@ from imbue.minds.desktop_client.backend_resolver import AgentDisplayInfo from imbue.minds.desktop_client.backend_resolver import BackendResolverInterface from imbue.minds.desktop_client.backend_resolver import MngrCliBackendResolver -from imbue.minds.desktop_client.backup_password_rotation import rotate_backup_master_password -from imbue.minds.desktop_client.backup_password_store import ensure_backup_password_hash -from imbue.minds.desktop_client.backup_password_store import has_saved_backup_password -from imbue.minds.desktop_client.backup_password_store import is_master_password_set from imbue.minds.desktop_client.cookie_manager import SESSION_COOKIE_NAME from imbue.minds.desktop_client.cookie_manager import create_session_cookie from imbue.minds.desktop_client.cookie_manager import verify_session_cookie +from imbue.minds.desktop_client.dek_store import is_master_password_set_for_account +from imbue.minds.desktop_client.dek_store import set_master_password_for_account from imbue.minds.desktop_client.destroying import DestroyingStatus from imbue.minds.desktop_client.destroying import delete_destroying from imbue.minds.desktop_client.destroying import is_host_still_active @@ -59,6 +57,7 @@ from imbue.minds.desktop_client.forward_cli import EnvelopeStreamConsumer from imbue.minds.desktop_client.help_modal_requests import OpenHelpRequest from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCli +from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCliError from imbue.minds.desktop_client.latchkey.gateway_client import LatchkeyGatewayClientError from imbue.minds.desktop_client.latchkey.handlers.predefined import LatchkeyPermissionGrantHandler from imbue.minds.desktop_client.latchkey.permission_overview import PermissionOverviewError @@ -100,8 +99,10 @@ from imbue.minds.desktop_client.state import set_state from imbue.minds.desktop_client.supertokens_routes import create_supertokens_blueprint from imbue.minds.desktop_client.supertokens_routes import signout_user_via_plugin +from imbue.minds.desktop_client.sync_scheduler import WorkspaceSyncScheduler from imbue.minds.desktop_client.system_interface_health import AgentHealth from imbue.minds.desktop_client.system_interface_health import SystemInterfaceHealthTracker +from imbue.minds.desktop_client.templates import RemoteWorkspaceTile from imbue.minds.desktop_client.templates import render_accounts_page from imbue.minds.desktop_client.templates import render_auth_error_page from imbue.minds.desktop_client.templates import render_chrome_page @@ -128,7 +129,11 @@ from imbue.minds.desktop_client.workspace_color import DEFAULT_WORKSPACE_COLOR from imbue.minds.desktop_client.workspace_color import pick_unused_create_color from imbue.minds.desktop_client.workspace_create import default_region_for_provider_with_config +from imbue.minds.desktop_client.workspace_record_store import RECORD_STATE_ACTIVE +from imbue.minds.desktop_client.workspace_record_store import WorkspaceRecordStore from imbue.minds.errors import InvalidJsonBodyError +from imbue.minds.errors import SyncCryptoError +from imbue.minds.errors import WorkspaceSyncError from imbue.minds.primitives import CreationId from imbue.minds.primitives import LaunchMode from imbue.minds.primitives import OneTimeCode @@ -436,14 +441,49 @@ def _handle_error_reporting_settings() -> Response: return make_response(status_code=200, content='{"ok": true}', media_type="application/json") +def _push_new_password_state( + record_store: WorkspaceRecordStore, + resolver: BackendResolverInterface, + user_id: str, + account_email: str, + bundle: Mapping[str, object], +) -> None: + """A non-empty password was just set: push the new bundle + any pending secrets.""" + if record_store.cli is not None: + record_store.cli.sync_bundle_push(account_email, bundle) + record_store.push_all_secrets(user_id, account_email, resolver) + + +def _scrub_cleared_password_server_state(record_store: WorkspaceRecordStore, account_email: str) -> None: + """The password was cleared: nothing secret may stay server-side.""" + if record_store.cli is None: + return + record_store.cli.sync_bundle_delete(account_email) + record_store.cli.sync_scrub_secrets(account_email) + + +def _is_any_account_password_set(paths: WorkspacePaths | None) -> bool: + """Whether any signed-in account has a non-empty master password (per its bundle mirror).""" + if paths is None: + return False + session_store = get_state().session_store + if session_store is None: + return False + return any( + is_master_password_set_for_account(paths, str(account.user_id)) for account in session_store.list_accounts() + ) + + def _handle_backup_password_change() -> Response: - """Rotate the shared backup master password (POST /_chrome/backup-password). + """Change the sync master password (POST /_chrome/backup-password). Deliberately a desktop-only cookie-auth route (not part of /api/v1): agents - must never be able to rotate the master password. The rotation is - synchronous -- it rekeys every existing backed-up workspace's repository -- - and the response carries per-workspace results for the Settings page to - render inline. + must never be able to change the master password. The password's only role + is wrapping each signed-in account's sync DEK: a change rewraps the DEK and + pushes the new bundle (plus any pending secrets) to the connector; clearing + the password deletes the server bundle and scrubs the synced secrets. + Workspace repositories are never touched. The response carries per-account + results for the Settings page to render inline. """ if not _is_request_authenticated(): return make_response(status_code=403, content='{"error":"Not authenticated"}', media_type="application/json") @@ -451,10 +491,11 @@ def _handle_backup_password_change() -> Response: if not isinstance(body, dict): return make_response(status_code=400, content='{"error": "Invalid JSON body"}', media_type="application/json") paths: WorkspacePaths | None = get_state().api_v1_paths - if paths is None: + session_store = get_state().session_store + if paths is None or session_store is None or session_store.record_store is None: return make_response( status_code=503, - content='{"error": "Backup management is unavailable in this configuration"}', + content='{"error": "Sync is unavailable in this configuration"}', media_type="application/json", ) # Wrapped in SecretStr immediately; the plaintext must never reach a log. @@ -464,33 +505,136 @@ def _handle_backup_password_change() -> Response: return make_response( status_code=400, content='{"error": "The two passwords do not match."}', media_type="application/json" ) - result = rotate_backup_master_password( - paths=paths, - resolver=get_state().backend_resolver, - new_password=new_password, - is_save_password=bool(body.get("save_password", False)), - parent_cg=get_state().root_concurrency_group, + accounts = session_store.list_accounts() + if not accounts: + return make_response( + status_code=400, + content='{"error": "Sign in to an account first -- the master password protects synced account data."}', + media_type="application/json", + ) + record_store = session_store.record_store + resolver = get_state().backend_resolver + # Accounts that are locked on this device must unlock first: rewrapping + # here would mint a fresh DEK and overwrite the server bundle that wraps + # the account's real one, orphaning every already-synced secret. + locked_user_ids = set(record_store.locked_account_user_ids([str(account.user_id) for account in accounts])) + results: list[dict[str, object]] = [] + for account in accounts: + if str(account.user_id) in locked_user_ids: + results.append( + { + "account": str(account.email), + "is_ok": False, + "error": "This account's synced secrets are locked on this device; " + "unlock them with the current master password first.", + } + ) + continue + try: + bundle = set_master_password_for_account(paths, str(account.user_id), new_password) + if bundle is not None: + _push_new_password_state(record_store, resolver, str(account.user_id), str(account.email), bundle) + else: + _scrub_cleared_password_server_state(record_store, str(account.email)) + results.append({"account": str(account.email), "is_ok": True, "error": None}) + except (SyncCryptoError, WorkspaceSyncError, ImbueCloudCliError) as exc: + logger.warning("Master password change failed for {}: {}", account.email, exc) + results.append({"account": str(account.email), "is_ok": False, "error": str(exc)}) + return make_response( + status_code=200, + content=json.dumps({"ok": all(bool(entry["is_ok"]) for entry in results), "results": results}), + media_type="application/json", ) + + +def _handle_sync_unlock() -> Response: + """Unlock synced secrets on this device (POST /_chrome/sync-unlock). + + Tries the typed master password against every locked signed-in account's + key bundle (fetched from the connector when no local mirror exists); + whichever accounts it unwraps get their DEK installed. Reports which + accounts remain locked -- they may need an older password. + """ + if not _is_request_authenticated(): + return make_response(status_code=403, content='{"error":"Not authenticated"}', media_type="application/json") + body = request.get_json(silent=True, force=True) + if not isinstance(body, dict): + return make_response(status_code=400, content='{"error": "Invalid JSON body"}', media_type="application/json") + session_store = get_state().session_store + if session_store is None or session_store.record_store is None: + return make_response( + status_code=503, content='{"error": "Sync is unavailable"}', media_type="application/json" + ) + password = SecretStr(str(body.get("password") or "")) + record_store = session_store.record_store + accounts = session_store.list_accounts() + locked_user_ids = record_store.locked_account_user_ids([str(account.user_id) for account in accounts]) + unlocked: list[str] = [] + still_locked: list[str] = [] + for account in accounts: + if str(account.user_id) not in locked_user_ids: + continue + if record_store.unlock_account(str(account.user_id), str(account.email), password): + unlocked.append(str(account.email)) + else: + still_locked.append(str(account.email)) + scheduler = get_state().sync_scheduler + if unlocked and scheduler is not None: + scheduler.kick() + if not unlocked and still_locked: + return make_response( + status_code=200, + content=json.dumps( + { + "ok": False, + "unlocked": unlocked, + "still_locked": still_locked, + "error": "That password did not unlock any account.", + } + ), + media_type="application/json", + ) return make_response( status_code=200, - content=json.dumps( - { - "ok": result.is_all_ok, - "results": [ - { - "agent_id": entry.agent_id, - "workspace_name": entry.workspace_name, - "is_ok": entry.is_ok, - "error": entry.error, - } - for entry in result.results - ], - } - ), + content=json.dumps({"ok": True, "unlocked": unlocked, "still_locked": still_locked}), media_type="application/json", ) +def _handle_remove_workspace_record() -> Response: + """Remove a synced workspace record outright (POST /_chrome/workspaces/remove-record). + + The manual escape hatch for stale/confusing rows on the landing list. + Requires connectivity (the record lives on the connector). + """ + if not _is_request_authenticated(): + return make_response(status_code=403, content='{"error":"Not authenticated"}', media_type="application/json") + body = request.get_json(silent=True, force=True) + if not isinstance(body, dict) or not str(body.get("host_id") or ""): + return make_response( + status_code=400, content='{"error": "host_id is required"}', media_type="application/json" + ) + host_id = str(body["host_id"]) + session_store = get_state().session_store + if session_store is None or session_store.record_store is None: + return make_response( + status_code=503, content='{"error": "Sync is unavailable"}', media_type="application/json" + ) + record_store = session_store.record_store + for account in session_store.list_accounts(): + owns_host = any(record.host_id == host_id for record in record_store.list_records(str(account.user_id))) + if not owns_host: + continue + try: + record_store.remove_record_or_raise(str(account.user_id), str(account.email), host_id) + except WorkspaceSyncError as exc: + return make_response( + status_code=502, content=json.dumps({"error": str(exc)}), media_type="application/json" + ) + return make_response(status_code=200, content='{"ok": true}', media_type="application/json") + return make_response(status_code=404, content='{"error": "No such record"}', media_type="application/json") + + def _sync_latchkey_forward_sentry_consent(minds_config: MindsConfig) -> None: """Rewrite the detached ``mngr latchkey forward`` daemon's live consent file after a consent change. @@ -666,6 +810,58 @@ def _handle_welcome_page() -> Response: return make_html_response(content=html) +def _collect_remote_workspace_tiles( + backend_resolver: BackendResolverInterface, + session_store: MultiAccountSessionStore | None, +) -> list[RemoteWorkspaceTile]: + """Workspaces known only from synced records (not in local discovery), for the landing list.""" + if session_store is None or session_store.record_store is None: + return [] + # "Not in local discovery" is only meaningful once discovery has produced + # its first complete snapshot; before that every record (including this + # device's own workspaces) would misclassify as remote. + if not backend_resolver.has_completed_initial_discovery(): + return [] + local_ids = {str(aid) for aid in backend_resolver.list_known_workspace_ids()} + tiles: list[RemoteWorkspaceTile] = [] + seen_agent_ids: set[str] = set() + for account in session_store.list_accounts(): + for record in session_store.record_store.list_records(str(account.user_id)): + is_remote_active = ( + record.state == RECORD_STATE_ACTIVE + and record.agent_id not in local_ids + and record.agent_id not in seen_agent_ids + ) + if not is_remote_active: + continue + seen_agent_ids.add(record.agent_id) + location = record.device_label or record.provider_kind or "another device" + tiles.append( + RemoteWorkspaceTile( + agent_id=record.agent_id, + name=record.display_name or record.agent_id, + accent=record.color or DEFAULT_WORKSPACE_COLOR, + location=location, + host_id=record.host_id, + ) + ) + return tiles + + +def _collect_locked_account_emails(session_store: MultiAccountSessionStore | None) -> list[str]: + """Emails of signed-in accounts whose synced secrets exist but whose key is absent here.""" + if session_store is None or session_store.record_store is None: + return [] + paths = get_state().api_v1_paths + if paths is None: + return [] + accounts = session_store.list_accounts() + locked_user_ids = set( + session_store.record_store.locked_account_user_ids([str(account.user_id) for account in accounts]) + ) + return [str(account.email) for account in accounts if str(account.user_id) in locked_user_ids] + + def _handle_landing_page() -> Response: if not _is_request_authenticated(): html = render_login_page() @@ -683,8 +879,10 @@ def _handle_landing_page() -> Response: paths: WorkspacePaths | None = get_state().api_v1_paths landing_session_store: MultiAccountSessionStore | None = get_state().session_store destroying_status_by_agent_id = _resolve_destroying_for_landing(paths, backend_resolver, landing_session_store) + remote_workspaces = _collect_remote_workspace_tiles(backend_resolver, landing_session_store) + locked_account_emails = _collect_locked_account_emails(landing_session_store) - if all_agent_ids: + if all_agent_ids or remote_workspaces: agent_names: dict[str, str] = {} agent_accents: dict[str, str] = {} agent_providers: dict[str, str] = {} @@ -712,6 +910,8 @@ def _handle_landing_page() -> Response: shutdown_capable_agent_ids=shutdown_capable_agent_ids, mind_liveness_by_agent_id=mind_liveness_by_agent_id, agent_providers=agent_providers, + remote_workspaces=remote_workspaces, + locked_account_emails=locked_account_emails, ) return make_html_response(content=html) @@ -731,20 +931,15 @@ def _handle_landing_page() -> Response: branch = request.args.get("branch", "") session_store: MultiAccountSessionStore | None = get_state().session_store minds_config: MindsConfig | None = get_state().minds_config - agent_creator: AgentCreator | None = get_state().agent_creator geo_cache: GeoLocationCache | None = get_state().geo_location_cache accounts = session_store.list_accounts() if session_store else [] default_account_id = minds_config.get_default_account_id() if minds_config else None - is_backup_password_saved = has_saved_backup_password(agent_creator.paths) if agent_creator is not None else False - is_backup_password_set = is_master_password_set(agent_creator.paths) if agent_creator is not None else False region_options, region_selected = _build_region_form_context(minds_config, geo_cache) html = render_create_form( git_url=git_url, branch=branch, accounts=accounts, default_account_id=default_account_id or "", - has_saved_backup_password=is_backup_password_saved, - is_master_password_set=is_backup_password_set, region_options_by_launch_mode=region_options, region_selected_by_launch_mode=region_selected, # A deep-link that pre-fills a repo/branch wants those advanced fields @@ -820,20 +1015,15 @@ def _handle_create_page() -> Response: branch = request.args.get("branch", "") session_store: MultiAccountSessionStore | None = get_state().session_store minds_config: MindsConfig | None = get_state().minds_config - agent_creator: AgentCreator | None = get_state().agent_creator geo_cache: GeoLocationCache | None = get_state().geo_location_cache accounts = session_store.list_accounts() if session_store else [] default_account_id = minds_config.get_default_account_id() if minds_config else None - is_backup_password_saved = has_saved_backup_password(agent_creator.paths) if agent_creator is not None else False - is_backup_password_set = is_master_password_set(agent_creator.paths) if agent_creator is not None else False region_options, region_selected = _build_region_form_context(minds_config, geo_cache) html = render_create_form( git_url=git_url, branch=branch, accounts=accounts, default_account_id=default_account_id or "", - has_saved_backup_password=is_backup_password_saved, - is_master_password_set=is_backup_password_set, region_options_by_launch_mode=region_options, region_selected_by_launch_mode=region_selected, # A deep-link that pre-fills a repo/branch wants those advanced fields @@ -920,17 +1110,29 @@ def _finalize_destroyed_workspace( paths: WorkspacePaths, session_store: MultiAccountSessionStore | None, ) -> None: - """Disassociate a fully-destroyed workspace from its account, then delete its record. - - Runs only once the host is confirmed gone (DONE). Disassociating here -- - rather than synchronously when the user clicks destroy -- means a failed or - partial teardown keeps the workspace visible instead of hiding a host that - is still running. + """Tombstone a fully-destroyed workspace's record, then delete the destroying marker. + + Runs only once the host is confirmed gone (DONE). The workspace record is + kept (state=DESTROYED, secrets intact) so the workspace's backups stay + reachable from any of the account's devices; it just disappears from the + active UI. Tombstoning here -- rather than synchronously when the user + clicks destroy -- means a failed or partial teardown keeps the workspace + visible instead of hiding a host that is still running. """ - if session_store is not None: - account = session_store.get_account_for_workspace(str(agent_id)) - if account is not None: - session_store.disassociate_workspace(str(account.user_id), str(agent_id)) + if session_store is not None and session_store.record_store is not None: + found = session_store.record_store.find_active_record(str(agent_id)) + if found is not None: + owner_user_id, _record = found + owner_email = session_store.get_account_email(owner_user_id) + if owner_email is not None: + session_store.record_store.tombstone_record(owner_user_id, owner_email, str(agent_id)) + else: + logger.warning( + "Skipping workspace-record tombstone for destroyed agent {}: owning account {} is not " + "signed in on this device; the owner's next signed-in reconcile will retire the record", + agent_id, + owner_user_id, + ) delete_destroying(agent_id, paths) @@ -1482,6 +1684,21 @@ def _build_workspace_list( if account is not None: entry["account"] = account.email workspaces.append(entry) + # Append workspaces known only from synced records (hosted on another + # device). They render greyed and non-navigable; ``location`` names where + # they live. + for tile in _collect_remote_workspace_tiles(backend_resolver, session_store): + remote_entry: dict[str, str] = { + "id": tile.agent_id, + "name": tile.name, + "accent": tile.accent, + "is_remote": "true", + "location": tile.location, + } + owner = session_store.get_account_for_workspace(tile.agent_id) if session_store is not None else None + if owner is not None: + remote_entry["account"] = owner.email + workspaces.append(remote_entry) return workspaces @@ -1749,7 +1966,7 @@ def _handle_settings_page() -> Response: file_sharing_grants=file_sharing_grants, workspace_delegation_grants=workspace_delegation_grants, permissions_unavailable=permissions_unavailable, - has_saved_backup_password=has_saved_backup_password(paths) if paths is not None else False, + is_master_password_set=_is_any_account_password_set(paths), ) return make_html_response(content=html) @@ -1991,13 +2208,6 @@ def _handle_workspace_settings( errored_provider_names = {str(name) for name in backend_resolver.get_provider_errors()} is_stale = _is_workspace_provider_errored(info, errored_provider_names) - # The backup section's configure form needs to know whether a shared - # master password is already saved (it then never re-prompts) and whether - # an account is associated (imbue_cloud backups require one). - paths = get_state().api_v1_paths - is_backup_password_saved = has_saved_backup_password(paths) if paths is not None else False - is_backup_password_set = is_master_password_set(paths) if paths is not None else False - html = render_workspace_settings( agent_id=agent_id, ws_name=ws_name, @@ -2007,8 +2217,6 @@ def _handle_workspace_settings( is_leased_imbue_cloud=is_leased_imbue_cloud, current_color=current_color, is_stale=is_stale, - has_saved_backup_password=is_backup_password_saved, - is_master_password_set=is_backup_password_set, has_account=current_account is not None, ) return make_html_response(content=html) @@ -2432,6 +2640,7 @@ def create_desktop_client( latchkey_forward_supervisor: LatchkeyForwardSupervisor | None = None, discovery_health_watchdog: DiscoveryHealthWatchdog | None = None, mngr_caller: MngrCaller | None = None, + sync_scheduler: WorkspaceSyncScheduler | None = None, ) -> Flask: """Create the bare-origin minds Flask application. @@ -2465,12 +2674,6 @@ def create_desktop_client( logger.warning("Missing static/app.min.css. Run `just minds-css` from the repo root to build it.") app = Flask(__name__, static_folder=str(_static_dir), static_url_path="/_static") - # The backup master-password hash must always exist (initially the hash of - # the empty password, or of a pre-hash install's saved plaintext) so every - # backup flow can validate against it. - if paths is not None: - ensure_backup_password_hash(paths) - @app.errorhandler(Exception) def _unhandled_exception_handler(exc: Exception) -> Response | HTTPException: # Let werkzeug's HTTP exceptions (404, 405, abort(401), ...) keep their @@ -2507,6 +2710,7 @@ def _unhandled_exception_handler(exc: Exception) -> Response | HTTPException: latchkey_forward_supervisor=latchkey_forward_supervisor, discovery_health_watchdog=discovery_health_watchdog, mngr_caller=mngr_caller, + sync_scheduler=sync_scheduler, ) set_state(app, state) @@ -2549,6 +2753,8 @@ def _unhandled_exception_handler(exc: Exception) -> Response | HTTPException: app.add_url_rule("/consent", view_func=_handle_consent_submit, methods=["POST"]) app.add_url_rule("/_chrome/error-reporting", view_func=_handle_error_reporting_settings, methods=["POST"]) app.add_url_rule("/_chrome/backup-password", view_func=_handle_backup_password_change, methods=["POST"]) + app.add_url_rule("/_chrome/sync-unlock", view_func=_handle_sync_unlock, methods=["POST"]) + app.add_url_rule("/_chrome/workspaces/remove-record", view_func=_handle_remove_workspace_record, methods=["POST"]) app.add_url_rule("/help", view_func=_handle_help_page) app.add_url_rule("/help/report", view_func=_handle_help_report, methods=["POST"]) app.add_url_rule("/help/assist", view_func=_handle_help_assist, methods=["POST"]) diff --git a/apps/minds/imbue/minds/desktop_client/backup_password_rotation.py b/apps/minds/imbue/minds/desktop_client/backup_password_rotation.py deleted file mode 100644 index 4c7c953765..0000000000 --- a/apps/minds/imbue/minds/desktop_client/backup_password_rotation.py +++ /dev/null @@ -1,151 +0,0 @@ -"""Change the shared backup master password across all existing workspaces. - -The master password is only ever a *recovery* key: every workspace repository -also carries its own random ``RESTIC_PASSWORD`` (in the canonical env), which -is what authenticates the rotation. Per workspace whose host still exists and -that has a canonical env, the rotation: - -1. ``restic key add``s the new master password (authenticated with the - workspace's random password), -2. removes every other key, so the repository ends in a clean two-key state - (the workspace's own key + the new master key). - -Destroyed workspaces are skipped: their repositories stay reachable under the -old password via their (never-deleted) canonical envs. Failures are collected -per workspace -- the flow is synchronous, idempotent, and safely re-runnable -(rerunning converges every repository to the same two-key state). - -After the per-repository work, the ``backup_password_hash`` becomes the hash -of the new password (this is the only flow allowed to change it), a stale -plaintext convenience copy is deleted, and -- when requested -- the new value -is saved as the fresh convenience copy. -""" - -from loguru import logger -from pydantic import Field -from pydantic import SecretStr - -from imbue.concurrency_group.concurrency_group import ConcurrencyGroup -from imbue.imbue_common.frozen_model import FrozenModel -from imbue.imbue_common.logging import log_span -from imbue.minds.config.data_types import WorkspacePaths -from imbue.minds.desktop_client import restic_cli -from imbue.minds.desktop_client.backend_resolver import BackendResolverInterface -from imbue.minds.desktop_client.backup_env_store import parse_restic_env -from imbue.minds.desktop_client.backup_env_store import read_canonical_env -from imbue.minds.desktop_client.backup_password_store import delete_saved_backup_password -from imbue.minds.desktop_client.backup_password_store import save_backup_password -from imbue.minds.desktop_client.backup_password_store import write_backup_password_hash -from imbue.minds.errors import BackupProvisioningError - - -class WorkspaceRotationResult(FrozenModel): - """The outcome of rekeying one workspace's repository.""" - - agent_id: str = Field(description="The workspace agent id") - workspace_name: str = Field(description="The workspace's display name (for the results list)") - is_ok: bool = Field(description="Whether the repository now carries the new master key (and only it + its own)") - error: str | None = Field(default=None, description="Why the rekey failed, when it did") - - -class BackupPasswordRotationResult(FrozenModel): - """The outcome of a whole master-password rotation.""" - - results: tuple[WorkspaceRotationResult, ...] = Field(description="One entry per existing workspace with backups") - is_all_ok: bool = Field(description="Whether every workspace rekeyed cleanly") - - -def _rekey_repository_to_new_master( - *, - canonical_env: str, - new_password: SecretStr, - parent_cg: ConcurrencyGroup | None, -) -> None: - """Add the new master key and strip the repository down to two keys. - - Authenticated entirely with the workspace's own random password, so the - old master password is never needed. Raises ``BackupProvisioningError`` - on any restic failure. - """ - env = parse_restic_env(canonical_env) - repository = env.get("RESTIC_REPOSITORY", "") - workspace_password = env.get("RESTIC_PASSWORD", "") - if not repository or not workspace_password: - raise BackupProvisioningError("The canonical restic.env lacks RESTIC_REPOSITORY/RESTIC_PASSWORD") - backend_env = {key: value for key, value in env.items() if key not in ("RESTIC_REPOSITORY", "RESTIC_PASSWORD")} - - # Diffing the key listing around the add identifies the new key without - # having to parse restic's human-readable add output. - keys_before = restic_cli.list_keys( - repository=repository, backend_env=backend_env, password=workspace_password, parent_cg=parent_cg - ) - restic_cli.add_password_key( - repository=repository, - backend_env=backend_env, - existing_password=workspace_password, - new_password=new_password.get_secret_value(), - parent_cg=parent_cg, - ) - keys_after = restic_cli.list_keys( - repository=repository, backend_env=backend_env, password=workspace_password, parent_cg=parent_cg - ) - ids_before = {key.key_id for key in keys_before} - added_ids = {key.key_id for key in keys_after if key.key_id not in ids_before} - - # Keep the workspace's own (current) key and the just-added master key; - # everything else -- the old master key and any strays -- goes. - for key in keys_after: - if key.is_current or key.key_id in added_ids: - continue - restic_cli.remove_key( - repository=repository, - backend_env=backend_env, - password=workspace_password, - key_id=key.key_id, - parent_cg=parent_cg, - ) - - -def rotate_backup_master_password( - *, - paths: WorkspacePaths, - resolver: BackendResolverInterface, - new_password: SecretStr, - is_save_password: bool, - parent_cg: ConcurrencyGroup | None = None, -) -> BackupPasswordRotationResult: - """Rekey every existing backed-up workspace to ``new_password`` and update the stores. - - The hash is updated even when some repositories failed (the user's intent - stands; the per-workspace errors point at what to re-run). A stale - plaintext convenience copy is deleted; ``is_save_password`` re-saves the - new value instead. - """ - results: list[WorkspaceRotationResult] = [] - with log_span("Rotating the backup master password"): - for agent_id in resolver.list_active_workspace_ids(): - canonical_env = read_canonical_env(paths, agent_id) - if canonical_env is None: - continue - workspace_name = resolver.get_workspace_name(agent_id) or str(agent_id) - try: - _rekey_repository_to_new_master( - canonical_env=canonical_env, new_password=new_password, parent_cg=parent_cg - ) - except BackupProvisioningError as e: - logger.warning("Rekeying the backup repository for {} failed: {}", agent_id, e) - results.append( - WorkspaceRotationResult( - agent_id=str(agent_id), workspace_name=workspace_name, is_ok=False, error=str(e) - ) - ) - continue - results.append(WorkspaceRotationResult(agent_id=str(agent_id), workspace_name=workspace_name, is_ok=True)) - - write_backup_password_hash(paths, new_password) - # The old plaintext copy no longer matches the hash; keep only a - # freshly-saved copy of the new value when the user asked for one. - delete_saved_backup_password(paths) - if is_save_password and new_password.get_secret_value(): - save_backup_password(paths, new_password) - return BackupPasswordRotationResult(results=tuple(results), is_all_ok=all(result.is_ok for result in results)) diff --git a/apps/minds/imbue/minds/desktop_client/backup_password_rotation_test.py b/apps/minds/imbue/minds/desktop_client/backup_password_rotation_test.py deleted file mode 100644 index cad7b6f56f..0000000000 --- a/apps/minds/imbue/minds/desktop_client/backup_password_rotation_test.py +++ /dev/null @@ -1,161 +0,0 @@ -"""Unit tests for the master-password rotation (real restic against local repos).""" - -from pathlib import Path - -import pytest -from pydantic import SecretStr - -from imbue.minds.config.data_types import WorkspacePaths -from imbue.minds.desktop_client import restic_cli -from imbue.minds.desktop_client.backup_env_store import write_canonical_env -from imbue.minds.desktop_client.backup_password_rotation import rotate_backup_master_password -from imbue.minds.desktop_client.backup_password_store import read_saved_backup_password -from imbue.minds.desktop_client.backup_password_store import save_backup_password -from imbue.minds.desktop_client.backup_password_store import verify_backup_password -from imbue.minds.desktop_client.backup_password_store import write_backup_password_hash -from imbue.minds.desktop_client.conftest import make_agents_json -from imbue.minds.desktop_client.conftest import make_resolver_with_data -from imbue.minds.errors import BackupProvisioningError -from imbue.mngr.primitives import AgentId - -_WORKSPACE_PASSWORD = "workspace-random-password" -_OLD_MASTER = "old-master-password" - - -def _paths(tmp_path: Path) -> WorkspacePaths: - data_dir = tmp_path / "minds-data" - data_dir.mkdir(exist_ok=True) - return WorkspacePaths(data_dir=data_dir) - - -def _provision_workspace_repo(tmp_path: Path, paths: WorkspacePaths, agent_id: AgentId, *, name: str) -> str: - """Init a local repo keyed like provisioning does (master key + workspace key).""" - repository = str(tmp_path / name) - restic_cli.init_repo(repository=repository, backend_env={}, password=_OLD_MASTER) - restic_cli.add_password_key( - repository=repository, - backend_env={}, - existing_password=_OLD_MASTER, - new_password=_WORKSPACE_PASSWORD, - ) - write_canonical_env(paths, agent_id, f"RESTIC_REPOSITORY={repository}\nRESTIC_PASSWORD={_WORKSPACE_PASSWORD}\n") - return repository - - -@pytest.mark.timeout(120) -def test_rotation_rekeys_the_repo_and_updates_the_hash(tmp_path: Path) -> None: - paths = _paths(tmp_path) - agent_id = AgentId() - repository = _provision_workspace_repo(tmp_path, paths, agent_id, name="repo-one") - write_backup_password_hash(paths, SecretStr(_OLD_MASTER)) - save_backup_password(paths, SecretStr(_OLD_MASTER)) - resolver = make_resolver_with_data(make_agents_json(agent_id)) - - result = rotate_backup_master_password( - paths=paths, - resolver=resolver, - new_password=SecretStr("new-master-password"), - is_save_password=False, - ) - - assert result.is_all_ok is True - assert [entry.agent_id for entry in result.results] == [str(agent_id)] - # The repo ends in the clean two-key state: workspace key + new master. - keys = restic_cli.list_keys(repository=repository, backend_env={}, password=_WORKSPACE_PASSWORD) - assert len(keys) == 2 - # The new master opens the repo; the old one no longer does. - assert restic_cli.list_keys(repository=repository, backend_env={}, password="new-master-password") - with pytest.raises(BackupProvisioningError): - restic_cli.list_keys(repository=repository, backend_env={}, password=_OLD_MASTER) - # The hash is the new password's, and the stale plaintext copy is gone. - assert verify_backup_password(paths, SecretStr("new-master-password")) is True - assert verify_backup_password(paths, SecretStr(_OLD_MASTER)) is False - assert read_saved_backup_password(paths) is None - - -@pytest.mark.timeout(120) -def test_rotation_to_the_empty_password_works(tmp_path: Path) -> None: - paths = _paths(tmp_path) - agent_id = AgentId() - repository = _provision_workspace_repo(tmp_path, paths, agent_id, name="repo-empty") - write_backup_password_hash(paths, SecretStr(_OLD_MASTER)) - resolver = make_resolver_with_data(make_agents_json(agent_id)) - - result = rotate_backup_master_password( - paths=paths, resolver=resolver, new_password=SecretStr(""), is_save_password=False - ) - - assert result.is_all_ok is True - # The empty-password key opens the repo (restic's --insecure-no-password path). - assert restic_cli.list_keys(repository=repository, backend_env={}, password=None) - assert verify_backup_password(paths, SecretStr("")) is True - - -@pytest.mark.timeout(120) -def test_rotation_saves_the_new_password_when_asked(tmp_path: Path) -> None: - paths = _paths(tmp_path) - agent_id = AgentId() - _provision_workspace_repo(tmp_path, paths, agent_id, name="repo-save") - resolver = make_resolver_with_data(make_agents_json(agent_id)) - - rotate_backup_master_password( - paths=paths, resolver=resolver, new_password=SecretStr("brand-new"), is_save_password=True - ) - - assert read_saved_backup_password(paths) == "brand-new" - - -@pytest.mark.timeout(120) -def test_rotation_skips_workspaces_without_backups(tmp_path: Path) -> None: - paths = _paths(tmp_path) - agent_id = AgentId() - resolver = make_resolver_with_data(make_agents_json(agent_id)) - - result = rotate_backup_master_password( - paths=paths, resolver=resolver, new_password=SecretStr("whatever"), is_save_password=False - ) - - assert result.results == () - assert result.is_all_ok is True - assert verify_backup_password(paths, SecretStr("whatever")) is True - - -@pytest.mark.timeout(120) -def test_rotation_reports_per_workspace_failures_and_still_updates_the_hash(tmp_path: Path) -> None: - paths = _paths(tmp_path) - agent_id = AgentId() - # A canonical env pointing at a repository that does not exist. - write_canonical_env( - paths, agent_id, f"RESTIC_REPOSITORY={tmp_path / 'missing-repo'}\nRESTIC_PASSWORD=irrelevant\n" - ) - resolver = make_resolver_with_data(make_agents_json(agent_id)) - - result = rotate_backup_master_password( - paths=paths, resolver=resolver, new_password=SecretStr("intended"), is_save_password=False - ) - - assert result.is_all_ok is False - assert len(result.results) == 1 - assert result.results[0].is_ok is False - assert result.results[0].error - # The user's intent stands: the hash moved to the new password so a re-run - # (retyping the same new password) can retry the failed repository. - assert verify_backup_password(paths, SecretStr("intended")) is True - - -@pytest.mark.timeout(120) -def test_rotation_is_idempotent_across_reruns(tmp_path: Path) -> None: - paths = _paths(tmp_path) - agent_id = AgentId() - repository = _provision_workspace_repo(tmp_path, paths, agent_id, name="repo-rerun") - resolver = make_resolver_with_data(make_agents_json(agent_id)) - - for _ in range(2): - result = rotate_backup_master_password( - paths=paths, resolver=resolver, new_password=SecretStr("same-new"), is_save_password=False - ) - assert result.is_all_ok is True - - keys = restic_cli.list_keys(repository=repository, backend_env={}, password=_WORKSPACE_PASSWORD) - assert len(keys) == 2 - assert restic_cli.list_keys(repository=repository, backend_env={}, password="same-new") diff --git a/apps/minds/imbue/minds/desktop_client/backup_password_store.py b/apps/minds/imbue/minds/desktop_client/backup_password_store.py deleted file mode 100644 index a12d30979a..0000000000 --- a/apps/minds/imbue/minds/desktop_client/backup_password_store.py +++ /dev/null @@ -1,175 +0,0 @@ -"""Persistence for the user's shared restic backup master/recovery password. - -One passphrase is shared across all of a user's workspaces. Two files live -under the activated minds env's data dir (``~/.minds/`` for the default env): - -* ``backup_password_hash`` -- an argon2 hash of the master password. Always - present (seeded at app startup: from the plaintext file when one exists, - else the hash of the empty string) and the single validation authority: - any flow that needs the master password verifies the candidate against it. - The application therefore starts in the "empty master password" state and - a new user can create workspaces without ever typing one. -* ``backup_password`` -- the optional plaintext convenience copy, so the user - does not have to retype the password on every repo-initializing flow. It - can only ever be (re)written with a value that was just validated against - the hash; establishing or *changing* the password is exclusively the - Settings-page rotation flow's job. - -The master password never enters a workspace: minds uses it solely to -``restic init`` each repository and authenticate key operations, all from -the minds machine. -""" - -import os -from pathlib import Path - -from argon2 import PasswordHasher -from argon2.exceptions import InvalidHashError -from argon2.exceptions import VerificationError -from argon2.exceptions import VerifyMismatchError -from pydantic import SecretStr - -from imbue.minds.config.data_types import WorkspacePaths -from imbue.minds.errors import BackupProvisioningError - -_BACKUP_PASSWORD_FILENAME = "backup_password" -_BACKUP_PASSWORD_HASH_FILENAME = "backup_password_hash" - -_PASSWORD_HASHER = PasswordHasher() - - -def backup_password_file_path(paths: WorkspacePaths) -> Path: - """Return the path of the shared master-password file for this minds env.""" - return paths.data_dir / _BACKUP_PASSWORD_FILENAME - - -def backup_password_hash_file_path(paths: WorkspacePaths) -> Path: - """Return the path of the master-password hash file for this minds env.""" - return paths.data_dir / _BACKUP_PASSWORD_HASH_FILENAME - - -def has_saved_backup_password(paths: WorkspacePaths) -> bool: - """Return whether a non-empty master password has already been saved.""" - return read_saved_backup_password(paths) is not None - - -def read_saved_backup_password(paths: WorkspacePaths) -> str | None: - """Return the saved master password, or None if none has been saved yet.""" - path = backup_password_file_path(paths) - if not path.is_file(): - return None - try: - content = path.read_text() - except OSError as e: - raise BackupProvisioningError(f"Could not read saved backup password at {path}: {e}") from e - stripped = content.strip() - return stripped or None - - -def _write_secret_file(path: Path, content: str) -> None: - """Atomically write a 0600 secret file (temp file + rename, never world-readable).""" - try: - path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = path.with_suffix(".tmp") - fd = os.open(tmp_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - try: - os.write(fd, content.encode("utf-8")) - finally: - os.close(fd) - tmp_path.rename(path) - except OSError as e: - raise BackupProvisioningError(f"Could not write {path}: {e}") from e - - -def save_backup_password(paths: WorkspacePaths, password: SecretStr) -> None: - """(Over)write the plaintext convenience copy of the master password. - - Callers must only pass a value already validated against the hash (or the - value the hash was just rotated to) -- this file is a convenience mirror, - never an authority. - """ - _write_secret_file(backup_password_file_path(paths), password.get_secret_value()) - - -def delete_saved_backup_password(paths: WorkspacePaths) -> None: - """Remove the plaintext convenience copy (e.g. it went stale on rotation).""" - try: - backup_password_file_path(paths).unlink(missing_ok=True) - except OSError as e: - raise BackupProvisioningError(f"Could not delete the saved backup password: {e}") from e - - -def write_backup_password_hash(paths: WorkspacePaths, password: SecretStr) -> None: - """Hash the password (argon2) and persist it as the validation authority.""" - _write_secret_file(backup_password_hash_file_path(paths), _PASSWORD_HASHER.hash(password.get_secret_value())) - - -def ensure_backup_password_hash(paths: WorkspacePaths) -> None: - """Seed ``backup_password_hash`` if it does not exist yet (app startup). - - Seeded from the plaintext convenience copy when one exists (pre-hash - installs keep working unchanged), else from the empty string -- the - application's initial "no master password" state. - """ - if backup_password_hash_file_path(paths).is_file(): - return - saved = read_saved_backup_password(paths) - write_backup_password_hash(paths, SecretStr(saved if saved is not None else "")) - - -def verify_backup_password(paths: WorkspacePaths, candidate: SecretStr) -> bool: - """Return whether ``candidate`` matches the stored master-password hash. - - Self-healing: seeds the hash file first if it is missing, so callers can - rely on a verdict even before the startup hook ran (e.g. in tests). - """ - ensure_backup_password_hash(paths) - path = backup_password_hash_file_path(paths) - try: - stored_hash = path.read_text().strip() - except OSError as e: - raise BackupProvisioningError(f"Could not read the backup password hash at {path}: {e}") from e - try: - _PASSWORD_HASHER.verify(stored_hash, candidate.get_secret_value()) - except (VerifyMismatchError, VerificationError): - return False - except InvalidHashError as e: - raise BackupProvisioningError(f"The backup password hash at {path} is not a valid argon2 hash") from e - return True - - -def is_master_password_set(paths: WorkspacePaths) -> bool: - """Return whether the master password is currently non-empty. - - Drives which forms need a password input at all: while the hash is still - the empty-password seed, no flow ever needs the user to type anything. - """ - return not verify_backup_password(paths, SecretStr("")) - - -def resolve_backup_password_for_use( - paths: WorkspacePaths, typed_password: SecretStr -) -> tuple[SecretStr | None, str | None]: - """Resolve the master password for a repo-initializing flow. - - A non-blank typed value must match the hash. A blank value falls back to - the saved plaintext copy when one exists, else means the empty password -- - both also validated against the hash. Returns ``(password, None)`` on - success or ``(None, user-facing error message)`` on a mismatch. - """ - typed_value = typed_password.get_secret_value() - if typed_value: - if not verify_backup_password(paths, typed_password): - return None, "The backup master password is incorrect." - return typed_password, None - saved = read_saved_backup_password(paths) - if saved is not None: - if not verify_backup_password(paths, SecretStr(saved)): - return None, ( - "The saved backup password no longer matches the current master password; " - "enter the master password explicitly." - ) - return SecretStr(saved), None - if not verify_backup_password(paths, SecretStr("")): - return None, "A backup master password is set; enter it to continue." - return SecretStr(""), None diff --git a/apps/minds/imbue/minds/desktop_client/backup_password_store_test.py b/apps/minds/imbue/minds/desktop_client/backup_password_store_test.py deleted file mode 100644 index a62d955aa4..0000000000 --- a/apps/minds/imbue/minds/desktop_client/backup_password_store_test.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Unit tests for the shared restic backup master-password store (hash + plaintext copy).""" - -import stat -from pathlib import Path - -from pydantic import SecretStr - -from imbue.minds.config.data_types import WorkspacePaths -from imbue.minds.desktop_client.backup_password_store import backup_password_file_path -from imbue.minds.desktop_client.backup_password_store import backup_password_hash_file_path -from imbue.minds.desktop_client.backup_password_store import delete_saved_backup_password -from imbue.minds.desktop_client.backup_password_store import ensure_backup_password_hash -from imbue.minds.desktop_client.backup_password_store import has_saved_backup_password -from imbue.minds.desktop_client.backup_password_store import is_master_password_set -from imbue.minds.desktop_client.backup_password_store import read_saved_backup_password -from imbue.minds.desktop_client.backup_password_store import resolve_backup_password_for_use -from imbue.minds.desktop_client.backup_password_store import save_backup_password -from imbue.minds.desktop_client.backup_password_store import verify_backup_password -from imbue.minds.desktop_client.backup_password_store import write_backup_password_hash - - -def _paths(tmp_path: Path) -> WorkspacePaths: - return WorkspacePaths(data_dir=tmp_path) - - -# -- Plaintext convenience copy -- - - -def test_read_returns_none_when_absent(tmp_path: Path) -> None: - assert read_saved_backup_password(_paths(tmp_path)) is None - assert has_saved_backup_password(_paths(tmp_path)) is False - - -def test_save_then_read_round_trips_and_overwrites(tmp_path: Path) -> None: - paths = _paths(tmp_path) - save_backup_password(paths, SecretStr("hunter2")) - assert read_saved_backup_password(paths) == "hunter2" - assert has_saved_backup_password(paths) is True - # The plaintext copy is a mirror of a validated value, so re-saving a new - # (validated) value overwrites it. - save_backup_password(paths, SecretStr("hunter3")) - assert read_saved_backup_password(paths) == "hunter3" - - -def test_delete_saved_backup_password_removes_the_copy(tmp_path: Path) -> None: - paths = _paths(tmp_path) - save_backup_password(paths, SecretStr("secret")) - delete_saved_backup_password(paths) - assert read_saved_backup_password(paths) is None - # Deleting an absent copy is a no-op. - delete_saved_backup_password(paths) - - -def test_saved_files_are_owner_only(tmp_path: Path) -> None: - paths = _paths(tmp_path) - save_backup_password(paths, SecretStr("secret")) - write_backup_password_hash(paths, SecretStr("secret")) - for path in (backup_password_file_path(paths), backup_password_hash_file_path(paths)): - assert stat.S_IMODE(path.stat().st_mode) == 0o600 - - -def test_read_treats_blank_file_as_unset(tmp_path: Path) -> None: - paths = _paths(tmp_path) - backup_password_file_path(paths).write_text(" \n") - assert read_saved_backup_password(paths) is None - assert has_saved_backup_password(paths) is False - - -def test_files_live_under_data_dir(tmp_path: Path) -> None: - paths = _paths(tmp_path) - assert backup_password_file_path(paths) == tmp_path / "backup_password" - assert backup_password_hash_file_path(paths) == tmp_path / "backup_password_hash" - - -# -- Hash seeding + verification -- - - -def test_ensure_seeds_the_empty_password_hash_on_first_start(tmp_path: Path) -> None: - paths = _paths(tmp_path) - ensure_backup_password_hash(paths) - assert backup_password_hash_file_path(paths).is_file() - assert verify_backup_password(paths, SecretStr("")) is True - assert verify_backup_password(paths, SecretStr("anything")) is False - - -def test_ensure_seeds_from_an_existing_plaintext_copy(tmp_path: Path) -> None: - # A pre-hash install has only the plaintext file; the seeded hash must - # accept that password (and reject the empty one). - paths = _paths(tmp_path) - save_backup_password(paths, SecretStr("legacy-password")) - ensure_backup_password_hash(paths) - assert verify_backup_password(paths, SecretStr("legacy-password")) is True - assert verify_backup_password(paths, SecretStr("")) is False - - -def test_ensure_never_overwrites_an_existing_hash(tmp_path: Path) -> None: - paths = _paths(tmp_path) - write_backup_password_hash(paths, SecretStr("established")) - save_backup_password(paths, SecretStr("a-different-plaintext")) - ensure_backup_password_hash(paths) - assert verify_backup_password(paths, SecretStr("established")) is True - - -def test_write_hash_stores_no_plaintext(tmp_path: Path) -> None: - paths = _paths(tmp_path) - write_backup_password_hash(paths, SecretStr("visible-nowhere")) - hash_content = backup_password_hash_file_path(paths).read_text() - assert "visible-nowhere" not in hash_content - assert hash_content.startswith("$argon2") - - -def test_is_master_password_set_reflects_the_hash_state(tmp_path: Path) -> None: - paths = _paths(tmp_path) - # Fresh install: the (seeded) hash is the empty password -> not set. - assert is_master_password_set(paths) is False - write_backup_password_hash(paths, SecretStr("something")) - assert is_master_password_set(paths) is True - write_backup_password_hash(paths, SecretStr("")) - assert is_master_password_set(paths) is False - - -# -- Resolution for repo-initializing flows -- - - -def test_resolve_blank_means_the_empty_password_when_nothing_is_saved(tmp_path: Path) -> None: - paths = _paths(tmp_path) - resolved, error = resolve_backup_password_for_use(paths, SecretStr("")) - assert error is None - assert resolved is not None and resolved.get_secret_value() == "" - - -def test_resolve_blank_falls_back_to_the_saved_copy(tmp_path: Path) -> None: - paths = _paths(tmp_path) - write_backup_password_hash(paths, SecretStr("shared-secret")) - save_backup_password(paths, SecretStr("shared-secret")) - resolved, error = resolve_backup_password_for_use(paths, SecretStr("")) - assert error is None - assert resolved is not None and resolved.get_secret_value() == "shared-secret" - - -def test_resolve_rejects_a_wrong_typed_password(tmp_path: Path) -> None: - paths = _paths(tmp_path) - write_backup_password_hash(paths, SecretStr("right")) - resolved, error = resolve_backup_password_for_use(paths, SecretStr("wrong")) - assert resolved is None - assert error is not None and "incorrect" in error - - -def test_resolve_rejects_blank_when_a_password_is_set_and_nothing_is_saved(tmp_path: Path) -> None: - paths = _paths(tmp_path) - write_backup_password_hash(paths, SecretStr("right")) - resolved, error = resolve_backup_password_for_use(paths, SecretStr("")) - assert resolved is None - assert error is not None and "master password" in error - - -def test_resolve_rejects_a_stale_saved_copy(tmp_path: Path) -> None: - # The hash rotated but a stale plaintext copy survived: blank must not - # silently use the stale value. - paths = _paths(tmp_path) - write_backup_password_hash(paths, SecretStr("new-password")) - save_backup_password(paths, SecretStr("old-password")) - resolved, error = resolve_backup_password_for_use(paths, SecretStr("")) - assert resolved is None - assert error is not None and "saved" in error - - -def test_resolve_accepts_a_correct_typed_password(tmp_path: Path) -> None: - paths = _paths(tmp_path) - write_backup_password_hash(paths, SecretStr("right")) - resolved, error = resolve_backup_password_for_use(paths, SecretStr("right")) - assert error is None - assert resolved is not None and resolved.get_secret_value() == "right" diff --git a/apps/minds/imbue/minds/desktop_client/backup_provisioning.py b/apps/minds/imbue/minds/desktop_client/backup_provisioning.py index 6bc4a748a3..04fb97e602 100644 --- a/apps/minds/imbue/minds/desktop_client/backup_provisioning.py +++ b/apps/minds/imbue/minds/desktop_client/backup_provisioning.py @@ -8,19 +8,20 @@ The key idea: minds initializes the restic repository itself (from the machine running minds) and gives each workspace its own random repository -password, so the workspace never holds the user's master password and -carries no repo-init logic. Concretely, enabling backups: +password -- the repo's single key. Disaster recovery does not need a repo +"master key": the canonical env (and therefore the random password) syncs +inside the account's encrypted workspace record, unlocked by the master +password via the account DEK (see ``dek_store``). Concretely, enabling +backups: 1. resolves the repository URL + backend credentials (``IMBUE_CLOUD``: create/reuse a per-workspace R2 bucket + readwrite key; ``API_KEY``: from the user's free-form env block), -2. generates a random per-workspace ``RESTIC_PASSWORD``, -3. ``restic init``s the repo using the user's master password (which may be - empty), -4. ``restic key add``s the random per-workspace password, -5. writes the canonical ``restic.env`` (repo + creds + random password) to +2. generates a random per-workspace ``RESTIC_PASSWORD`` and ``restic init``s + the repo with it, +3. writes the canonical ``restic.env`` (repo + creds + random password) to the minds-side store (see ``backup_env_store``), and -6. injects that whole file into the workspace at +4. injects that whole file into the workspace at ``runtime/secrets/restic.env`` via ``mngr exec``. ``CONFIGURE_LATER`` is a no-op. Re-provisioning is idempotent: if a @@ -37,7 +38,6 @@ from loguru import logger from pydantic import Field -from pydantic import SecretStr from imbue.concurrency_group.concurrency_group import ConcurrencyGroup from imbue.concurrency_group.subprocess_utils import FinishedProcess @@ -81,14 +81,6 @@ class BackupSetupRequest(FrozenModel): """The inputs needed to configure backups for one host.""" backup_provider: BackupProvider = Field(description="Which backup provider to configure") - master_password: SecretStr | None = Field( - default=None, - description=( - "The user's master/recovery password used (only) to `restic init` the repo. None means " - "the master password is empty -- the repo is initialized with an empty password. " - "This is never written into the workspace; the workspace gets its own random password." - ), - ) api_key_env_text: str = Field( default="", description=( @@ -339,7 +331,7 @@ def configure_backups_for_host( return with log_span("Configuring {} backups for agent {}", request.backup_provider.value, agent_id): - # restic must be available on the minds machine to init the repo + add the key. + # restic must be available on the minds machine to init the repo. restic_cli.ensure_restic_available() # Idempotent re-provision: the canonical env is the source of truth. @@ -353,21 +345,13 @@ def configure_backups_for_host( repository, backend_env = _resolve_repository_and_backend_env( request, host_id, imbue_cloud_cli=imbue_cloud_cli ) - master_password = request.master_password.get_secret_value() if request.master_password is not None else None workspace_password = generate_workspace_password() - # Initialize the repo with the master (or empty) password, then add the - # random per-workspace password as an additional key. The workspace only - # ever receives the random password. + # Initialize the repo with the workspace's own random password -- its + # single key. Cross-device and disaster-recovery access come from the + # synced (encrypted) canonical env, not from extra repo keys. restic_cli.init_repo( - repository=repository, backend_env=backend_env, password=master_password, parent_cg=parent_cg - ) - restic_cli.add_password_key( - repository=repository, - backend_env=backend_env, - existing_password=master_password, - new_password=workspace_password, - parent_cg=parent_cg, + repository=repository, backend_env=backend_env, password=workspace_password, parent_cg=parent_cg ) canonical_env = build_canonical_env_content( @@ -443,8 +427,8 @@ def change_backup_destination_for_host( Archives the existing canonical env minds-side (the old repository stays reachable through the archive), then runs the ordinary idempotent provisioning against the new inputs: new random per-workspace password, - ``restic init`` with the master (or empty) password, ``restic key add``, - canonical env write, and injection (which rotates the workspace copy). + ``restic init`` keyed solely by that password, canonical env write, and + injection (which rotates the workspace copy). Existing snapshots stay in the old repository; the new destination starts fresh. """ diff --git a/apps/minds/imbue/minds/desktop_client/conftest.py b/apps/minds/imbue/minds/desktop_client/conftest.py index 5fc58ad3aa..a8adfa0f2b 100644 --- a/apps/minds/imbue/minds/desktop_client/conftest.py +++ b/apps/minds/imbue/minds/desktop_client/conftest.py @@ -11,6 +11,7 @@ from pydantic import Field from imbue.concurrency_group.concurrency_group import ConcurrencyGroup +from imbue.minds.config.data_types import WorkspacePaths from imbue.minds.desktop_client.backend_resolver import MngrCliBackendResolver from imbue.minds.desktop_client.backend_resolver import ParsedAgentsResult from imbue.minds.desktop_client.backend_resolver import ServiceLogRecord @@ -18,8 +19,11 @@ from imbue.minds.desktop_client.backend_resolver import parse_service_log_records from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudAuthAccount from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCli +from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCliError +from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudSyncConflictCliError from imbue.minds.desktop_client.notification import NotificationDispatcher from imbue.minds.desktop_client.session_store import MultiAccountSessionStore +from imbue.minds.desktop_client.workspace_record_store import WorkspaceRecordStore from imbue.minds.primitives import ServiceName from imbue.minds.utils.mngr_caller import MngrCaller from imbue.minds.utils.testing import RecordingMngrCaller @@ -79,6 +83,71 @@ def add_account( def remove_account(self, user_id: str) -> None: self.accounts_to_return = [a for a in self.accounts_to_return if a.user_id != user_id] + # -- In-memory workspace-sync backend (mirrors the connector's semantics) -- + + sync_records_by_email: dict[str, dict[str, dict[str, object]]] = Field( + default_factory=dict, description="email -> host_id -> wire record (the fake server state)" + ) + sync_bundle_by_email: dict[str, dict[str, object]] = Field( + default_factory=dict, description="email -> key bundle (the fake server state)" + ) + is_sync_offline: bool = Field(default=False, description="When True, every sync call raises (connector down)") + + def _check_sync_online(self, command_repr: str) -> None: + if self.is_sync_offline: + raise ImbueCloudCliError(f"{command_repr}: connector unreachable (fake offline)") + + def sync_records_pull(self, account: str) -> list[dict[str, object]]: + self._check_sync_online("sync records pull") + return [dict(record) for record in self.sync_records_by_email.get(account, {}).values()] + + def sync_record_push(self, account: str, record: Mapping[str, object]) -> dict[str, object]: + self._check_sync_online("sync records push") + by_host = self.sync_records_by_email.setdefault(account, {}) + host_id = str(record["host_id"]) + existing = by_host.get(host_id) + pushed_revision = int(str(record["revision"])) + if existing is not None and pushed_revision != int(str(existing["revision"])) + 1: + conflict = ImbueCloudSyncConflictCliError("sync records push: revision conflict") + conflict.stored_record = dict(existing) + raise conflict + if str(record.get("state")) == "active": + for other_host_id, other in by_host.items(): + is_other = other_host_id != host_id + if is_other and other.get("agent_id") == record.get("agent_id") and other.get("state") == "active": + agent_conflict = ImbueCloudSyncConflictCliError("sync records push: active agent conflict") + agent_conflict.stored_record = None + raise agent_conflict + stored = dict(record) + by_host[host_id] = stored + return dict(stored) + + def sync_record_delete(self, account: str, host_id: str) -> None: + self._check_sync_online("sync records delete") + self.sync_records_by_email.get(account, {}).pop(host_id, None) + + def sync_scrub_secrets(self, account: str) -> int: + self._check_sync_online("sync scrub-secrets") + scrubbed = 0 + for record in self.sync_records_by_email.get(account, {}).values(): + if record.get("encrypted_secrets") is not None: + record["encrypted_secrets"] = None + scrubbed += 1 + return scrubbed + + def sync_bundle_pull(self, account: str) -> dict[str, object] | None: + self._check_sync_online("sync bundle pull") + bundle = self.sync_bundle_by_email.get(account) + return dict(bundle) if bundle is not None else None + + def sync_bundle_push(self, account: str, bundle: Mapping[str, object]) -> None: + self._check_sync_online("sync bundle push") + self.sync_bundle_by_email[account] = dict(bundle) + + def sync_bundle_delete(self, account: str) -> None: + self._check_sync_online("sync bundle delete") + self.sync_bundle_by_email.pop(account, None) + def make_fake_imbue_cloud_cli() -> FakeImbueCloudCli: """Build a :class:`FakeImbueCloudCli` rooted at a fresh ``ConcurrencyGroup``.""" @@ -88,8 +157,15 @@ def make_fake_imbue_cloud_cli() -> FakeImbueCloudCli: def make_session_store_for_test(data_dir: Path, cli: ImbueCloudCli | None = None) -> MultiAccountSessionStore: - """Build a :class:`MultiAccountSessionStore` with a fake CLI by default.""" - return MultiAccountSessionStore(data_dir=data_dir, cli=cli or make_fake_imbue_cloud_cli()) + """Build a :class:`MultiAccountSessionStore` (with its record store) over a fake CLI by default.""" + effective_cli = cli or make_fake_imbue_cloud_cli() + record_store = WorkspaceRecordStore( + paths=WorkspacePaths(data_dir=data_dir), + cli=effective_cli, + device_id="device-test", + device_label="test-device", + ) + return MultiAccountSessionStore(data_dir=data_dir, cli=effective_cli, record_store=record_store) @pytest.fixture @@ -201,7 +277,9 @@ def make_resolver_with_data( raw = json.loads(agents_json) discovered = tuple( DiscoveredAgent( - host_id=HostId("host-00000000000000000000000000000000"), + # Honor a per-agent host id when the test data provides one so + # multi-workspace tests get distinct hosts; else the fixed id. + host_id=HostId(a.get("host", {}).get("id", _FIXED_TEST_HOST_ID)), agent_id=AgentId(a["id"]), agent_name=AgentName(a.get("name", a["id"])), provider_name=ProviderInstanceName("local"), diff --git a/apps/minds/imbue/minds/desktop_client/dek_store.py b/apps/minds/imbue/minds/desktop_client/dek_store.py new file mode 100644 index 0000000000..a9ad66f420 --- /dev/null +++ b/apps/minds/imbue/minds/desktop_client/dek_store.py @@ -0,0 +1,358 @@ +"""Per-account data-encryption-key (DEK) storage for workspace sync. + +Each signed-in account has a random 32-byte DEK that encrypts its workspace +records' secret payloads. Two files per account live under +``/keys/``: + +* ``.dek`` -- the raw DEK (0600). Its presence means the account is + "unlocked" on this device; day-to-day operation never needs the master + password. Created on first use. +* ``.bundle.json`` -- the local mirror of the password-wrapped DEK + bundle (the same JSON pushed to the connector). Present exactly when the + account's master password is non-empty; its absence IS the "no master + password" state, and password verification is an unwrap attempt against it. + +The master password's only role in the application is wrapping DEKs: setting +or changing it rewraps each account's DEK (nothing else moves), and clearing +it deletes the bundle (locally and server-side, where the caller also scrubs +synced secrets). + +The legacy per-install files (``backup_password_hash`` argon2 hash + +``backup_password`` plaintext convenience copy) are converted once by +:func:`convert_legacy_password_files` and renamed aside with a ``.pre-sync`` +suffix; no code path reads them afterwards. +""" + +import base64 +import json +import os +from collections.abc import Mapping +from collections.abc import Sequence +from pathlib import Path + +from argon2 import PasswordHasher +from argon2.exceptions import Argon2Error +from argon2.exceptions import InvalidHashError +from argon2.exceptions import VerificationError +from argon2.exceptions import VerifyMismatchError +from loguru import logger +from pydantic import SecretStr + +from imbue.imbue_common.secret_wrapping import KEY_LENGTH_BYTES +from imbue.imbue_common.secret_wrapping import KdfParameters +from imbue.imbue_common.secret_wrapping import SecretWrappingError +from imbue.imbue_common.secret_wrapping import derive_kek +from imbue.imbue_common.secret_wrapping import generate_dek +from imbue.imbue_common.secret_wrapping import generate_kdf_parameters +from imbue.imbue_common.secret_wrapping import unwrap_dek +from imbue.imbue_common.secret_wrapping import wrap_dek +from imbue.minds.config.data_types import WorkspacePaths +from imbue.minds.errors import SyncCryptoError + +_KEYS_DIRNAME = "keys" +_LEGACY_PASSWORD_FILENAME = "backup_password" +_LEGACY_PASSWORD_HASH_FILENAME = "backup_password_hash" +_LEGACY_RETIRED_SUFFIX = ".pre-sync" + +_PASSWORD_HASHER = PasswordHasher() + + +def keys_dir(paths: WorkspacePaths) -> Path: + """Return the directory holding per-account DEK + bundle-mirror files.""" + return paths.data_dir / _KEYS_DIRNAME + + +def dek_file_path(paths: WorkspacePaths, user_id: str) -> Path: + return keys_dir(paths) / f"{user_id}.dek" + + +def bundle_mirror_path(paths: WorkspacePaths, user_id: str) -> Path: + return keys_dir(paths) / f"{user_id}.bundle.json" + + +def _write_secret_bytes(path: Path, content: bytes) -> None: + """Atomically write a 0600 secret file (temp file + rename, never world-readable).""" + try: + path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = path.with_suffix(path.suffix + ".tmp") + fd = os.open(tmp_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + try: + os.write(fd, content) + finally: + os.close(fd) + tmp_path.rename(path) + except OSError as e: + raise SyncCryptoError(f"Could not write {path}: {e}") from e + + +def load_dek(paths: WorkspacePaths, user_id: str) -> bytes | None: + """Return the account's raw DEK, or None when this device is locked for it. + + A present-but-wrong-length file (truncated write, disk corruption) raises + :class:`SyncCryptoError` rather than being treated as locked: callers like + :func:`ensure_dek` would otherwise mint a fresh DEK and fork the account's + key lineage away from what the server bundle and other devices hold. + """ + path = dek_file_path(paths, user_id) + if not path.is_file(): + return None + try: + dek = path.read_bytes() + except OSError as e: + raise SyncCryptoError(f"Could not read the DEK file at {path}: {e}") from e + if len(dek) != KEY_LENGTH_BYTES: + raise SyncCryptoError( + f"The DEK file at {path} is corrupt ({len(dek)} bytes, expected {KEY_LENGTH_BYTES}); " + "remove it and unlock the account with the master password to restore the key" + ) + return dek + + +def ensure_dek(paths: WorkspacePaths, user_id: str) -> bytes: + """Return the account's DEK, generating and persisting a fresh one if absent.""" + existing = load_dek(paths, user_id) + if existing is not None: + return existing + dek = generate_dek() + _write_secret_bytes(dek_file_path(paths, user_id), dek) + logger.debug("Generated a fresh sync DEK for account {}", user_id[:8]) + return dek + + +def is_account_unlocked(paths: WorkspacePaths, user_id: str) -> bool: + """Whether this device holds the account's DEK (day-to-day secrets access works).""" + return dek_file_path(paths, user_id).is_file() + + +def delete_dek(paths: WorkspacePaths, user_id: str) -> None: + """Remove the account's local DEK file (used only by tests / explicit lock flows).""" + try: + dek_file_path(paths, user_id).unlink(missing_ok=True) + except OSError as e: + raise SyncCryptoError(f"Could not delete the DEK file for {user_id}: {e}") from e + + +# --------------------------------------------------------------------------- +# Bundle (password-wrapped DEK) helpers +# --------------------------------------------------------------------------- + + +def wrap_dek_to_bundle_json(dek: bytes, password: SecretStr, key_epoch: int) -> dict[str, object]: + """Wrap the DEK under the master password; returns the wire-shaped bundle JSON.""" + parameters = generate_kdf_parameters() + kek = derive_kek(password, parameters) + wrapped = wrap_dek(kek, dek) + return { + "kdf_salt": base64.b64encode(parameters.salt).decode("ascii"), + "kdf_time_cost": parameters.time_cost, + "kdf_memory_kib": parameters.memory_kib, + "kdf_parallelism": parameters.parallelism, + "wrapped_dek": base64.b64encode(wrapped).decode("ascii"), + "key_epoch": key_epoch, + } + + +def unwrap_bundle_json(bundle: Mapping[str, object], password: SecretStr) -> bytes: + """Recover the DEK from a wire-shaped bundle. + + Raises ``SecretWrappingError`` on a wrong password AND on a structurally + malformed bundle (missing fields, bad base64, invalid KDF costs) -- the + callers treat both as "this password cannot unlock this bundle", and a + bundle written by an unknown client version must never escape as an + untyped KeyError/ValueError. + """ + try: + parameters = KdfParameters( + salt=base64.b64decode(str(bundle["kdf_salt"])), + time_cost=int(str(bundle["kdf_time_cost"])), + memory_kib=int(str(bundle["kdf_memory_kib"])), + parallelism=int(str(bundle["kdf_parallelism"])), + ) + wrapped = base64.b64decode(str(bundle["wrapped_dek"])) + kek = derive_kek(password, parameters) + except (KeyError, ValueError, TypeError, Argon2Error) as e: + raise SecretWrappingError(f"Malformed key bundle: {e}") from e + return unwrap_dek(kek, wrapped) + + +def read_bundle_mirror(paths: WorkspacePaths, user_id: str) -> dict[str, object] | None: + """Return the locally-mirrored bundle JSON, or None when no master password is set.""" + path = bundle_mirror_path(paths, user_id) + if not path.is_file(): + return None + try: + raw = path.read_text() + except OSError as e: + raise SyncCryptoError(f"Could not read the bundle mirror at {path}: {e}") from e + try: + parsed = json.loads(raw) + except json.JSONDecodeError as e: + # The mirror is internal state; a corrupt copy must not brick startup. + # The server copy (or a fresh password set) rewrites it. + logger.warning("Ignoring corrupt bundle mirror at {}: {}", path, e) + return None + return parsed if isinstance(parsed, dict) else None + + +def write_bundle_mirror(paths: WorkspacePaths, user_id: str, bundle: Mapping[str, object]) -> None: + _write_secret_bytes(bundle_mirror_path(paths, user_id), json.dumps(dict(bundle), indent=2).encode("utf-8")) + + +def delete_bundle_mirror(paths: WorkspacePaths, user_id: str) -> None: + try: + bundle_mirror_path(paths, user_id).unlink(missing_ok=True) + except OSError as e: + raise SyncCryptoError(f"Could not delete the bundle mirror for {user_id}: {e}") from e + + +def is_master_password_set_for_account(paths: WorkspacePaths, user_id: str) -> bool: + """Whether the account has a non-empty master password (a bundle exists exactly then).""" + return bundle_mirror_path(paths, user_id).is_file() + + +def verify_master_password_for_account(paths: WorkspacePaths, user_id: str, candidate: SecretStr) -> bool: + """Check ``candidate`` by attempting the unwrap; no bundle means only the empty password matches.""" + bundle = read_bundle_mirror(paths, user_id) + if bundle is None: + return candidate.get_secret_value() == "" + try: + unwrap_bundle_json(bundle, candidate) + except SecretWrappingError: + return False + return True + + +def set_master_password_for_account( + paths: WorkspacePaths, user_id: str, new_password: SecretStr +) -> dict[str, object] | None: + """(Re)wrap the account's DEK under ``new_password`` and update the local mirror. + + Returns the new bundle JSON (for the caller to push to the connector), or + None when the new password is empty -- the mirror is deleted and the caller + is responsible for the server-side bundle delete + secrets scrub. + """ + dek = ensure_dek(paths, user_id) + previous = read_bundle_mirror(paths, user_id) + key_epoch = int(str(previous["key_epoch"])) if previous is not None and "key_epoch" in previous else 1 + if not new_password.get_secret_value(): + delete_bundle_mirror(paths, user_id) + return None + bundle = wrap_dek_to_bundle_json(dek, new_password, key_epoch) + write_bundle_mirror(paths, user_id, bundle) + return bundle + + +def unlock_account_with_bundle( + paths: WorkspacePaths, user_id: str, bundle: Mapping[str, object], password: SecretStr +) -> bytes: + """Unwrap a (server-fetched) bundle and persist both the DEK and the mirror. + + Raises ``SecretWrappingError`` when the password is wrong. This is the + new-device unlock: afterwards the account works without the password. + """ + dek = unwrap_bundle_json(bundle, password) + _write_secret_bytes(dek_file_path(paths, user_id), dek) + write_bundle_mirror(paths, user_id, bundle) + return dek + + +# --------------------------------------------------------------------------- +# Legacy backup_password / backup_password_hash conversion +# --------------------------------------------------------------------------- + + +def _legacy_password_path(paths: WorkspacePaths) -> Path: + return paths.data_dir / _LEGACY_PASSWORD_FILENAME + + +def _legacy_hash_path(paths: WorkspacePaths) -> Path: + return paths.data_dir / _LEGACY_PASSWORD_HASH_FILENAME + + +def _matches_legacy_hash(stored_hash: str, candidate: str) -> bool: + if not stored_hash: + return candidate == "" + try: + _PASSWORD_HASHER.verify(stored_hash, candidate) + except (VerifyMismatchError, VerificationError, InvalidHashError): + return False + return True + + +def _read_legacy_master_password(paths: WorkspacePaths) -> SecretStr | None: + """Recover the legacy master password when it is knowable, else None. + + Knowable means: the plaintext convenience copy exists and matches the + hash (or there is no hash at all -- the pre-hash layout, where the + plaintext IS the password, exactly as ``ensure_backup_password_hash`` + used to seed it), or the hash is the empty-password seed (password = + ""). A non-empty hash with no (valid) plaintext copy is unknowable -- + the user keeps their repos (workspace keys are untouched) but must set + a fresh password. + """ + hash_path = _legacy_hash_path(paths) + stored_hash = "" + if hash_path.is_file(): + try: + stored_hash = hash_path.read_text().strip() + except OSError as e: + logger.warning("Could not read the legacy password hash at {}: {}", hash_path, e) + return None + plaintext_path = _legacy_password_path(paths) + if plaintext_path.is_file(): + try: + saved = plaintext_path.read_text().strip() + except OSError as e: + logger.warning("Could not read the legacy password copy at {}: {}", plaintext_path, e) + saved = "" + if saved and (not stored_hash or _matches_legacy_hash(stored_hash, saved)): + return SecretStr(saved) + if _matches_legacy_hash(stored_hash, ""): + return SecretStr("") + return None + + +def _retire_legacy_file(path: Path) -> None: + if not path.is_file(): + return + try: + path.rename(path.with_name(path.name + _LEGACY_RETIRED_SUFFIX)) + except OSError as e: + logger.warning("Could not retire the legacy password file {}: {}", path, e) + + +def convert_legacy_password_files(paths: WorkspacePaths, user_ids: Sequence[str]) -> None: + """One-shot conversion of the legacy per-install password files into per-account bundles. + + For each signed-in account: ensure a DEK exists, and -- when the legacy + master password is recoverable and non-empty -- wrap the DEK with it so + the user's password carries over seamlessly. The legacy files are then + renamed aside (``.pre-sync``); an unrecoverable non-empty password means + the user starts in the "no master password" state and sets a fresh one + (their repos stay reachable through the untouched workspace keys). + + Idempotent: a second run finds no legacy files and does nothing. No-op + (files kept) when no account is signed in yet, so a pre-signin install + converts on the first signed-in run. + """ + has_legacy = _legacy_hash_path(paths).is_file() or _legacy_password_path(paths).is_file() + if not has_legacy or not user_ids: + return + legacy_password = _read_legacy_master_password(paths) + for user_id in user_ids: + ensure_dek(paths, user_id) + if ( + legacy_password is not None + and legacy_password.get_secret_value() + and not is_master_password_set_for_account(paths, user_id) + ): + set_master_password_for_account(paths, user_id, legacy_password) + logger.info("Carried the legacy backup master password over to account {}", user_id[:8]) + if legacy_password is None: + logger.warning( + "The legacy backup master password could not be recovered (hash present, no saved copy); " + "existing repos stay reachable via their workspace keys, but a fresh master password must be set " + "on the Settings page before secrets can sync." + ) + _retire_legacy_file(_legacy_password_path(paths)) + _retire_legacy_file(_legacy_hash_path(paths)) diff --git a/apps/minds/imbue/minds/desktop_client/dek_store_test.py b/apps/minds/imbue/minds/desktop_client/dek_store_test.py new file mode 100644 index 0000000000..5cf1dc5c77 --- /dev/null +++ b/apps/minds/imbue/minds/desktop_client/dek_store_test.py @@ -0,0 +1,253 @@ +import stat +from pathlib import Path +from uuid import uuid4 + +import pytest +from argon2 import PasswordHasher +from pydantic import SecretStr + +from imbue.imbue_common.secret_wrapping import SecretWrappingError +from imbue.minds.config.data_types import WorkspacePaths +from imbue.minds.desktop_client.dek_store import bundle_mirror_path +from imbue.minds.desktop_client.dek_store import convert_legacy_password_files +from imbue.minds.desktop_client.dek_store import dek_file_path +from imbue.minds.desktop_client.dek_store import ensure_dek +from imbue.minds.desktop_client.dek_store import is_account_unlocked +from imbue.minds.desktop_client.dek_store import is_master_password_set_for_account +from imbue.minds.desktop_client.dek_store import load_dek +from imbue.minds.desktop_client.dek_store import read_bundle_mirror +from imbue.minds.desktop_client.dek_store import set_master_password_for_account +from imbue.minds.desktop_client.dek_store import unlock_account_with_bundle +from imbue.minds.desktop_client.dek_store import unwrap_bundle_json +from imbue.minds.desktop_client.dek_store import verify_master_password_for_account +from imbue.minds.errors import SyncCryptoError + + +@pytest.fixture +def paths(tmp_path: Path) -> WorkspacePaths: + return WorkspacePaths(data_dir=tmp_path) + + +def _user_id() -> str: + return uuid4().hex + + +def test_ensure_dek_creates_a_0600_file_and_is_stable(paths: WorkspacePaths) -> None: + user_id = _user_id() + dek = ensure_dek(paths, user_id) + + assert len(dek) == 32 + assert ensure_dek(paths, user_id) == dek + assert load_dek(paths, user_id) == dek + mode = stat.S_IMODE(dek_file_path(paths, user_id).stat().st_mode) + assert mode == 0o600 + assert is_account_unlocked(paths, user_id) + + +def test_load_dek_returns_none_when_locked(paths: WorkspacePaths) -> None: + assert load_dek(paths, _user_id()) is None + assert not is_account_unlocked(paths, _user_id()) + + +def test_load_dek_raises_a_typed_error_for_a_corrupt_file(paths: WorkspacePaths) -> None: + # A truncated DEK must raise (typed) rather than read as locked: treating + # it as absent would let ensure_dek mint a fresh DEK and fork the account's + # key lineage away from the server bundle and other devices. + user_id = _user_id() + ensure_dek(paths, user_id) + dek_file_path(paths, user_id).write_bytes(b"truncated") + + with pytest.raises(SyncCryptoError): + load_dek(paths, user_id) + with pytest.raises(SyncCryptoError): + ensure_dek(paths, user_id) + + # Re-unlocking with the wrapped bundle rewrites a valid DEK over the wreck. + other_device = WorkspacePaths(data_dir=paths.data_dir / "other-device") + dek = ensure_dek(other_device, user_id) + bundle = set_master_password_for_account(other_device, user_id, SecretStr("hunter2")) + assert bundle is not None + assert unlock_account_with_bundle(paths, user_id, bundle, SecretStr("hunter2")) == dek + assert load_dek(paths, user_id) == dek + + +def test_set_master_password_writes_bundle_and_verification_works(paths: WorkspacePaths) -> None: + user_id = _user_id() + bundle = set_master_password_for_account(paths, user_id, SecretStr("hunter2")) + + assert bundle is not None + assert is_master_password_set_for_account(paths, user_id) + assert verify_master_password_for_account(paths, user_id, SecretStr("hunter2")) + assert not verify_master_password_for_account(paths, user_id, SecretStr("wrong")) + assert not verify_master_password_for_account(paths, user_id, SecretStr("")) + # The bundle unwraps back to the on-disk DEK. + assert unwrap_bundle_json(bundle, SecretStr("hunter2")) == load_dek(paths, user_id) + + +def test_empty_password_state_verifies_only_the_empty_password(paths: WorkspacePaths) -> None: + user_id = _user_id() + ensure_dek(paths, user_id) + + assert not is_master_password_set_for_account(paths, user_id) + assert verify_master_password_for_account(paths, user_id, SecretStr("")) + assert not verify_master_password_for_account(paths, user_id, SecretStr("anything")) + + +def test_clearing_the_password_deletes_the_bundle_mirror(paths: WorkspacePaths) -> None: + user_id = _user_id() + set_master_password_for_account(paths, user_id, SecretStr("hunter2")) + + result = set_master_password_for_account(paths, user_id, SecretStr("")) + + assert result is None + assert not is_master_password_set_for_account(paths, user_id) + # The DEK itself is untouched by a password change. + assert load_dek(paths, user_id) is not None + + +def test_password_change_rewraps_without_changing_the_dek(paths: WorkspacePaths) -> None: + user_id = _user_id() + first_bundle = set_master_password_for_account(paths, user_id, SecretStr("old")) + dek_before = load_dek(paths, user_id) + + second_bundle = set_master_password_for_account(paths, user_id, SecretStr("new")) + + assert load_dek(paths, user_id) == dek_before + assert second_bundle is not None and first_bundle is not None + assert second_bundle["key_epoch"] == first_bundle["key_epoch"] + assert unwrap_bundle_json(second_bundle, SecretStr("new")) == dek_before + with pytest.raises(SecretWrappingError): + unwrap_bundle_json(second_bundle, SecretStr("old")) + + +def test_unlock_account_with_bundle_installs_dek_and_mirror(paths: WorkspacePaths) -> None: + # Simulate device A wrapping, then device B (fresh paths) unlocking. + user_id = _user_id() + device_a = WorkspacePaths(data_dir=paths.data_dir / "device-a") + dek = ensure_dek(device_a, user_id) + bundle = set_master_password_for_account(device_a, user_id, SecretStr("hunter2")) + assert bundle is not None + + device_b = WorkspacePaths(data_dir=paths.data_dir / "device-b") + with pytest.raises(SecretWrappingError): + unlock_account_with_bundle(device_b, user_id, bundle, SecretStr("wrong")) + unlocked = unlock_account_with_bundle(device_b, user_id, bundle, SecretStr("hunter2")) + + assert unlocked == dek + assert load_dek(device_b, user_id) == dek + assert read_bundle_mirror(device_b, user_id) == bundle + + +def test_convert_legacy_files_carries_over_a_saved_password(paths: WorkspacePaths) -> None: + user_id = _user_id() + (paths.data_dir / "backup_password").write_text("legacy-pass\n") + (paths.data_dir / "backup_password_hash").write_text(PasswordHasher().hash("legacy-pass")) + + convert_legacy_password_files(paths, [user_id]) + + assert is_master_password_set_for_account(paths, user_id) + assert verify_master_password_for_account(paths, user_id, SecretStr("legacy-pass")) + assert not (paths.data_dir / "backup_password").exists() + assert not (paths.data_dir / "backup_password_hash").exists() + assert (paths.data_dir / "backup_password.pre-sync").exists() + assert (paths.data_dir / "backup_password_hash.pre-sync").exists() + + +def test_convert_legacy_files_carries_over_a_pre_hash_plaintext_password(paths: WorkspacePaths) -> None: + # The pre-hash layout: only the plaintext file exists. The retired + # ensure_backup_password_hash seeded the hash from it, so the plaintext + # IS the master password and must carry over. + user_id = _user_id() + (paths.data_dir / "backup_password").write_text("pre-hash-pass\n") + + convert_legacy_password_files(paths, [user_id]) + + assert is_master_password_set_for_account(paths, user_id) + assert verify_master_password_for_account(paths, user_id, SecretStr("pre-hash-pass")) + assert (paths.data_dir / "backup_password.pre-sync").exists() + + +def test_convert_legacy_files_with_empty_password_seed_sets_no_password(paths: WorkspacePaths) -> None: + user_id = _user_id() + (paths.data_dir / "backup_password_hash").write_text(PasswordHasher().hash("")) + + convert_legacy_password_files(paths, [user_id]) + + assert is_account_unlocked(paths, user_id) + assert not is_master_password_set_for_account(paths, user_id) + assert not (paths.data_dir / "backup_password_hash").exists() + + +def test_convert_legacy_files_with_unknowable_password_starts_fresh(paths: WorkspacePaths) -> None: + user_id = _user_id() + (paths.data_dir / "backup_password_hash").write_text(PasswordHasher().hash("forgotten")) + + convert_legacy_password_files(paths, [user_id]) + + assert is_account_unlocked(paths, user_id) + assert not is_master_password_set_for_account(paths, user_id) + assert (paths.data_dir / "backup_password_hash.pre-sync").exists() + + +def test_convert_legacy_files_is_a_no_op_without_accounts(paths: WorkspacePaths) -> None: + (paths.data_dir / "backup_password_hash").write_text(PasswordHasher().hash("keep-me")) + + convert_legacy_password_files(paths, []) + + # Files stay for the first signed-in run to convert. + assert (paths.data_dir / "backup_password_hash").exists() + + +def test_convert_legacy_files_is_idempotent(paths: WorkspacePaths) -> None: + user_id = _user_id() + (paths.data_dir / "backup_password").write_text("legacy-pass\n") + (paths.data_dir / "backup_password_hash").write_text(PasswordHasher().hash("legacy-pass")) + + convert_legacy_password_files(paths, [user_id]) + bundle_after_first = read_bundle_mirror(paths, user_id) + convert_legacy_password_files(paths, [user_id]) + + assert read_bundle_mirror(paths, user_id) == bundle_after_first + + +def test_unwrap_bundle_json_raises_typed_error_for_malformed_bundles(paths: WorkspacePaths) -> None: + # A bundle written by an unknown client version (or a corrupt connector + # row) must surface as SecretWrappingError, never KeyError/ValueError. + user_id = _user_id() + bundle = set_master_password_for_account(paths, user_id, SecretStr("hunter2")) + assert bundle is not None + + missing_field = {key: value for key, value in bundle.items() if key != "kdf_salt"} + with pytest.raises(SecretWrappingError): + unwrap_bundle_json(missing_field, SecretStr("hunter2")) + + bad_base64 = dict(bundle) + bad_base64["wrapped_dek"] = "not base64!!!" + with pytest.raises(SecretWrappingError): + unwrap_bundle_json(bad_base64, SecretStr("hunter2")) + + bad_cost = dict(bundle) + bad_cost["kdf_time_cost"] = "not-a-number" + with pytest.raises(SecretWrappingError): + unwrap_bundle_json(bad_cost, SecretStr("hunter2")) + + +def test_verify_master_password_returns_false_for_a_wrong_shaped_mirror(paths: WorkspacePaths) -> None: + # Valid JSON dict, wrong shape: verification must fail closed, not raise. + user_id = _user_id() + ensure_dek(paths, user_id) + bundle_mirror_path(paths, user_id).parent.mkdir(parents=True, exist_ok=True) + bundle_mirror_path(paths, user_id).write_text('{"unexpected": "shape"}') + + assert not verify_master_password_for_account(paths, user_id, SecretStr("x")) + + +def test_corrupt_bundle_mirror_is_treated_as_no_password(paths: WorkspacePaths) -> None: + user_id = _user_id() + ensure_dek(paths, user_id) + bundle_mirror_path(paths, user_id).parent.mkdir(parents=True, exist_ok=True) + bundle_mirror_path(paths, user_id).write_text("{not json") + + assert read_bundle_mirror(paths, user_id) is None + assert not verify_master_password_for_account(paths, user_id, SecretStr("x")) + assert verify_master_password_for_account(paths, user_id, SecretStr("")) diff --git a/apps/minds/imbue/minds/desktop_client/e2e_workspace_runner.py b/apps/minds/imbue/minds/desktop_client/e2e_workspace_runner.py index 3be5cbc642..530a665a3e 100644 --- a/apps/minds/imbue/minds/desktop_client/e2e_workspace_runner.py +++ b/apps/minds/imbue/minds/desktop_client/e2e_workspace_runner.py @@ -794,6 +794,68 @@ def _attempt_create_workspace_via_electron( browser.close() +@contextmanager +def electron_app_session( + workspace_git_url: Path, + debug_port: int, + host_config_dir: Path | None = None, +) -> Iterator[tuple[Browser, Page]]: + """Launch Electron + attach Playwright and yield ``(browser, content_page)``. + + The generic sibling of :func:`create_workspace_via_electron` for flows that + drive arbitrary app pages (sign-in, settings, the landing list) instead of + the create form. The launch + CDP attach is retried with a fresh Electron + process and port up to ``_ELECTRON_LAUNCH_ATTEMPTS`` times (the same + wedged-handshake flake recovery); once the session is yielded, caller + exceptions propagate unchanged and tear the app down. + + The same caller contract as :func:`create_workspace_via_electron` applies + (``MINDS_ROOT_NAME`` set, ``debug_port`` free, ``host_config_dir`` for the + pytest config guard). ``workspace_git_url`` only seeds the create form's + repo prefill; sessions that never open the create form still need a real + path here. + """ + last_error: _ElectronConnectError | None = None + for attempt in range(1, _ELECTRON_LAUNCH_ATTEMPTS + 1): + attempt_port = debug_port if attempt == 1 else find_free_port() + with _launched_electron(workspace_git_url, attempt_port, host_config_dir): + with sync_playwright() as playwright: + try: + _wait_for_cdp(attempt_port, _CDP_READY_TIMEOUT_SECONDS) + browser = playwright.chromium.connect_over_cdp( + f"http://127.0.0.1:{attempt_port}", timeout=_CDP_CONNECT_TIMEOUT_MS + ) + except (PlaywrightError, TimeoutError) as exc: + last_error = _ElectronConnectError(f"Electron CDP attach failed on port {attempt_port}: {exc}") + logger.warning( + "Electron launch/CDP attempt {}/{} failed; relaunching: {}", + attempt, + _ELECTRON_LAUNCH_ATTEMPTS, + last_error, + ) + continue + try: + try: + page = _pick_content_page(browser, _BACKEND_READY_TIMEOUT_SECONDS) + except (PlaywrightError, TimeoutError) as exc: + last_error = _ElectronConnectError(f"Electron CDP attach failed on port {attempt_port}: {exc}") + logger.warning( + "Electron launch/CDP attempt {}/{} failed; relaunching: {}", + attempt, + _ELECTRON_LAUNCH_ATTEMPTS, + last_error, + ) + continue + _attach_renderer_diagnostics(page) + yield browser, page + return + finally: + browser.close() + raise PlaywrightTimeoutError( + f"Electron CDP attach failed after {_ELECTRON_LAUNCH_ATTEMPTS} relaunch attempts (last error: {last_error})" + ) + + def create_workspace_via_electron( default_workspace_template_path: Path, workspace_name: str, diff --git a/apps/minds/imbue/minds/desktop_client/imbue_cloud_cli.py b/apps/minds/imbue/minds/desktop_client/imbue_cloud_cli.py index 21118460f2..6c2bbe3213 100644 --- a/apps/minds/imbue/minds/desktop_client/imbue_cloud_cli.py +++ b/apps/minds/imbue/minds/desktop_client/imbue_cloud_cli.py @@ -16,6 +16,8 @@ """ import json as _json +import os +import tempfile import time from collections.abc import Mapping from collections.abc import Sequence @@ -70,6 +72,16 @@ class ImbueCloudUnavailableError(ImbueCloudCliError): """Subclass of CliError indicating the connector returned 503 (no matching pool host).""" +class ImbueCloudSyncConflictCliError(ImbueCloudCliError): + """A record push hit a 409 (revision CAS or active-agent conflict). + + ``stored_record`` carries the server's current row when the conflict was a + revision CAS failure, so the caller can rebase and retry; None otherwise. + """ + + stored_record: dict[str, Any] | None = None + + class ImbueCloudAuthSession(FrozenModel): """Result of a successful auth signin/signup/oauth invocation.""" @@ -631,6 +643,115 @@ def create_bucket_key( body = self._expect_success(result, "bucket keys create") return R2BucketKeyMaterial.model_validate(body) + # ------------------------------------------------------------------ + # Workspace sync (records + key bundle) + # ------------------------------------------------------------------ + + def sync_records_pull(self, account: str) -> list[dict[str, Any]]: + result = self._run(["sync", "records", "pull", "--account", account], cg_name="imbue-cloud-sync-records-pull") + body = self._expect_success(result, "sync records pull") + records = body.get("records", []) if isinstance(body, dict) else [] + return [entry for entry in records if isinstance(entry, dict)] + + def sync_record_push(self, account: str, record: Mapping[str, Any]) -> dict[str, Any]: + """Push one record; returns the stored row. Raises ImbueCloudSyncConflictCliError on a 409. + + The record JSON travels via a 0600 temp file (--input-file) so secret + payloads never ride a command line or a log. + """ + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle: + os.fchmod(handle.fileno(), 0o600) + _json.dump(dict(record), handle) + input_path = handle.name + try: + result = self._run( + ["sync", "records", "push", "--account", account, "--input-file", input_path], + cg_name="imbue-cloud-sync-record-push", + ) + finally: + Path(input_path).unlink(missing_ok=True) + if result.returncode != 0 and "ImbueCloudSyncConflictError" in result.stderr: + conflict = ImbueCloudSyncConflictCliError("sync records push: revision/agent conflict") + conflict.exit_code = result.returncode if result.returncode is not None else 1 + conflict.stdout = result.stdout + conflict.stderr = result.stderr + conflict.stored_record = _parse_conflict_stored(result.stderr) + raise conflict + body = self._expect_success(result, "sync records push") + return body if isinstance(body, dict) else {} + + def sync_record_delete(self, account: str, host_id: str) -> None: + result = self._run( + ["sync", "records", "delete", host_id, "--account", account], + cg_name="imbue-cloud-sync-record-delete", + ) + self._expect_success(result, "sync records delete") + + def sync_scrub_secrets(self, account: str) -> int: + result = self._run(["sync", "scrub-secrets", "--account", account], cg_name="imbue-cloud-sync-scrub") + body = self._expect_success(result, "sync scrub-secrets") + return int(body.get("scrubbed", 0)) if isinstance(body, dict) else 0 + + def sync_bundle_pull(self, account: str) -> dict[str, Any] | None: + result = self._run(["sync", "bundle", "pull", "--account", account], cg_name="imbue-cloud-sync-bundle-pull") + body = self._expect_success(result, "sync bundle pull") + bundle = body.get("bundle") if isinstance(body, dict) else None + return bundle if isinstance(bundle, dict) else None + + def sync_bundle_push(self, account: str, bundle: Mapping[str, Any]) -> None: + with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle: + os.fchmod(handle.fileno(), 0o600) + _json.dump(dict(bundle), handle) + input_path = handle.name + try: + result = self._run( + ["sync", "bundle", "push", "--account", account, "--input-file", input_path], + cg_name="imbue-cloud-sync-bundle-push", + ) + finally: + Path(input_path).unlink(missing_ok=True) + self._expect_success(result, "sync bundle push") + + def sync_bundle_delete(self, account: str) -> None: + result = self._run( + ["sync", "bundle", "delete", "--account", account], cg_name="imbue-cloud-sync-bundle-delete" + ) + self._expect_success(result, "sync bundle delete") + + +def _parse_conflict_stored(stderr: str) -> dict[str, Any] | None: + """Extract the ``stored`` row from a sync-push conflict's JSON error body, if present. + + The body is indent-formatted JSON (its first line is a bare ``{``) that + may be surrounded by log lines, so each candidate document is raw-decoded + from the opening brace's actual byte offset -- that consumes exactly one + document regardless of what precedes or follows it on the stream. + """ + decoder = _json.JSONDecoder() + offset = 0 + is_any_document_parsed = False + for line in stderr.splitlines(keepends=True): + lstripped = line.lstrip() + if lstripped.startswith("{"): + try: + parsed, _consumed_until = decoder.raw_decode(stderr, offset + len(line) - len(lstripped)) + except _json.JSONDecodeError as exc: + # Some other output line merely started with a brace; keep + # scanning for the real error body. + logger.warning( + "Skipping a brace-prefixed non-JSON stderr line while locating the conflict body: {}", exc + ) + parsed = None + if isinstance(parsed, dict): + is_any_document_parsed = True + stored = parsed.get("stored") + if isinstance(stored, dict): + return stored + offset += len(line) + if not is_any_document_parsed: + logger.warning("Could not locate a JSON error body on the sync-conflict stderr") + return None + def _parse_stdout_json(stdout: str, command_repr: str) -> Any: """Parse the JSON document the plugin emits on a successful invocation. diff --git a/apps/minds/imbue/minds/desktop_client/imbue_cloud_cli_test.py b/apps/minds/imbue/minds/desktop_client/imbue_cloud_cli_test.py index bd8eefe159..8813491423 100644 --- a/apps/minds/imbue/minds/desktop_client/imbue_cloud_cli_test.py +++ b/apps/minds/imbue/minds/desktop_client/imbue_cloud_cli_test.py @@ -8,6 +8,7 @@ from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCli from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCliError from imbue.minds.desktop_client.imbue_cloud_cli import _CONNECTOR_URL_SUBPROCESS_ENV +from imbue.minds.desktop_client.imbue_cloud_cli import _parse_conflict_stored from imbue.minds.utils.mngr_caller import MngrCallResult from imbue.minds.utils.testing import RecordingMngrCaller @@ -53,3 +54,21 @@ def test_run_routes_through_mngr_caller_with_home_cwd_and_connector_env() -> Non assert recorded.cwd == Path.home() # The trailing slash is stripped so the plugin builds clean URLs. assert recorded.env_overrides == {_CONNECTOR_URL_SUBPROCESS_ENV: "https://connector.example"} + + +def test_parse_conflict_stored_survives_surrounding_log_lines() -> None: + """The indent-formatted error body may be preceded by log lines containing + braces and followed by trailing output; the stored row must still parse.""" + body = json.dumps({"error": "conflict", "error_class": "X", "stored": {"host_id": "h1", "revision": 4}}, indent=2) + stderr = ( + "2026-07-12 10:00:00 | WARNING | retrying {attempt 1} after HTTP 409\n" + body + "\nsome trailing log line\n" + ) + assert _parse_conflict_stored(stderr) == {"host_id": "h1", "revision": 4} + + +def test_parse_conflict_stored_returns_none_without_a_stored_row() -> None: + # The active-agent-conflict shape carries no stored row. + body = json.dumps({"error": "another ACTIVE record exists", "stored": None}, indent=2) + assert _parse_conflict_stored(body) is None + # Brace-free stderr (no JSON document at all) parses to None too. + assert _parse_conflict_stored("plain traceback text\nwithout any json\n") is None diff --git a/apps/minds/imbue/minds/desktop_client/restic_cli.py b/apps/minds/imbue/minds/desktop_client/restic_cli.py index 86093912ca..3e8d0f7fd8 100644 --- a/apps/minds/imbue/minds/desktop_client/restic_cli.py +++ b/apps/minds/imbue/minds/desktop_client/restic_cli.py @@ -18,7 +18,6 @@ from datetime import datetime from datetime import timezone from pathlib import Path -from tempfile import TemporaryDirectory from typing import Final from typing import NoReturn from typing import TypeVar @@ -223,132 +222,6 @@ def init_repo( ) -def _add_password_key_once( - *, - repository: str, - backend_env: Mapping[str, str], - existing_password: str | None, - new_password: str, - parent_cg: ConcurrencyGroup | None, -) -> None: - """Run a single ``restic key add`` attempt, authenticating with ``existing_password``. - - An empty ``new_password`` adds the empty-password key via - ``--new-insecure-no-password`` (restic rejects an empty password file). - """ - env, flags = _env_and_flags(repository, backend_env, existing_password) - if not new_password: - result = _run_restic( - [*flags, "key", "add", "--new-insecure-no-password"], - env_overrides=env, - parent_cg=parent_cg, - timeout_seconds=_DEFAULT_TIMEOUT_SECONDS, - ) - if result.returncode != 0: - _raise_restic_failure("restic key add", result.returncode, result.stderr) - return - with TemporaryDirectory() as temp_dir: - new_password_file = Path(temp_dir) / "new_password" - # 0600 temp file so the random key isn't briefly world-readable on disk. - fd = os.open(new_password_file, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - try: - os.write(fd, new_password.encode("utf-8")) - finally: - os.close(fd) - result = _run_restic( - [*flags, "key", "add", "--new-password-file", str(new_password_file)], - env_overrides=env, - parent_cg=parent_cg, - timeout_seconds=_DEFAULT_TIMEOUT_SECONDS, - ) - if result.returncode != 0: - _raise_restic_failure("restic key add", result.returncode, result.stderr) - - -def add_password_key( - *, - repository: str, - backend_env: Mapping[str, str], - existing_password: str | None, - new_password: str, - parent_cg: ConcurrencyGroup | None = None, -) -> None: - """Add ``new_password`` as an additional key, authenticating with ``existing_password``. - - Retries a transient auth failure for a bounded window (see ``init_repo``). - """ - _retry_on_transient_auth( - lambda: _add_password_key_once( - repository=repository, - backend_env=backend_env, - existing_password=existing_password, - new_password=new_password, - parent_cg=parent_cg, - ) - ) - - -class ResticKey(FrozenModel): - """One repository key as reported by ``restic key list --json``.""" - - key_id: str = Field(description="The key's id (hex prefix restic accepts for key remove)") - is_current: bool = Field(description="Whether this is the key the listing authenticated with") - - -def list_keys( - *, - repository: str, - backend_env: Mapping[str, str], - password: str | None, - parent_cg: ConcurrencyGroup | None = None, - timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS, -) -> tuple[ResticKey, ...]: - """List the repository's keys, marking the one used to authenticate.""" - env, flags = _env_and_flags(repository, backend_env, password) - result = _run_restic( - [*flags, "--no-lock", "key", "list", "--json"], - env_overrides=env, - parent_cg=parent_cg, - timeout_seconds=timeout_seconds, - ) - if result.returncode != 0: - raise BackupProvisioningError(f"restic key list failed (exit {result.returncode}): {result.stderr.strip()}") - try: - raw = json.loads(result.stdout or "[]") - except ValueError as e: - raise BackupProvisioningError(f"restic key list returned non-JSON output: {e}") from e - if not isinstance(raw, list): - raise BackupProvisioningError(f"restic key list returned a non-list JSON payload: {type(raw).__name__}") - keys: list[ResticKey] = [] - for entry in raw: - if not isinstance(entry, dict) or not entry.get("id"): - logger.warning("Skipping malformed restic key entry: {!r}", entry) - continue - keys.append(ResticKey(key_id=str(entry["id"]), is_current=bool(entry.get("current")))) - return tuple(keys) - - -def remove_key( - *, - repository: str, - backend_env: Mapping[str, str], - password: str | None, - key_id: str, - parent_cg: ConcurrencyGroup | None = None, - timeout_seconds: float = _DEFAULT_TIMEOUT_SECONDS, -) -> None: - """Remove one key from the repository (restic refuses to remove the current key).""" - env, flags = _env_and_flags(repository, backend_env, password) - result = _run_restic( - [*flags, "key", "remove", key_id], - env_overrides=env, - parent_cg=parent_cg, - timeout_seconds=timeout_seconds, - ) - if result.returncode != 0: - raise BackupProvisioningError(f"restic key remove failed (exit {result.returncode}): {result.stderr.strip()}") - - def restore_snapshot( *, repository: str, diff --git a/apps/minds/imbue/minds/desktop_client/restic_cli_test.py b/apps/minds/imbue/minds/desktop_client/restic_cli_test.py index 9c643fca15..847e7853ce 100644 --- a/apps/minds/imbue/minds/desktop_client/restic_cli_test.py +++ b/apps/minds/imbue/minds/desktop_client/restic_cli_test.py @@ -148,16 +148,12 @@ def test_ensure_restic_available_does_not_raise() -> None: @pytest.mark.timeout(60) -def test_init_add_key_and_status_against_local_repo(tmp_path: Path) -> None: +def test_init_and_status_against_local_repo(tmp_path: Path) -> None: repo = str(tmp_path / "repo") - master = "master-passphrase" workspace_password = "workspace-random-key" - # Init with the master password, then add the random workspace key. - restic_cli.init_repo(repository=repo, backend_env={}, password=master) - restic_cli.add_password_key( - repository=repo, backend_env={}, existing_password=master, new_password=workspace_password - ) + # Init with the workspace's own random password -- the repo's single key. + restic_cli.init_repo(repository=repo, backend_env={}, password=workspace_password) now = datetime.now(timezone.utc) # Fresh repo: no snapshots, no in-progress lock -- queried with the diff --git a/apps/minds/imbue/minds/desktop_client/session_store.py b/apps/minds/imbue/minds/desktop_client/session_store.py index 6e3728c102..08d665e571 100644 --- a/apps/minds/imbue/minds/desktop_client/session_store.py +++ b/apps/minds/imbue/minds/desktop_client/session_store.py @@ -2,20 +2,16 @@ The mngr_imbue_cloud plugin owns the SuperTokens session state on disk (tokens, the email -> user_id index, and the active-account marker). -Minds keeps only one piece of state the plugin can't know about: the -association between a workspace agent_id and the user_id of the account -that owns it. - -When callers need account *identity* (email, display_name) the store -fetches it on demand from the plugin via -``ImbueCloudCli.auth_list()``. Results are cached in memory so the -chrome SSE / workspace list rendering paths don't fan out into -subprocesses on every poll. Sign-in / sign-out flows must call -:meth:`invalidate_identity_cache` so the cache stays in sync with the -plugin's view of who is signed in. +Association is record existence: a workspace belongs to the account whose +workspace-record replica (see ``workspace_record_store``) holds an ACTIVE +record for it. This store joins that view with account *identity* (email, +display_name), which it fetches on demand from the plugin via +``ImbueCloudCli.auth_list()`` and caches in memory so the chrome SSE / +workspace list rendering paths don't fan out into subprocesses on every +poll. Sign-in / sign-out flows must call :meth:`invalidate_identity_cache` +so the cache stays in sync with the plugin's view of who is signed in. """ -import json import threading from pathlib import Path @@ -26,13 +22,17 @@ from imbue.imbue_common.frozen_model import FrozenModel from imbue.imbue_common.mutable_model import MutableModel from imbue.imbue_common.primitives import NonEmptyStr +from imbue.minds.desktop_client.backend_resolver import BackendResolverInterface from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudAuthAccount from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCli from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCliError +from imbue.minds.desktop_client.workspace_record_store import ReplicaRecord +from imbue.minds.desktop_client.workspace_record_store import WorkspaceRecordStore +from imbue.minds.errors import WorkspaceSyncError -_WORKSPACE_ASSOCIATIONS_FILENAME = "workspace_associations.json" -_LEGACY_SESSIONS_FILENAME = "sessions.json" _USER_ID_PREFIX_LENGTH = 16 +_LEGACY_ASSOCIATIONS_FILENAME = "workspace_associations.json" +_LEGACY_SESSIONS_FILENAME = "sessions.json" class SuperTokensUserId(NonEmptyStr): @@ -52,8 +52,7 @@ class AccountSession(FrozenModel): Built on demand by :class:`MultiAccountSessionStore` from ``ImbueCloudCli.auth_list()`` (identity: ``user_id`` / ``email`` / - ``display_name``) and the local on-disk associations file - (``workspace_ids``). + ``display_name``) and the workspace-record replica (``workspace_ids``). """ user_id: SuperTokensUserId = Field(description="SuperTokens user ID") @@ -81,86 +80,24 @@ def derive_user_id_prefix(user_id: str) -> UserIdPrefix: class MultiAccountSessionStore(MutableModel): - """Joins plugin-owned auth identity with minds-local workspace associations. - - Disk layout: ``/workspace_associations.json`` mapping - ``user_id -> [agent_id, ...]``. No identity fields are stored. + """Joins plugin-owned auth identity with the workspace-record association view. Identity is sourced from ``ImbueCloudCli.auth_list()`` and cached in memory; sign-in / sign-out callers must invoke - :meth:`invalidate_identity_cache` so the cache stays consistent - with the plugin's view. + :meth:`invalidate_identity_cache` so the cache stays consistent with the + plugin's view. Associations come from (and are written through) the + :class:`WorkspaceRecordStore`; when none is configured every workspace + reads as private and association writes raise. """ data_dir: Path = Field(frozen=True, description="Root data directory (e.g. ~/.minds)") cli: ImbueCloudCli = Field(frozen=True, description="Plugin CLI used to source account identity") - _disk_lock: threading.Lock = PrivateAttr(default_factory=threading.Lock) + record_store: WorkspaceRecordStore | None = Field( + default=None, description="Association source of truth; None disables associations entirely" + ) _cache_lock: threading.Lock = PrivateAttr(default_factory=threading.Lock) _identity_cache: dict[str, ImbueCloudAuthAccount] | None = PrivateAttr(default=None) - @property - def _associations_path(self) -> Path: - return self.data_dir / _WORKSPACE_ASSOCIATIONS_FILENAME - - @property - def _legacy_sessions_path(self) -> Path: - return self.data_dir / _LEGACY_SESSIONS_FILENAME - - # -- Disk: workspace associations --------------------------------------- - - def _read_associations_unlocked(self) -> dict[str, list[str]]: - """Read ``user_id -> [agent_id, ...]`` from disk. - - Falls back to the legacy ``sessions.json`` (which used to store - full identity records) when ``workspace_associations.json`` - doesn't yet exist, extracting just the ``workspace_ids`` field - from each entry. Returns an empty dict on missing / corrupt - files so a brand-new install starts clean. - """ - path = self._associations_path - if path.exists(): - try: - raw = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError) as e: - logger.warning("Failed to load workspace associations: {}", e) - return {} - if not isinstance(raw, dict): - return {} - result: dict[str, list[str]] = {} - for user_id, value in raw.items(): - if not isinstance(value, list): - continue - result[user_id] = [str(v) for v in value if isinstance(v, str)] - return result - - legacy = self._legacy_sessions_path - if not legacy.exists(): - return {} - try: - raw = json.loads(legacy.read_text()) - except (OSError, json.JSONDecodeError) as e: - logger.warning("Failed to load legacy sessions file: {}", e) - return {} - if not isinstance(raw, dict): - return {} - result_legacy: dict[str, list[str]] = {} - for user_id, data in raw.items(): - if not isinstance(data, dict): - continue - workspace_ids = data.get("workspace_ids", []) - if isinstance(workspace_ids, list): - result_legacy[user_id] = [str(v) for v in workspace_ids if isinstance(v, str)] - return result_legacy - - def _write_associations_unlocked(self, associations: dict[str, list[str]]) -> None: - """Persist ``user_id -> [agent_id, ...]`` atomically.""" - self.data_dir.mkdir(parents=True, exist_ok=True) - path = self._associations_path - tmp_path = path.with_suffix(".tmp") - tmp_path.write_text(json.dumps(associations, indent=2)) - tmp_path.chmod(0o600) - tmp_path.rename(path) - # -- Identity cache (sourced from the plugin) --------------------------- def invalidate_identity_cache(self) -> None: @@ -184,79 +121,33 @@ def _identity_by_user_id(self, refresh: bool = False) -> dict[str, ImbueCloudAut accounts = self.cli.auth_list() except ImbueCloudCliError as exc: logger.warning("Failed to list imbue_cloud accounts: {}", exc) - # Don't poison the cache with the empty fallback: a - # transient subprocess failure would otherwise stick - # ``no accounts`` until the next sign-in / sign-out - # invalidates the cache. Return an empty mapping for - # this call only and let the next read retry. Also - # skip the orphan-association GC below -- nuking - # associations because a subprocess crashed would be a - # disastrous data-loss bug. + # Don't poison the cache with the empty fallback: a transient + # subprocess failure would otherwise stick ``no accounts`` + # until the next sign-in / sign-out invalidates the cache. return {} self._identity_cache = {account.user_id: account for account in accounts} - result = dict(self._identity_cache) - # GC orphan associations OUTSIDE the cache lock so we don't hold - # two locks (cache + disk) in a fixed order across this method's - # full extent. The cache is already committed by the time we - # land here; even if another thread reads it between unlock and - # GC, they see the new identity but possibly stale associations, - # which is harmless (any lookup just returns the same result the - # GC is about to converge to). - # - # Only GC on explicit refresh: the default-read path is hot (a - # FastAPI request handler iterating workspaces, the chrome SSE - # loop, etc.) and we don't want to take the disk lock + walk - # every association on every read. Refresh is the moment we know - # the identity is freshly-authoritative, which is exactly when - # GC is safe to run. - if refresh: - self._gc_orphan_associations(known_user_ids=frozenset(result.keys())) - return result - - def _gc_orphan_associations(self, *, known_user_ids: frozenset[str]) -> None: - """Remove user_id keys from ``workspace_associations.json`` whose user_id no longer exists. - - Called from the refresh branch of :meth:`_identity_by_user_id` - with the freshly-authoritative set of current user_ids. Any - association keyed under a user_id NOT in that set is presumed - orphaned (the SuperTokens user was deleted server-side, or the - local sessions were rotated to a fresh user_id for the same - email) and the orphan key is removed. - - Without this GC, workspace_associations.json grows monotonically - across signin/signout cycles and old user_ids become permanent - residents of the file. Worse, when minds creates a workspace - under an orphan user_id (because the form rendered between an - identity transition), that workspace stays associated with the - orphan forever -- ``get_account_for_workspace`` returns None - because the orphan isn't in current identity, and the UI shows - "no associated account" with no way to recover short of editing - the file by hand. - - No-op when the file is missing or contains only known user_ids. - """ - with self._disk_lock: - associations = self._read_associations_unlocked() - orphan_user_ids = [uid for uid in associations if uid not in known_user_ids] - if not orphan_user_ids: - return - for uid in orphan_user_ids: - logger.info( - "GCing orphan workspace_associations entry for user {} (workspaces={}); " - "user_id no longer present in current auth list", - uid[:8], - associations[uid], - ) - del associations[uid] - self._write_associations_unlocked(associations) + return dict(self._identity_cache) + + def _associations_view(self) -> dict[str, list[str]]: + if self.record_store is None: + return {} + return self.record_store.associations_view() + + def _require_account(self, user_id: str) -> ImbueCloudAuthAccount: + """Resolve a signed-in account by user_id, refreshing the cache once on a miss.""" + account = self._identity_by_user_id().get(user_id) + if account is None: + account = self._identity_by_user_id(refresh=True).get(user_id) + if account is None: + raise WorkspaceSyncError(f"No signed-in account matches user id {user_id[:8]}") + return account # -- Public read API ---------------------------------------------------- def list_accounts(self) -> list[AccountSession]: """Return every signed-in account, joined with any workspaces it owns.""" identity = self._identity_by_user_id() - with self._disk_lock: - associations = self._read_associations_unlocked() + associations = self._associations_view() return [_build_session(account, associations.get(user_id, [])) for user_id, account in identity.items()] def get_session(self, user_id: str) -> AccountSession | None: @@ -265,9 +156,7 @@ def get_session(self, user_id: str) -> AccountSession | None: account = identity.get(user_id) if account is None: return None - with self._disk_lock: - associations = self._read_associations_unlocked() - return _build_session(account, associations.get(user_id, [])) + return _build_session(account, self._associations_view().get(user_id, [])) def get_account_email(self, user_id: str) -> str | None: """Return the email for ``user_id``, or None if not signed in.""" @@ -291,19 +180,11 @@ def get_user_info(self, user_id: str) -> UserInfo | None: def get_account_for_workspace(self, agent_id: str) -> AccountSession | None: """Find the account that owns ``agent_id`` (or None if private). - When the on-disk association points to a user_id whose identity - isn't in the cached ``auth list`` snapshot, we refresh the cache - once and retry before giving up. This recovers from the case - where the identity cache was populated under a prior user_id - (e.g. before a signin/oauth that rotated to a new id) and never - learned about the rotation -- without the retry, the UI would - permanently render the workspace as "no associated account" - even though the association is correct on disk. The refresh - also runs the orphan-association GC, so this lookup-miss path - doubles as the maintenance trigger. + When the replica's owner isn't in the cached ``auth list`` snapshot, + the cache is refreshed once before giving up -- this recovers from an + identity cache populated before a signin rotated to a new user_id. """ - with self._disk_lock: - associations = self._read_associations_unlocked() + associations = self._associations_view() for user_id, workspace_ids in associations.items(): if agent_id in workspace_ids: identity = self._identity_by_user_id() @@ -321,59 +202,74 @@ def is_any_signed_in(self) -> bool: return bool(self._identity_by_user_id()) def has_signed_in_before(self) -> bool: - """Whether the user has ever signed in (associations file or plugin reports anything).""" - if self._associations_path.exists() or self._legacy_sessions_path.exists(): + """Whether the user has ever signed in (replica/legacy state exists or the plugin reports anything). + + The legacy files are matched by prefix so their retired + (``.pre-sync``-renamed) copies still count after the one-shot + record-store conversion. + """ + has_local_state = bool(self._associations_view()) or any( + any(self.data_dir.glob(f"{name}*")) for name in (_LEGACY_ASSOCIATIONS_FILENAME, _LEGACY_SESSIONS_FILENAME) + ) + if has_local_state: return True return self.is_any_signed_in() # -- Public write API (workspace associations) ------------------------- - def associate_workspace(self, user_id: str, agent_id: str) -> None: - """Bind ``agent_id`` to ``user_id`` on disk, dropping any prior owner. - - A workspace has at most one owning user_id at a time: re-binding - to a new owner first removes ``agent_id`` from every OTHER - user_id's workspace list, then appends it to ``user_id``'s. - Without this, a workspace that gets re-associated after a - SuperTokens user_id rotation (or after the operator manually - flipped the dropdown on the settings page) would end up listed - under both the old and new user_ids -- ``get_account_for_workspace`` - would return whichever owner happens to iterate first, which is - a no-error-but-wrong-account footgun. + def associate_workspace(self, user_id: str, agent_id: str, resolver: BackendResolverInterface) -> None: + """Bind ``agent_id`` to ``user_id`` by creating its workspace record (settings semantics). + + Pushes synchronously: raises ``WorkspaceSyncError`` when the connector + is unreachable, when the workspace isn't locally discovered, or when it + is owned by another account (disassociate first, then associate). + """ + if self.record_store is None: + raise WorkspaceSyncError("Workspace sync is not configured; cannot associate workspaces") + account = self._require_account(user_id) + self.record_store.associate_workspace_or_raise(user_id, account.email, agent_id, resolver) + logger.info("Associated workspace {} with user {}", agent_id, user_id[:8]) + + def associate_created_workspace( + self, + user_id: str, + agent_id: str, + host_id: str, + display_name: str, + color: str | None, + is_cloud_row: bool, + ) -> None: + """Create-path association: seed a minimal record now, queued for push. + + Runs right after ``mngr create`` returns the canonical ids -- before + discovery has seen the workspace -- so the record starts with just the + form metadata. The reconcile's metadata refresh enriches it (provider, + secrets) once discovery catches up. Never blocks or fails creation: + a push failure just leaves the row dirty for the reconcile. """ - with self._disk_lock: - associations = self._read_associations_unlocked() - # Strip ``agent_id`` from every other user_id first. - changed = False - for other_user_id, workspace_ids in list(associations.items()): - if other_user_id == user_id or agent_id not in workspace_ids: - continue - associations[other_user_id] = [wid for wid in workspace_ids if wid != agent_id] - changed = True - logger.info( - "Re-associating workspace {} away from user {} to {}", - agent_id, - other_user_id[:8], - user_id[:8], - ) - existing = associations.get(user_id, []) - if agent_id not in existing: - associations[user_id] = [*existing, agent_id] - changed = True - logger.info("Associated workspace {} with user {}", agent_id, user_id[:8]) - if changed: - self._write_associations_unlocked(associations) + if self.record_store is None: + logger.warning("Workspace sync is not configured; created workspace {} stays private", agent_id) + return + account = self._require_account(user_id) + seed = ReplicaRecord( + host_id=host_id, + agent_id=agent_id, + display_name=display_name or agent_id, + color=color, + provider_kind="", + hosting_device_id=None if is_cloud_row else self.record_store.device_id, + device_label=self.record_store.device_label, + ) + self.record_store.upsert_local_record(user_id, account.email, seed) + logger.info("Associated created workspace {} with user {}", agent_id, user_id[:8]) def disassociate_workspace(self, user_id: str, agent_id: str) -> None: - """Remove ``agent_id`` from ``user_id``'s workspace list.""" - with self._disk_lock: - associations = self._read_associations_unlocked() - existing = associations.get(user_id, []) - if agent_id not in existing: - return - associations[user_id] = [wid for wid in existing if wid != agent_id] - self._write_associations_unlocked(associations) - logger.info("Disassociated workspace {} from user {}", agent_id, user_id[:8]) + """Remove ``agent_id``'s record (the workspace becomes private; requires connectivity).""" + if self.record_store is None: + return + account = self._require_account(user_id) + self.record_store.disassociate_workspace_or_raise(user_id, account.email, agent_id) + logger.info("Disassociated workspace {} from user {}", agent_id, user_id[:8]) def _build_session(account: ImbueCloudAuthAccount, workspace_ids: list[str]) -> AccountSession: diff --git a/apps/minds/imbue/minds/desktop_client/session_store_test.py b/apps/minds/imbue/minds/desktop_client/session_store_test.py index cb1b780c73..04c3aa71dd 100644 --- a/apps/minds/imbue/minds/desktop_client/session_store_test.py +++ b/apps/minds/imbue/minds/desktop_client/session_store_test.py @@ -1,10 +1,21 @@ +import json from pathlib import Path +import pytest + +from imbue.minds.desktop_client.backend_resolver import MngrCliBackendResolver from imbue.minds.desktop_client.conftest import FakeImbueCloudCli from imbue.minds.desktop_client.conftest import make_fake_imbue_cloud_cli +from imbue.minds.desktop_client.conftest import make_resolver_with_data from imbue.minds.desktop_client.conftest import make_session_store_for_test from imbue.minds.desktop_client.session_store import MultiAccountSessionStore from imbue.minds.desktop_client.session_store import derive_user_id_prefix +from imbue.minds.errors import WorkspaceSyncError +from imbue.mngr.primitives import AgentId +from imbue.mngr.primitives import HostId + +_AGENT_A = str(AgentId.generate()) +_AGENT_B = str(AgentId.generate()) def _make_store_with_users( @@ -19,6 +30,19 @@ def _make_store_with_users( return store, cli +def _resolver_for_agents(*agent_ids: str) -> MngrCliBackendResolver: + """Build a resolver where each agent lives on its own host (distinct host_ids).""" + agents = [ + { + "id": agent_id, + "labels": {"is_primary": "true"}, + "host": {"id": str(HostId.generate()), "name": agent_id[:12]}, + } + for agent_id in agent_ids + ] + return make_resolver_with_data(agents_json=json.dumps({"agents": agents})) + + def test_add_and_load_session(tmp_path: Path) -> None: """A signed-in user is reachable via get_session(user_id).""" store, _cli = _make_store_with_users(tmp_path, [("user-aaa", "aaa@example.com", None)]) @@ -65,20 +89,24 @@ def test_remove_account_disappears_after_invalidate(tmp_path: Path) -> None: def test_associate_and_disassociate_workspace(tmp_path: Path) -> None: - """Workspace association and disassociation persist on disk.""" - store, _cli = _make_store_with_users(tmp_path, [("user-1", "a@b.com", None)]) + """Association creates a workspace record; disassociation removes it.""" + store, cli = _make_store_with_users(tmp_path, [("user-1", "a@b.com", None)]) + resolver = _resolver_for_agents(_AGENT_A, _AGENT_B) - store.associate_workspace("user-1", "agent-aaa") - store.associate_workspace("user-1", "agent-bbb") + store.associate_workspace("user-1", _AGENT_A, resolver) + store.associate_workspace("user-1", _AGENT_B, resolver) session = store.get_session("user-1") assert session is not None - assert session.workspace_ids == ["agent-aaa", "agent-bbb"] + assert sorted(session.workspace_ids) == sorted([_AGENT_A, _AGENT_B]) + # The records landed on the (fake) connector. + assert len(cli.sync_records_by_email["a@b.com"]) == 2 - store.disassociate_workspace("user-1", "agent-aaa") + store.disassociate_workspace("user-1", _AGENT_A) session = store.get_session("user-1") assert session is not None - assert session.workspace_ids == ["agent-bbb"] + assert session.workspace_ids == [_AGENT_B] + assert len(cli.sync_records_by_email["a@b.com"]) == 1 def test_get_account_for_workspace(tmp_path: Path) -> None: @@ -87,14 +115,15 @@ def test_get_account_for_workspace(tmp_path: Path) -> None: tmp_path, [("user-1", "one@example.com", None), ("user-2", "two@example.com", None)], ) - store.associate_workspace("user-1", "agent-aaa") - store.associate_workspace("user-2", "agent-bbb") + resolver = _resolver_for_agents(_AGENT_A, _AGENT_B) + store.associate_workspace("user-1", _AGENT_A, resolver) + store.associate_workspace("user-2", _AGENT_B, resolver) - account = store.get_account_for_workspace("agent-aaa") + account = store.get_account_for_workspace(_AGENT_A) assert account is not None assert account.email == "one@example.com" - account = store.get_account_for_workspace("agent-bbb") + account = store.get_account_for_workspace(_AGENT_B) assert account is not None assert account.email == "two@example.com" @@ -104,12 +133,66 @@ def test_get_account_for_workspace(tmp_path: Path) -> None: def test_duplicate_associate_is_idempotent(tmp_path: Path) -> None: """Associating the same workspace twice doesn't create duplicates.""" store, _cli = _make_store_with_users(tmp_path, [("user-1", "a@b.com", None)]) - store.associate_workspace("user-1", "agent-aaa") - store.associate_workspace("user-1", "agent-aaa") + resolver = _resolver_for_agents(_AGENT_A) + store.associate_workspace("user-1", _AGENT_A, resolver) + store.associate_workspace("user-1", _AGENT_A, resolver) + + session = store.get_session("user-1") + assert session is not None + assert session.workspace_ids == [_AGENT_A] + + +def test_associate_while_offline_raises(tmp_path: Path) -> None: + """Settings-page association requires connectivity and fails cleanly offline.""" + store, cli = _make_store_with_users(tmp_path, [("user-1", "a@b.com", None)]) + cli.is_sync_offline = True + resolver = _resolver_for_agents(_AGENT_A) + + with pytest.raises(WorkspaceSyncError): + store.associate_workspace("user-1", _AGENT_A, resolver) + assert store.get_account_for_workspace(_AGENT_A) is None + + +def test_associate_created_workspace_seeds_a_queued_record(tmp_path: Path) -> None: + """The create-path association seeds a record with form metadata (no resolver needed).""" + store, cli = _make_store_with_users(tmp_path, [("user-1", "a@b.com", None)]) + + store.associate_created_workspace( + user_id="user-1", + agent_id="agent-new", + host_id="host-new", + display_name="my new workspace", + color="#112233", + is_cloud_row=False, + ) session = store.get_session("user-1") assert session is not None - assert session.workspace_ids == ["agent-aaa"] + assert session.workspace_ids == ["agent-new"] + pushed = cli.sync_records_by_email["a@b.com"]["host-new"] + assert pushed["display_name"] == "my new workspace" + assert pushed["color"] == "#112233" + assert pushed["hosting_device_id"] == "device-test" + + +def test_associate_created_workspace_queues_offline(tmp_path: Path) -> None: + """A connector outage never fails creation: the record queues locally.""" + store, cli = _make_store_with_users(tmp_path, [("user-1", "a@b.com", None)]) + cli.is_sync_offline = True + + store.associate_created_workspace( + user_id="user-1", + agent_id="agent-new", + host_id="host-new", + display_name="ws", + color=None, + is_cloud_row=False, + ) + + session = store.get_session("user-1") + assert session is not None + assert session.workspace_ids == ["agent-new"] + assert "a@b.com" not in cli.sync_records_by_email def test_get_user_info(tmp_path: Path) -> None: @@ -126,15 +209,6 @@ def test_get_user_info(tmp_path: Path) -> None: assert str(info.user_id_prefix) == "abcd123456789abc" -def test_corrupt_associations_file_returns_empty(tmp_path: Path) -> None: - """A corrupt workspace_associations.json doesn't break list_accounts.""" - store, _cli = _make_store_with_users(tmp_path, [("user-1", "a@b.com", None)]) - (tmp_path / "workspace_associations.json").write_text("not valid json {{{") - accounts = store.list_accounts() - assert len(accounts) == 1 - assert accounts[0].workspace_ids == [] - - def test_is_any_signed_in(tmp_path: Path) -> None: """is_any_signed_in reflects whether the plugin reports any accounts.""" store, cli = _make_store_with_users(tmp_path, []) @@ -151,10 +225,11 @@ def test_derive_user_id_prefix() -> None: assert str(prefix) == "abcd123456789abc" -def test_disassociate_from_nonexistent_user_is_noop(tmp_path: Path) -> None: - """Disassociating from a user with no associations does nothing.""" +def test_disassociate_from_unknown_user_raises(tmp_path: Path) -> None: + """Disassociating for a user that isn't signed in raises (no account to resolve).""" store, _cli = _make_store_with_users(tmp_path, []) - store.disassociate_workspace("nonexistent-user", "agent-xyz") + with pytest.raises(WorkspaceSyncError): + store.disassociate_workspace("nonexistent-user", "agent-xyz") def test_disassociate_nonexistent_workspace_is_noop(tmp_path: Path) -> None: @@ -166,16 +241,12 @@ def test_disassociate_nonexistent_workspace_is_noop(tmp_path: Path) -> None: assert session.workspace_ids == [] -def test_associate_for_unsigned_user_writes_disk_but_not_listed(tmp_path: Path) -> None: - """Disk associations are independent of signed-in identity. - - Associating an agent_id with a user_id that isn't signed in persists - the association, but list_accounts (driven by the plugin's auth - list) won't include the user. This mirrors the post-signout state - where workspace_ids linger until the user signs back in. - """ +def test_associate_for_unsigned_user_raises(tmp_path: Path) -> None: + """Associating with a user_id that isn't signed in raises instead of writing state.""" store, _cli = _make_store_with_users(tmp_path, []) - store.associate_workspace("nonexistent-user", "agent-xyz") + resolver = _resolver_for_agents(_AGENT_A) + with pytest.raises(WorkspaceSyncError): + store.associate_workspace("nonexistent-user", _AGENT_A, resolver) assert store.list_accounts() == [] @@ -198,45 +269,48 @@ def test_get_user_info_nonexistent_returns_none(tmp_path: Path) -> None: def test_has_signed_in_before_with_associations(tmp_path: Path) -> None: - """has_signed_in_before returns True once the associations file exists.""" - store, _cli = _make_store_with_users(tmp_path, []) + """has_signed_in_before returns True once any workspace record exists.""" + store, cli = _make_store_with_users(tmp_path, []) assert not store.has_signed_in_before() - store.associate_workspace("user-1", "agent-x") + cli.add_account(user_id="user-1", email="a@b.com") + store.invalidate_identity_cache() + store.associate_created_workspace( + user_id="user-1", agent_id="agent-x", host_id="host-x", display_name="x", color=None, is_cloud_row=False + ) assert store.has_signed_in_before() def test_has_signed_in_before_when_plugin_reports_account(tmp_path: Path) -> None: - """has_signed_in_before is True even with no associations file when the plugin has a session.""" + """has_signed_in_before is True even with no records when the plugin has a session.""" store, _cli = _make_store_with_users(tmp_path, [("user-1", "a@b.com", None)]) assert store.has_signed_in_before() +def test_has_signed_in_before_with_legacy_files(tmp_path: Path) -> None: + """A pre-sync install's legacy files still count as having signed in.""" + store, _cli = _make_store_with_users(tmp_path, []) + (tmp_path / "workspace_associations.json").write_text("{}") + assert store.has_signed_in_before() + + +def test_has_signed_in_before_with_retired_legacy_files(tmp_path: Path) -> None: + """Legacy files renamed aside by the one-shot conversion still count as having signed in.""" + store, _cli = _make_store_with_users(tmp_path, []) + assert not store.has_signed_in_before() + (tmp_path / "sessions.json.pre-sync").write_text("{}") + assert store.has_signed_in_before() + + def test_persistence_across_store_instances(tmp_path: Path) -> None: - """Workspace associations written by one store instance are readable by another.""" + """Workspace records written by one store instance are readable by another.""" cli = make_fake_imbue_cloud_cli() cli.add_account(user_id="user-1", email="persist@test.com") store1 = make_session_store_for_test(tmp_path, cli=cli) - store1.associate_workspace("user-1", "agent-xyz") + resolver = _resolver_for_agents(_AGENT_A) + store1.associate_workspace("user-1", _AGENT_A, resolver) store2 = make_session_store_for_test(tmp_path, cli=cli) session = store2.get_session("user-1") assert session is not None assert session.email == "persist@test.com" - assert session.workspace_ids == ["agent-xyz"] - - -def test_legacy_sessions_file_migration_reads_workspace_ids(tmp_path: Path) -> None: - """The legacy ~/.minds/sessions.json shape is read for workspace_ids on first run.""" - legacy = tmp_path / "sessions.json" - legacy.write_text( - '{"user-legacy": {"user_id": "user-legacy", "email": "old@example.com",' - ' "display_name": null, "workspace_ids": ["agent-old"]}}' - ) - cli = make_fake_imbue_cloud_cli() - cli.add_account(user_id="user-legacy", email="old@example.com") - store = make_session_store_for_test(tmp_path, cli=cli) - - session = store.get_session("user-legacy") - assert session is not None - assert session.email == "old@example.com" - assert session.workspace_ids == ["agent-old"] + assert session.workspace_ids == [_AGENT_A] diff --git a/apps/minds/imbue/minds/desktop_client/settings_routes_test.py b/apps/minds/imbue/minds/desktop_client/settings_routes_test.py index 3a672f8b43..db533c29ab 100644 --- a/apps/minds/imbue/minds/desktop_client/settings_routes_test.py +++ b/apps/minds/imbue/minds/desktop_client/settings_routes_test.py @@ -171,7 +171,7 @@ def test_settings_sidebar_groups_nav_into_sections(tmp_path: Path) -> None: # The two group eyebrows and the compact nav labels. assert 'type-section text-tertiary px-2 mb-1">Permissions' in nav assert 'type-section text-tertiary px-2 mt-4 mb-1">Other' in nav - for label in ("Connectors", "Local files", "Workspaces", "Error reporting", "Backup password"): + for label in ("Connectors", "Local files", "Workspaces", "Error reporting", "Master password"): assert label in nav # The switchable entries are the nav buttons (the eyebrows are not buttons). assert nav.count("data-settings-nav=") == 5 diff --git a/apps/minds/imbue/minds/desktop_client/sharing_handler_test.py b/apps/minds/imbue/minds/desktop_client/sharing_handler_test.py index 91f6172616..e06c482638 100644 --- a/apps/minds/imbue/minds/desktop_client/sharing_handler_test.py +++ b/apps/minds/imbue/minds/desktop_client/sharing_handler_test.py @@ -12,6 +12,7 @@ from imbue.minds.desktop_client.sharing_handler import is_share_ready_from_edge_response from imbue.minds.primitives import ServiceName from imbue.mngr.primitives import AgentId +from imbue.mngr.primitives import HostId def test_is_share_ready_from_edge_response_true_for_access_login_redirect() -> None: @@ -119,7 +120,14 @@ def test_disable_sharing_is_idempotent_when_service_already_absent(tmp_path: Pat ) cli.add_account(user_id="u-1", email="owner@example.com") store = make_session_store_for_test(tmp_path / "sessions", cli=cli) - store.associate_workspace("u-1", str(agent_id)) + store.associate_created_workspace( + user_id="u-1", + agent_id=str(agent_id), + host_id=str(HostId.generate()), + display_name="", + color=None, + is_cloud_row=False, + ) disable_sharing(agent_id, ServiceName("web"), cli, store) diff --git a/apps/minds/imbue/minds/desktop_client/state.py b/apps/minds/imbue/minds/desktop_client/state.py index aec6687162..1697850509 100644 --- a/apps/minds/imbue/minds/desktop_client/state.py +++ b/apps/minds/imbue/minds/desktop_client/state.py @@ -39,6 +39,7 @@ from imbue.minds.desktop_client.request_events import RequestInbox from imbue.minds.desktop_client.request_handler import RequestEventHandler from imbue.minds.desktop_client.session_store import MultiAccountSessionStore +from imbue.minds.desktop_client.sync_scheduler import WorkspaceSyncScheduler from imbue.minds.desktop_client.system_interface_health import SystemInterfaceHealthTracker from imbue.minds.desktop_client.workspace_operations import InMemoryWorkspaceOperationRegistry from imbue.minds.desktop_client.workspace_operations import WorkspaceOperationRegistryInterface @@ -94,6 +95,9 @@ class DesktopClientState(MutableModel): session_store: MultiAccountSessionStore | None = Field( default=None, frozen=True, description="Multi-account session store" ) + sync_scheduler: WorkspaceSyncScheduler | None = Field( + default=None, frozen=True, description="Background workspace-record sync loop (kicked on auth changes)" + ) request_inbox: RequestInbox | None = Field( default=None, description="Immutable pending-request inbox (reassigned)" ) diff --git a/apps/minds/imbue/minds/desktop_client/static/chrome.js b/apps/minds/imbue/minds/desktop_client/static/chrome.js index 312e92089e..62a89f58ea 100644 --- a/apps/minds/imbue/minds/desktop_client/static/chrome.js +++ b/apps/minds/imbue/minds/desktop_client/static/chrome.js @@ -515,6 +515,8 @@ withOpenNew: false, }); row.addEventListener('click', function (e) { + // Rows for workspaces on another device are informational only. + if (w.is_remote) return; if (e.target.closest('[data-open-settings]')) { navigateContent('/workspace/' + w.id + '/settings'); closeSidebar(); diff --git a/apps/minds/imbue/minds/desktop_client/static/sidebar_workspace_row.js b/apps/minds/imbue/minds/desktop_client/static/sidebar_workspace_row.js index efe193c862..2217420830 100644 --- a/apps/minds/imbue/minds/desktop_client/static/sidebar_workspace_row.js +++ b/apps/minds/imbue/minds/desktop_client/static/sidebar_workspace_row.js @@ -69,10 +69,13 @@ // No outer margin: row-to-row spacing is the parent container's flex // ``gap``, keeping this element positioning-free and composable. + var isRemote = !!workspace.is_remote; var row = document.createElement('div'); row.className = - 'sidebar-item group flex items-center gap-2 h-8 px-2 rounded-md cursor-pointer type-body text-primary' - + (isCurrent ? ' is-current bg-fill-active' : ' hover:bg-fill-hover'); + 'sidebar-item group flex items-center gap-2 h-8 px-2 rounded-md type-body ' + + (isRemote + ? 'is-remote text-secondary opacity-60 cursor-default' + : ('cursor-pointer text-primary' + (isCurrent ? ' is-current bg-fill-active' : ' hover:bg-fill-hover'))); row.setAttribute('data-agent-id', workspace.id); var dot = document.createElement('span'); @@ -84,6 +87,15 @@ label.textContent = workspace.name || workspace.id; row.appendChild(label); + // A workspace hosted on another device (known via its synced record): + // greyed, non-navigable, with a location badge instead of action icons. + if (isRemote) { + var locationBadge = document.createElement('span'); + locationBadge.className = 'inline-flex items-center px-1.5 py-0.5 rounded-md type-label bg-fill-subtle text-tertiary shrink-0'; + locationBadge.textContent = 'on ' + (workspace.location || 'another device'); + row.appendChild(locationBadge); + } + // Backup-service problem detected for this workspace (outdated code, // drifted credentials, service down, unconfigured, or unverifiable): // one warning badge style for all causes; the tooltip carries the @@ -115,8 +127,10 @@ btn.classList.add('inline-flex'); row.appendChild(btn); } - if (withOpenNew) addActionIcon(buildOpenNewBtn(workspace.id)); - addActionIcon(buildSettingsBtn(workspace.id)); + if (!isRemote) { + if (withOpenNew) addActionIcon(buildOpenNewBtn(workspace.id)); + addActionIcon(buildSettingsBtn(workspace.id)); + } // Accent: prefer the server-attached value; otherwise resolve it // asynchronously via the shared workspace_accent helper (if loaded). diff --git a/apps/minds/imbue/minds/desktop_client/static/workspace_backups.js b/apps/minds/imbue/minds/desktop_client/static/workspace_backups.js index 8fa78727b3..25c7320a0f 100644 --- a/apps/minds/imbue/minds/desktop_client/static/workspace_backups.js +++ b/apps/minds/imbue/minds/desktop_client/static/workspace_backups.js @@ -36,9 +36,6 @@ var configureToggleBtn = document.getElementById('backup-configure-toggle-btn'); var configureForm = document.getElementById('backup-configure-form'); var providerSelect = document.getElementById('backup-provider-select'); - var masterPasswordRow = document.getElementById('backup-master-password-row'); - var masterPasswordInput = document.getElementById('backup-master-password-input'); - var savePasswordInput = document.getElementById('backup-save-password-input'); var apiKeyRow = document.getElementById('backup-api-key-row'); var apiKeyEnvInput = document.getElementById('backup-api-key-env-input'); var configureSubmitBtn = document.getElementById('backup-configure-submit-btn'); @@ -296,10 +293,6 @@ function syncConfigureFormVisibility() { var provider = providerSelect.value; apiKeyRow.classList.toggle('hidden', provider !== 'API_KEY'); - // The master-password row only exists when a non-empty master password is - // set; it is only *needed* when a repository will be initialized, which - // "None" (disable) never does. - if (masterPasswordRow) masterPasswordRow.classList.toggle('hidden', provider === 'NONE'); } configureToggleBtn.addEventListener('click', function () { configureForm.classList.toggle('hidden'); @@ -307,9 +300,9 @@ }); providerSelect.addEventListener('change', syncConfigureFormVisibility); - // The master password (blank = use the saved copy, else the empty password) - // is validated server-side against the stored hash; save_password persists - // the just-validated typed value as the plaintext convenience copy. + // No password is involved: repositories are keyed by each workspace's own + // random password, and the master password's only role is wrapping the + // account's sync key (see the app-level Settings page). configureSubmitBtn.addEventListener('click', function () { if (providerSelect.value === 'NONE') { startOperation('/api/v1/workspaces/' + encodeURIComponent(agentId) + '/backup-service/disable', {}, false); @@ -318,8 +311,6 @@ var body = { backup_provider: providerSelect.value, api_key_env: apiKeyEnvInput ? apiKeyEnvInput.value : '', - master_password: masterPasswordInput ? masterPasswordInput.value : '', - save_password: savePasswordInput ? savePasswordInput.checked : false, }; startOperation('/api/v1/workspaces/' + encodeURIComponent(agentId) + '/backup-service/configure', body, false); }); diff --git a/apps/minds/imbue/minds/desktop_client/supertokens_routes.py b/apps/minds/imbue/minds/desktop_client/supertokens_routes.py index 885619c2c2..92c0c3e825 100644 --- a/apps/minds/imbue/minds/desktop_client/supertokens_routes.py +++ b/apps/minds/imbue/minds/desktop_client/supertokens_routes.py @@ -225,6 +225,7 @@ def _store_session_from_auth_result( """ assert result.user is not None, "AuthResult missing user" session_store.invalidate_identity_cache() + _kick_sync_scheduler() minds_config: MindsConfig | None = get_state().minds_config if minds_config is not None and minds_config.get_default_account_id() is None: minds_config.set_default_account_id(result.user.user_id) @@ -395,6 +396,7 @@ def signout_user_via_plugin(user_id: str) -> None: else: logger.warning("No mirrored account for user {}; skipping plugin signout", user_id[:8]) session_store.invalidate_identity_cache() + _kick_sync_scheduler() if signed_out_email and unset_imbue_cloud_provider_for_account(signed_out_email): _bounce_forward_observe() @@ -572,6 +574,7 @@ def _run_oauth_subprocess( return session_store.invalidate_identity_cache() + _kick_sync_scheduler() if minds_config is not None and minds_config.get_default_account_id() is None: minds_config.set_default_account_id(str(result.user_id)) @@ -744,6 +747,13 @@ def _handle_settings_page() -> Response: ) +def _kick_sync_scheduler() -> None: + """Request an immediate workspace-record sync pass after an auth change.""" + scheduler = get_state().sync_scheduler + if scheduler is not None: + scheduler.kick() + + def create_supertokens_blueprint() -> Blueprint: """Create a Flask blueprint with the auth routes (mounted under /auth).""" blueprint = Blueprint("supertokens", __name__, url_prefix="/auth") diff --git a/apps/minds/imbue/minds/desktop_client/sync_scheduler.py b/apps/minds/imbue/minds/desktop_client/sync_scheduler.py new file mode 100644 index 0000000000..1b603208ec --- /dev/null +++ b/apps/minds/imbue/minds/desktop_client/sync_scheduler.py @@ -0,0 +1,87 @@ +"""Background scheduler for workspace-record sync. + +Runs the reconcile (pull + legacy migration + metadata refresh + dirty pushes ++ definitively-absent tombstoning) event-driven: once after the session's +first complete discovery snapshot, then on a slow periodic tick, and +immediately whenever :meth:`WorkspaceSyncScheduler.kick` is called (sign-in / +sign-out, password changes). The legacy password-file conversion runs at the +start of every pass (a no-op once converted). + +One daemon thread; every pass is wrapped so an expected failure (a connector +outage, or any sync / crypto / file error) can never kill the loop. +""" + +import threading + +from loguru import logger +from pydantic import Field +from pydantic import PrivateAttr + +from imbue.concurrency_group.concurrency_group import ConcurrencyGroup +from imbue.imbue_common.mutable_model import MutableModel +from imbue.imbue_common.secret_wrapping import SecretWrappingError +from imbue.minds.desktop_client.backend_resolver import BackendResolverInterface +from imbue.minds.desktop_client.dek_store import convert_legacy_password_files +from imbue.minds.desktop_client.imbue_cloud_cli import ImbueCloudCliError +from imbue.minds.desktop_client.session_store import MultiAccountSessionStore +from imbue.minds.desktop_client.workspace_record_store import WorkspaceRecordStore +from imbue.minds.errors import SyncCryptoError +from imbue.minds.errors import WorkspaceSyncError + +# How often the loop re-reconciles when nothing kicks it. Records change +# rarely; this bounds how stale another device's view can get. +_SYNC_TICK_SECONDS = 60.0 +# How often the loop re-checks for the first complete discovery snapshot +# before the first reconcile can run. +_DISCOVERY_POLL_SECONDS = 2.0 + + +class WorkspaceSyncScheduler(MutableModel): + """Owns the background reconcile loop for workspace records.""" + + record_store: WorkspaceRecordStore = Field(frozen=True, description="The sync engine the loop drives") + session_store: MultiAccountSessionStore = Field(frozen=True, description="Source of the signed-in account list") + resolver: BackendResolverInterface = Field(frozen=True, description="Discovery view records are built from") + _kick_event: threading.Event = PrivateAttr(default_factory=threading.Event) + _stop_event: threading.Event = PrivateAttr(default_factory=threading.Event) + + def kick(self) -> None: + """Request an immediate sync pass (sign-in/out, password change, association).""" + self._kick_event.set() + + def stop(self) -> None: + self._stop_event.set() + self._kick_event.set() + + def run_one_pass(self) -> None: + """One full sync pass: legacy conversion + reconcile for every signed-in account.""" + accounts = {str(account.user_id): str(account.email) for account in self.session_store.list_accounts()} + convert_legacy_password_files(self.record_store.paths, list(accounts.keys())) + self.record_store.reconcile(accounts, self.resolver) + + def _loop(self) -> None: + # The first pass waits for a complete discovery snapshot so records + # can be built with real metadata (names, providers, host ids). + while not self._stop_event.is_set() and not self.resolver.has_completed_initial_discovery(): + self._stop_event.wait(_DISCOVERY_POLL_SECONDS) + while not self._stop_event.is_set(): + # Clear BEFORE the pass: a kick arriving mid-pass then stays set, + # so the wait below returns immediately and the request is served + # by the next pass instead of being lost. + self._kick_event.clear() + try: + self.run_one_pass() + except (ImbueCloudCliError, WorkspaceSyncError, SyncCryptoError, SecretWrappingError, OSError) as e: + # The loop is the only writer-side repair mechanism; log loudly + # but never die on a single bad pass (e.g. a connector outage). + logger.opt(exception=e).error("Workspace-record sync pass failed") + self._kick_event.wait(_SYNC_TICK_SECONDS) + + def start(self, concurrency_group: ConcurrencyGroup) -> None: + """Start the daemon loop on the app's root concurrency group.""" + concurrency_group.start_new_thread( + target=self._loop, + name="workspace-record-sync", + daemon=True, + is_checked=False, + ) diff --git a/apps/minds/imbue/minds/desktop_client/templates.py b/apps/minds/imbue/minds/desktop_client/templates.py index 8f6ce14355..87f0c4dcff 100644 --- a/apps/minds/imbue/minds/desktop_client/templates.py +++ b/apps/minds/imbue/minds/desktop_client/templates.py @@ -25,7 +25,9 @@ from jinja2 import Environment from jinja2 import select_autoescape from jinjax import Catalog +from pydantic import Field +from imbue.imbue_common.frozen_model import FrozenModel from imbue.imbue_common.pure import pure from imbue.minds.desktop_client.agent_creator import AgentCreationInfo from imbue.minds.desktop_client.state import get_state @@ -212,6 +214,16 @@ def _build_catalog() -> Catalog: CATALOG: Final[Catalog] = _build_catalog() +class RemoteWorkspaceTile(FrozenModel): + """A workspace known only from another device's synced record, for the landing list.""" + + agent_id: str = Field(description="The workspace agent id (drives backup status/download)") + name: str = Field(description="Display name from the record") + accent: str = Field(description="Accent color hex") + location: str = Field(description="Where it lives (the other device's label, or a provider name)") + host_id: str = Field(description="The record's host id (drives remove-from-list)") + + # -- Page renderers -- @@ -226,6 +238,8 @@ def render_landing_page( shutdown_capable_agent_ids: Sequence[AgentId] | None = None, mind_liveness_by_agent_id: dict[str, str] | None = None, agent_providers: dict[str, str] | None = None, + remote_workspaces: Sequence[RemoteWorkspaceTile] | None = None, + locked_account_emails: Sequence[str] | None = None, ) -> str: """Render the landing page listing accessible workspaces. @@ -263,6 +277,8 @@ def render_landing_page( shutdown_capable_agent_id_strings = [str(aid) for aid in (shutdown_capable_agent_ids or ())] return CATALOG.render( "pages.Landing", + remote_workspaces=list(remote_workspaces or []), + locked_account_emails=list(locked_account_emails or []), agent_ids=accessible_agent_ids, agent_accents=effective_accents, mngr_forward_origin=mngr_forward_origin, @@ -419,8 +435,6 @@ def render_create_form( docker_runtime: DockerRuntime | None = None, backup_provider: BackupProvider | None = None, backup_api_key_env: str = "", - has_saved_backup_password: bool = False, - is_master_password_set: bool = False, accounts: Sequence[object] | None = None, default_account_id: str = "", anthropic_api_key: str = "", @@ -455,10 +469,6 @@ def render_create_form( ``start_advanced`` opens the advanced view on first paint -- used when re-rendering a submit error, whose fields live there. - ``is_master_password_set`` renders the master-password input at all (a - still-empty master password never needs typing); ``has_saved_backup_password`` - adds the "leave blank to use your saved password" helper under it. - ``host_name`` is an optional explicit workspace name, exposed as a "Name" field in the advanced view. When empty the name is chosen automatically server-side (the next free ``workspace-N`` via ``resolve_create_host_name``); a @@ -514,8 +524,6 @@ def render_create_form( backup_providers=list(BackupProvider), selected_backup_provider=effective_backup_provider.value, backup_api_key_env=backup_api_key_env, - has_saved_backup_password=has_saved_backup_password, - is_master_password_set=is_master_password_set, accounts=accounts or [], default_account_id=default_account_id, anthropic_api_key=anthropic_api_key, @@ -1689,8 +1697,6 @@ def render_workspace_settings( is_leased_imbue_cloud: bool = False, current_color: str = DEFAULT_WORKSPACE_COLOR, is_stale: bool = False, - has_saved_backup_password: bool = False, - is_master_password_set: bool = False, has_account: bool = False, ) -> str: """Render the workspace settings page. @@ -1721,8 +1727,6 @@ def render_workspace_settings( is_leased_imbue_cloud=is_leased_imbue_cloud, current_color=current_color, is_stale=is_stale, - has_saved_backup_password=has_saved_backup_password, - is_master_password_set=is_master_password_set, has_account=has_account, palette=WORKSPACE_PALETTE, ) @@ -1775,7 +1779,7 @@ def render_settings_page( file_sharing_grants: Sequence[object] | None = None, workspace_delegation_grants: Sequence[object] | None = None, permissions_unavailable: bool = False, - has_saved_backup_password: bool = False, + is_master_password_set: bool = False, ) -> str: """Render the app-level settings page (reachable from the sidebar's "Settings" entry). @@ -1784,9 +1788,9 @@ def render_settings_page( ``report_unexpected_errors`` / ``include_error_logs`` seed the per-machine error-reporting toggles hosted on this page (the same settings the - first-launch consent screen records); ``has_saved_backup_password`` feeds - the backup master-password section's helper text. All are global to the - machine, not account-scoped. + first-launch consent screen records); ``is_master_password_set`` feeds the + master-password section's helper copy (whether any signed-in account + already has a non-empty sync master password). ``services_overview`` is a sequence of :class:`~imbue.minds.desktop_client.latchkey.permission_overview.ServicePermissionOverview` @@ -1811,5 +1815,5 @@ def render_settings_page( file_sharing_grants=list(file_sharing_grants or []), workspace_delegation_grants=list(workspace_delegation_grants or []), permissions_unavailable=permissions_unavailable, - has_saved_backup_password=has_saved_backup_password, + is_master_password_set=is_master_password_set, ) diff --git a/apps/minds/imbue/minds/desktop_client/templates/pages/Create.jinja b/apps/minds/imbue/minds/desktop_client/templates/pages/Create.jinja index 4cfc933816..8a16e3ec1d 100644 --- a/apps/minds/imbue/minds/desktop_client/templates/pages/Create.jinja +++ b/apps/minds/imbue/minds/desktop_client/templates/pages/Create.jinja @@ -1,4 +1,4 @@ -{#def git_url, branch, host_name, launch_modes, selected_launch_mode, ai_providers, selected_ai_provider, docker_runtimes, selected_docker_runtime, backup_providers, selected_backup_provider, backup_api_key_env, has_saved_backup_password, is_master_password_set, accounts, default_account_id, anthropic_api_key, error_message, region_options_by_launch_mode, region_selected_by_launch_mode, selected_preset="remote", start_advanced=False, color="#0b292b" #} +{#def git_url, branch, host_name, launch_modes, selected_launch_mode, ai_providers, selected_ai_provider, docker_runtimes, selected_docker_runtime, backup_providers, selected_backup_provider, backup_api_key_env, accounts, default_account_id, anthropic_api_key, error_message, region_options_by_launch_mode, region_selected_by_launch_mode, selected_preset="remote", start_advanced=False, color="#0b292b" #} {# Workspace creation form. @@ -56,8 +56,6 @@ var aiProviderError = document.getElementById('ai-provider-account-error'); var backupSelect = document.getElementById('backup_provider'); var backupApiKeyRow = document.getElementById('backup-api-key-row'); - var backupMasterPasswordRow = document.getElementById('backup-master-password-row'); - var backupMasterPasswordInput = document.getElementById('backup_master_password'); var backupProviderError = document.getElementById('backup-provider-account-error'); var submitBtn = document.getElementById('create-submit'); var regionRow = document.getElementById('region-row'); @@ -73,7 +71,6 @@ var hostNameError = document.getElementById('host-name-error'); var colorInput = document.getElementById('create-color-input'); var backupApiKeyEnvInput = document.getElementById('backup_api_key_env'); - var backupSavePasswordInput = document.getElementById('backup_save_password'); var createError = document.getElementById('create-error'); // The field highlighted by the last create error, cleared before the next try. var lastErrorField = null; @@ -341,8 +338,6 @@ anthropic_api_key: fieldValue(apiKeyInput).trim(), backup_provider: backupSelect.value, backup_api_key_env: fieldValue(backupApiKeyEnvInput), - backup_master_password: fieldValue(backupMasterPasswordInput), - backup_save_password: backupSavePasswordInput ? backupSavePasswordInput.checked : false, region: fieldValue(regionSelect), // The container runtime only applies to the local Docker provider; the // select is disabled for every other compute provider, so omit it then @@ -483,16 +478,9 @@ apiKeyInput.required = isApiKey; var backupValue = backupSelect.value; - var isBackupConfigured = backupValue === 'IMBUE_CLOUD' || backupValue === 'API_KEY'; var isBackupApiKey = backupValue === 'API_KEY'; backupApiKeyRow.classList.toggle('hidden', !isBackupApiKey); - // The master password field shows for any real backup provider. It is - // never required: blank means "use the saved password, else the empty - // password" -- the server validates whichever value results against the - // stored hash. - if (backupMasterPasswordRow) backupMasterPasswordRow.classList.toggle('hidden', !isBackupConfigured); - var launchInvalid = !hasAccount && launchSelect.value === 'IMBUE_CLOUD'; var aiInvalid = !hasAccount && aiProviderSelect.value === 'IMBUE_CLOUD'; var backupInvalid = !hasAccount && backupValue === 'IMBUE_CLOUD'; @@ -698,23 +686,6 @@ :value="backup_api_key_env if backup_api_key_env else _default_restic_env" /> - {% if is_master_password_set %} - {# Only rendered when a non-empty master password exists (it is needed - to initialize the repository); the JS shows it only while a real - backup provider is selected. #} - - {% endif %}