From 5b7e53ac65215ea49de8bbfd4af10c2f32c6b925 Mon Sep 17 00:00:00 2001 From: Danyil Date: Sun, 28 Jun 2026 00:52:03 +0300 Subject: [PATCH 001/108] feat(sync): hybrid snapshot + per-device delta cloud sync over GitHub Gist Adds an opt-in cloud sync that keeps tab groups consistent across devices via a private GitHub Gist, using a compacted base snapshot plus per-device delta logs (disjoint keys, never clobbered): - pure engine: delta-log, delta-capture, replay/merge with conflict rules, plan-sync, lazy compaction, url-sync (incl. about: stub), container identity mapping, group-relative indexing, tab-sleep, applied-navigation echo guard; - transport: multi-file Gist API with ETag conditional GET (304 fast path), XHR upload progress, and an advisory server-clock lock to serialize cycles; - safety: per-device watermark dedup, deferred self-truncation (a clobbered snapshot never loses data), user-priority lock so user actions beat sync; - wiring: background/constants/utils + sync settings UI (options). --- addon/src/js/constants.js | 26 +- addon/src/js/sync/cloud/cloud.js | 154 +- addon/src/js/sync/cloud/githubgist.js | 605 ++++- addon/src/js/sync/cloud/provider.js | 150 ++ addon/src/js/sync/delta/applied-nav-echo.js | 59 + addon/src/js/sync/delta/compaction.js | 259 ++ addon/src/js/sync/delta/container-map.js | 279 ++ addon/src/js/sync/delta/delta-capture.js | 743 ++++++ addon/src/js/sync/delta/delta-log.js | 346 +++ addon/src/js/sync/delta/delta-sync.js | 2346 +++++++++++++++++ addon/src/js/sync/delta/device-id.js | 68 + .../src/js/sync/delta/group-relative-index.js | 42 + addon/src/js/sync/delta/layout.js | 64 + addon/src/js/sync/delta/lock.js | 101 + addon/src/js/sync/delta/option-keys.js | 75 + addon/src/js/sync/delta/plan-sync.js | 682 +++++ addon/src/js/sync/delta/replay.js | 692 +++++ addon/src/js/sync/delta/seed.js | 70 + addon/src/js/sync/delta/tab-sleep.js | 75 + addon/src/js/sync/delta/url-sync.js | 199 ++ addon/src/js/sync/delta/user-priority-lock.js | 329 +++ addon/src/js/utils.js | 4 + addon/src/options/Options.vue | 1 + addon/src/options/github-gist.vue | 98 + 24 files changed, 7399 insertions(+), 68 deletions(-) create mode 100644 addon/src/js/sync/cloud/provider.js create mode 100644 addon/src/js/sync/delta/applied-nav-echo.js create mode 100644 addon/src/js/sync/delta/compaction.js create mode 100644 addon/src/js/sync/delta/container-map.js create mode 100644 addon/src/js/sync/delta/delta-capture.js create mode 100644 addon/src/js/sync/delta/delta-log.js create mode 100644 addon/src/js/sync/delta/delta-sync.js create mode 100644 addon/src/js/sync/delta/device-id.js create mode 100644 addon/src/js/sync/delta/group-relative-index.js create mode 100644 addon/src/js/sync/delta/layout.js create mode 100644 addon/src/js/sync/delta/lock.js create mode 100644 addon/src/js/sync/delta/option-keys.js create mode 100644 addon/src/js/sync/delta/plan-sync.js create mode 100644 addon/src/js/sync/delta/replay.js create mode 100644 addon/src/js/sync/delta/seed.js create mode 100644 addon/src/js/sync/delta/tab-sleep.js create mode 100644 addon/src/js/sync/delta/url-sync.js create mode 100644 addon/src/js/sync/delta/user-priority-lock.js diff --git a/addon/src/js/constants.js b/addon/src/js/constants.js index ccbbed1b..5cb24588 100644 --- a/addon/src/js/constants.js +++ b/addon/src/js/constants.js @@ -491,6 +491,7 @@ export const DEFAULT_OPTIONS = Object.freeze({ 'remove', 'update-thumbnail', 'set-group-icon', + 'pin-in-group', 'move-tab-to-group', ], contextMenuGroup: [ @@ -515,11 +516,30 @@ export const DEFAULT_OPTIONS = Object.freeze({ autoBackupIncludeTabFavIcons: true, syncEnable: true, + syncProvider: 'github-gist', // local (per-device) option, see PROVIDER_* in sync/cloud/provider.js syncOptionsLocation: IS_AVAILABLE_SYNC_STORAGE ? SYNC_STORAGE_FSYNC : SYNC_STORAGE_LOCAL, syncLastUpdate: "1970-01-01T00:00:00Z", - syncIntervalKey: INTERVAL_KEY.days, // hours, days - syncIntervalValue: 1, + syncIntervalKey: INTERVAL_KEY.minutes, // minutes, hours, days + syncIntervalValue: 5, syncTabFavIcons: false, + // Safety net: take a local STG backup before delta-sync applies any browser- + // mutating change, so a bad sync can be fully rolled back. Default ON (safety + // first). Name starts with `sync` ⇒ LOCAL-ONLY (never roams between devices), + // see sync/delta/option-keys.js LOCAL_ONLY_OPTION_KEY_PREFIXES. + syncBackupBeforeApply: true, + // "Sleep synced GROUP tabs by default" — when true, non-pinned tabs created by a + // sync apply are created asleep/discarded: they show title+favicon and only load + // when the user activates them. Default ON. All `sync*` keys are LOCAL-ONLY (never + // roam between devices), see sync/delta/option-keys.js LOCAL_ONLY_OPTION_KEY_PREFIXES. + syncSleepNewTabs: true, + // PINNED tabs are a separate axis: they LOAD by default, because Firefox forbids + // creating a discarded pinned tab. When this is true STG sleeps them anyway via + // create-then-discard. Default OFF (pinned tabs load). + syncSleepPinnedTabs: false, + // When true, a synced tab that was active/loaded (not discarded) on the SOURCE + // machine is activated (loaded) here, overriding sleep-by-default for those tabs. + // Reads the additive `loaded` field on the tab record (absent ⇒ asleep). Default OFF. + syncActivatePreviouslyActiveTabs: false, colorScheme: 'auto', // auto, light, dark @@ -549,7 +569,7 @@ export const ALL_OPTION_KEYS = Object.freeze(DEFAULT_OPTION_KEYS.filter(key => ! export const ON_UPDATED_TAB_PROPERTIES = Object.freeze([ // browser.tabs not defined into web page scripts browser.tabs?.UpdatePropertyName.TITLE, // for cache browser.tabs?.UpdatePropertyName.STATUS, // for check update url and thumbnail - // browser.tabs?.UpdatePropertyName.URL, // for check update url and thumbnail + browser.tabs?.UpdatePropertyName.URL, // sync: a grouped/pinned tab navigating to a new url must propagate even without a title change (getRealTabStateChanged then exposes changeInfo.url) browser.tabs?.UpdatePropertyName.FAVICONURL, // for session browser.tabs?.UpdatePropertyName.HIDDEN, browser.tabs?.UpdatePropertyName.PINNED, diff --git a/addon/src/js/sync/cloud/cloud.js b/addon/src/js/sync/cloud/cloud.js index 258a0e5e..b908f61b 100644 --- a/addon/src/js/sync/cloud/cloud.js +++ b/addon/src/js/sync/cloud/cloud.js @@ -9,7 +9,7 @@ import * as Utils from '/js/utils.js'; import Lang from '/js/lang.js'; import JSON from '/js/json.js'; import Logger, {nativeErrorToObject, objectToNativeError} from '/js/logger.js'; -import GithubGist from './githubgist.js'; +import {createCloudProvider} from './provider.js'; import * as CloudBroadcast from '/js/broadcast.js?channel=cloud'; import * as SyncStorage from '../sync-storage.js'; import * as Storage from '/js/storage.js'; @@ -32,6 +32,14 @@ const canDoSynchronization = params.has('can-do-synchronization'); export const TRUST_LOCAL = 'trust-local'; export const TRUST_CLOUD = 'trust-cloud'; +// Entry-point switch for the live sync path (Phase P3b). When true, the manual Sync +// button and the auto/retry alarm route through the delta pipeline +// (`delta-sync.js` deltaSynchronization) instead of the legacy URL-based +// `synchronization()` below. The legacy code is retained-but-bypassed: flip this to +// false to revert to the old flow with no other change. There is intentionally no +// user-facing toggle. +export const USE_DELTA_SYNC = true; + export const ALARM_NAME = 'cloud'; export const ALARM_NAME_RETRY = 'cloud-retry'; export const ERROR_NOTIFICATION_ID = 'cloud-error'; @@ -42,6 +50,9 @@ export const TRIGGER_RETRY = 'cloud-trigger-retry'; export const NETWORK_RETRY_DELAY_MINUTES = 3; const MAX_NETWORK_RETRY_ATTEMPTS = 3; +// Cap the rate-limit backoff so a far-future / bogus reset timestamp can't park the retry +// alarm for hours. A GitHub primary-limit window is at most ~60 min; we wake a bit past it. +const MAX_RATE_LIMIT_BACKOFF_MINUTES = 65; let inProgress = false; @@ -71,7 +82,9 @@ export class CloudError extends Error { } } -function send(action, data = {}) { +// Exported so the delta transport (delta-sync.js) reuses the same broadcast channel +// and the UI/background progress wiring works identically for both sync paths. +export function send(action, data = {}) { CloudBroadcast.send({action, ...data}); } @@ -141,7 +154,7 @@ async function sync(trust = null, revision = null, progressFunc = null) { log.throwError('unknown source of trust argument'); } - const {syncOptionsLocation} = await Storage.get('syncOptionsLocation'); + const {syncOptionsLocation, syncProvider} = await Storage.get(['syncOptionsLocation', 'syncProvider']); if (syncOptionsLocation === Constants.SYNC_STORAGE_FSYNC) { if (!SyncStorage.IS_AVAILABLE) { @@ -162,14 +175,11 @@ async function sync(trust = null, revision = null, progressFunc = null) { let cloudInstance; try { - cloudInstance = new GithubGist( - syncOptions.githubGistToken, - syncOptions.githubGistFileName - ); + cloudInstance = createCloudProvider(syncProvider, syncOptions); } catch (error) { const cloudError = new CloudError(error.message, {cause: error}); storage.lastError = String(cloudError); - log.throwError('create GithubGist instance', cloudError); + log.throwError('create cloud provider instance', cloudError); } const Cloud = cloudInstance; @@ -197,7 +207,7 @@ async function sync(trust = null, revision = null, progressFunc = null) { } else { const cloudError = new CloudError(error.message, {cause: error}); storage.lastError = String(cloudError); - log.throwError('get GithubGist content', cloudError); + log.throwError('get cloud content', cloudError); } } @@ -259,7 +269,7 @@ async function sync(trust = null, revision = null, progressFunc = null) { } catch (error) { const cloudError = new CloudError(error.message, {cause: error}); storage.lastError = String(cloudError); - log.throwError('set GithubGist content', cloudError); + log.throwError('set cloud content', cloudError); } syncResult.changes.local = true; // sync date must be equal in cloud and local @@ -267,33 +277,26 @@ async function sync(trust = null, revision = null, progressFunc = null) { progressFunc?.(85); - // remove unnecessary groups + // C5: DEFER the destructive local browser mutations (group unload + tab removal) until + // AFTER the local DB commit (saveOptions/Groups.save) below succeeds. Previously these ran + // first; if the later commit threw, the cloud push had already landed AND windows/tabs were + // already torn down, but the local options/groups DB still held the OLD state — a partial, + // non-rolled-back local mutation. Here we only do the non-destructive bookkeeping (mark + // `changes.local`, and collect the loaded-group tabs to remove); the actual removals happen + // post-commit, so a commit failure leaves the live browser untouched (idempotent: a re-run + // recomputes the same plan and removes them then). for (const groupToRemove of syncResult.changes.groupsToRemove) { syncResult.changes.local = true; - if (Groups.isLoaded(groupToRemove.id)) { - // remove group from windows - await Groups.unload(groupToRemove.id); - - // remove tabs - if (!groupToRemove.isArchive) { - for (const tabToRemove of groupToRemove.tabs) { - syncResult.changes.tabsToRemove.add(tabToRemove); - } + if (Groups.isLoaded(groupToRemove.id) && !groupToRemove.isArchive) { + for (const tabToRemove of groupToRemove.tabs) { + syncResult.changes.tabsToRemove.add(tabToRemove); } } } progressFunc?.(90); - // remove unnecessary tabs - if (syncResult.changes.tabsToRemove.size) { - // if has local changes - do silent remove. "Cloud.sync-end" event will trigger "Groups.updated.all" event and reload all groups with tabs - await Tabs.remove(Array.from(syncResult.changes.tabsToRemove), syncResult.changes.local); - } - - progressFunc?.(95); - // set last-update before call saveOptions, saveOptions will reset alarm and it depends on last-update time storage.githubGistFileName = syncOptions.githubGistFileName; mainStorage.autoSyncLastTimeStamp = Utils.unixNow(); @@ -343,6 +346,25 @@ async function sync(trust = null, revision = null, progressFunc = null) { await Groups.save(syncResult.localData.groups); } + progressFunc?.(95); + + // C5: local DB commit has now succeeded (or there were no local changes) — only NOW perform + // the destructive browser-side teardown for removed groups/tabs. If anything above threw, we + // never reach here, so the live browser is left intact and consistent with the un-updated + // local DB. + for (const groupToRemove of syncResult.changes.groupsToRemove) { + if (Groups.isLoaded(groupToRemove.id)) { + // remove group from windows + await Groups.unload(groupToRemove.id); + } + } + + // remove unnecessary tabs + if (syncResult.changes.tabsToRemove.size) { + // if has local changes - do silent remove. "Cloud.sync-end" event will trigger "Groups.updated.all" event and reload all groups with tabs + await Tabs.remove(Array.from(syncResult.changes.tabsToRemove), syncResult.changes.local); + } + progressFunc?.(100); log.stop(); @@ -477,7 +499,8 @@ async function syncGroups(localData, cloudData, sourceOfTruth, changes) { } if (!changes.cloud) { - changes.cloud = JSON.stringify(resultCloudGroups) !== JSON.stringify(cloudGroups); + // ignore additive identity fields (uid/lastModified) so they don't trigger needless cloud writes + changes.cloud = stringifyGroupsForChangeDetection(resultCloudGroups) !== stringifyGroupsForChangeDetection(cloudGroups); } } else if (sourceOfTruth === TRUST_CLOUD) { @@ -988,6 +1011,32 @@ function isEqual(value1, value2) { return JSON.stringify(value1) === JSON.stringify(value2); } +// New per-tab identity fields (uid/lastModified) are additive metadata for sync (see B3). +// They must NOT influence whether a cloud write is needed: lastModified bumps on every +// title change and uid differs across machines, so including them in the equality check +// would trigger needless cloud writes. The uploaded snapshot still carries them - this +// only ignores them when deciding if a write is required. +const CHANGE_DETECTION_IGNORE_TAB_KEYS = ['uid', 'lastModified']; + +function stringifyGroupsForChangeDetection(groups) { + const stripped = groups.map(group => { + if (!Array.isArray(group.tabs)) { + return group; + } + + return { + ...group, + tabs: group.tabs.map(tab => { + const cleanTab = {...tab}; + CHANGE_DETECTION_IGNORE_TAB_KEYS.forEach(key => delete cleanTab[key]); + return cleanTab; + }), + }; + }); + + return JSON.stringify(stripped); +} + // alarm utils function isNetworkError(error) { const message = String(error); @@ -1006,6 +1055,37 @@ function isNetworkError(error) { return isNetErr; } +// C3: GitHub rate limiting (primary, secondary/abuse, or 429) is a TRANSIENT, retryable +// condition — the provider encodes it as a `githubRateLimit:` langId (see +// githubgist.js). Previously the retry classifier only recognised raw NetworkError strings, +// so a rate-limited sync got NO backoff retry (and abuse-limit 403s were even mis-mapped to a +// non-retryable auth error). Treat it as retryable and honour the reset time as the backoff. +// The rate-limit signal can arrive either as a CloudError `langId` (legacy synchronization(), +// which wraps provider errors) or only in the raw error `message` (delta-sync, whose catch +// copies `String(e)` into `syncResult.message` and leaves `langId` undefined for un-wrapped +// provider errors). Inspect BOTH so retry classification works on both sync paths. +function syncErrorText(syncResult) { + return [syncResult?.langId, syncResult?.message].filter(s => typeof s === 'string').join(' '); +} + +function getRateLimitResetMs(syncResult) { + const match = syncErrorText(syncResult).match(/githubRateLimit:(\d+)/); + if (!match) { + return null; + } + const resetMs = Number(match[1]); + return Number.isFinite(resetMs) ? resetMs : null; +} + +// A retryable (transient) failure: a raw network error, a GitHub rate-limit, or a snapshot +// If-Match precondition failure (a peer wrote concurrently — C1). All warrant an automatic +// backoff retry (which re-pulls + re-merges) rather than a user-facing hard error. +function isRetryableSyncError(syncResult) { + return isNetworkError(objectToNativeError(syncResult)) + || getRateLimitResetMs(syncResult) !== null + || syncErrorText(syncResult).includes('githubPreconditionFailed'); +} + export function onSyncUiRequestListener() { return CloudBroadcast.on('sync-ui-request', () => send('sync-ui-response')); } @@ -1034,7 +1114,9 @@ export async function shouldShowSyncErrorNotification(syncResult, trigger) { return result; } - if (trigger === TRIGGER_AUTO && !isNetworkError(objectToNativeError(syncResult))) { + // A transient/retryable failure (network OR rate-limit) is silently retried in the + // background until attempts are exhausted; only then surface a notification. + if (trigger === TRIGGER_AUTO && !isRetryableSyncError(syncResult)) { return true; } @@ -1047,11 +1129,21 @@ export async function getSyncRetryDelayInMinutes(syncResult, trigger) { return 0; } - if (isNetworkError(objectToNativeError(syncResult))) { + if (isRetryableSyncError(syncResult)) { const networkRetryAttempt = (syncStorage.networkRetryAttempt ?? 0) + 1; if (networkRetryAttempt <= MAX_NETWORK_RETRY_ATTEMPTS) { syncStorage.networkRetryAttempt = networkRetryAttempt; + + // For a rate-limit, back off until GitHub's reset time (rounded up to whole + // minutes, min 1) rather than the fixed network cadence — retrying sooner just + // burns another rejected request. Fall back to the network cadence otherwise. + const resetMs = getRateLimitResetMs(syncResult); + if (resetMs !== null) { + const minutesUntilReset = Math.ceil((resetMs - Date.now()) / 60_000); + return Math.min(Math.max(minutesUntilReset, 1), MAX_RATE_LIMIT_BACKOFF_MINUTES); + } + return networkRetryAttempt * NETWORK_RETRY_DELAY_MINUTES; } } diff --git a/addon/src/js/sync/cloud/githubgist.js b/addon/src/js/sync/cloud/githubgist.js index ce63843e..66b7676c 100644 --- a/addon/src/js/sync/cloud/githubgist.js +++ b/addon/src/js/sync/cloud/githubgist.js @@ -2,9 +2,36 @@ import '/js/prefixed-storage.js'; import * as Constants from '/js/constants.js'; import * as Utils from '/js/utils.js'; +import {SNAPSHOT_FILE_NAME, DELTA_FILE_PREFIX, LEGACY_BACKUP_FILE_NAME, LOCK_FILE_NAME} from '../delta/layout.js'; +import {canWriteLock, didWinLock, makeLockStamp, LOCK_CONFIRM_DELAY_MS} from '../delta/lock.js'; const storage = localStorage.create(Constants.MODULES.CLOUD); +// Per-gist persisted ETag for conditional pulls (delta-sync fast path). Stored in the +// synchronous CLOUD localStorage (same store as `lastUpdate`) so it survives restarts. +// Keyed by gist id so a token re-pointed at a different gist never reuses a stale ETag. +// Value is GitHub's opaque ETag string captured from the last gist GET / write response. +// See {@link GithubGist#isUnchangedSince} and the delta-sync conditional fast path. +const ETAG_STORAGE_KEY = 'gistEtag'; + +function readEtagMap() { + const map = storage[ETAG_STORAGE_KEY]; + return (map && typeof map === 'object') ? map : {}; +} + +function getStoredEtag(gistId) { + return gistId ? (readEtagMap()[gistId] ?? null) : null; +} + +function setStoredEtag(gistId, etag) { + if (!gistId || !etag) { + return; + } + const map = readEtagMap(); + map[gistId] = etag; + storage[ETAG_STORAGE_KEY] = map; +} + export default class GithubGist { #token = null; #fileName = null; @@ -12,6 +39,19 @@ export default class GithubGist { #perPage = null; // max = 100 + // Set true once a write RESPONSE's ETag has been captured into the store for the current + // gist revision (C2). While set, refreshEtagFromWrite() must NOT re-read via a follow-up + // GET — that GET could observe a THIRD device's interleaved write and overwrite our + // exact-revision marker, re-introducing the very skip-a-real-change bug C2 fixes. + #haveFreshWriteEtag = false; + + // Latest SERVER time (ms) parsed from any gist response's `Date` header. The advisory + // lock's TTL is computed against the SERVER clock (not this device's, which may be skewed) + // so a stale lock is judged by the same clock every device sees. Refreshed on every + // request that returns a `Date` header; null until the first such response. See + // {@link GithubGist#getServerTimeMs} and {@link GithubGist#acquireLock}. + #lastServerTimeMs = null; + constructor(token, fileName, perPage = 30) { if (!token) { throw new Error('githubInvalidToken', {cause: {isEmpty: true}}); @@ -61,10 +101,10 @@ export default class GithubGist { } } - async #findGist() { + async #findGist(markerFileName = this.#fileName) { this.#gistId = null; - const gist = await this.#findGistRecursive(); + const gist = await this.#findGistRecursive(markerFileName); if (gist) { this.#gistId = gist.id; @@ -72,18 +112,26 @@ export default class GithubGist { } } - async #findGistRecursive(page = 1) { + // `matcher` identifies "our" private gist: either an exact file name (string, + // the single-file flow keying on `this.#fileName` — unchanged) or a predicate + // over file names (the multi-file/delta flow, which matches ANY STG file so two + // devices converge on ONE gist even before a snapshot file exists). + async #findGistRecursive(matcher, page = 1) { const gists = await this.#request('GET', this.#mainUrl, { page, per_page: this.#perPage, }); - const gist = gists.find(g => !g.public && g.files[this.#fileName]); + const matches = typeof matcher === 'function' + ? files => Object.keys(files).some(matcher) + : files => Boolean(files[matcher]); + + const gist = gists.find(g => !g.public && matches(g.files)); if (gist) { return gist; } else if (gists.length === this.#perPage) { - return this.#findGistRecursive(++page); + return this.#findGistRecursive(matcher, ++page); } return null; @@ -114,16 +162,7 @@ export default class GithubGist { const gist = await this.getInfo(revision, progressApiFunc); - const file = gist.files[this.#fileName]; - - let content; - - if (file.truncated) { - content = await this.#request('GET', file.raw_url, undefined, undefined, progressRawFunc); - } else { - content = JSON.parse(file.content); - progressRawFunc(100); - } + const content = await this.#readFileContent(gist.files[this.#fileName], progressRawFunc); return withInfo ? [content, gist] : content; } catch (e) { @@ -145,19 +184,10 @@ export default class GithubGist { const progressSend = this.#createProgress(0, 70, progressFunc); const progressGet = this.#createProgress(70, 100, progressFunc); - if (this.hasGist) { - await this.#request('PATCH', this.#gistUrl, { - files, - }, undefined, progressSend); - } else { - const gist = await this.#request('POST', this.#mainUrl, { - public: false, - description, - files, - }, undefined, progressSend); - - this.#gistId = gist.id; - } + // Capture the write-response ETag (C2) for the next conditional PULL. We do NOT send + // If-Match here: GitHub does not support conditional requests on unsafe methods (PATCH), + // and rejects them with a bare 400 — see #patchOrCreate. + await this.#patchOrCreate({files}, description, progressSend); // sometimes git make wrong update the field "updated_at" minus 1 second :( // thats why we have to get info after update gist @@ -182,7 +212,465 @@ export default class GithubGist { return this.#processInfo(gist); } + // ------------------------------------------------------------------------- + // Multi-file (delta-era) API — ADDITIVE. The single-file methods above keep + // working unchanged for the current sync flow; these handle the new gist + // layout (STG-snapshot.json + per-device STG-delta-*.json). The delta gist is + // located by the presence of the snapshot file rather than `this.#fileName`. + // See `.project/DESIGN_DELTA_SYNC.md` and ../delta/layout.js. + // ------------------------------------------------------------------------- + + // Locate the gist holding the delta layout. Match ANY STG delta-era file + // (snapshot, a per-device delta, or the legacy backup) — NOT only the snapshot, + // which may not exist yet. Otherwise each device fails to find the shared gist + // and creates its own, so the devices never sync with each other. + async #findDeltaGist() { + this.hasGist || await this.#findGist(name => + name === SNAPSHOT_FILE_NAME + || name.startsWith(DELTA_FILE_PREFIX) + || name === LEGACY_BACKUP_FILE_NAME + // Also match the advisory-lock file: acquireLock runs BEFORE the first pull and may + // create a gist holding ONLY the lock; without this a later pull would #findGist and + // miss that gist (no snapshot/delta yet), creating a SECOND gist the devices never + // converge on. Matching the lock keeps both devices on the one gist. + || name === LOCK_FILE_NAME + ); + } + + /** + * Conditional "did the gist change since our last pull?" probe for the delta-sync + * fast path. Does a single `GET /gists/:id` carrying `If-None-Match: `; + * GitHub answers `304 Not Modified` (empty body, NOT counted against the rate limit) + * when nothing changed, or `200` with the gist + a fresh ETag when it did. + * + * FAIL-SAFE = "changed" (returns false): correctness must be identical to an + * unconditional sync, so ANY doubt forces a full fetch. We return `false` when there + * is no gist yet, no stored ETag (first sync / discovery), or on ANY transport/HTTP + * error. The skip is purely an optimization layered on top — never a correctness gate. + * + * On a 200 the new ETag is persisted immediately, so even when the caller proceeds to + * a full fetch the next cycle's probe is primed. (A successful write also refreshes the + * stored ETag via {@link GithubGist#refreshEtagFromWrite}.) + * + * @param {?function} progressFunc + * @returns {Promise} `true` ONLY when GitHub confirmed 304 (safe to skip the + * pull/apply); `false` in every other case (caller must do a full fetch). + */ + async isUnchangedSince(progressFunc = null) { + try { + await this.#findDeltaGist(); + + if (!this.hasGist) { + return false; // no gist discovered yet ⇒ full fetch (first sync / discovery) + } + + const storedEtag = getStoredEtag(this.#gistId); + if (!storedEtag) { + return false; // nothing to compare against ⇒ full fetch, then capture below + } + + const {status, etag} = await this.#conditionalGet(this.#gistUrl, storedEtag, progressFunc); + + if (status === 304) { + return true; // confirmed unchanged ⇒ safe to skip pull/apply + } + + // 200 (or anything else 2xx): the gist changed. Capture the fresh ETag so the + // upcoming full fetch's result is primed for the next cycle, then report changed. + if (etag) { + setStoredEtag(this.#gistId, etag); + } + + return false; + } catch { + // transport / parse / HTTP error ⇒ fail safe to a full fetch. The optimization + // never suppresses a real remote change; worst case is one extra unconditional pull. + return false; + } + } + + /** + * After a successful write (push), refresh the stored ETag from the gist's current + * state so the NEXT cycle's {@link GithubGist#isUnchangedSince} probe compares against + * the post-push revision (otherwise our own push would always read back as "changed"). + * Best-effort: a failure just means the next probe does a full fetch (fail-safe). + * @param {?function} progressFunc + * @returns {Promise} + */ + async refreshEtagFromWrite(progressFunc = null) { + try { + if (!this.hasGist) { + return; + } + // C2: if our last write already captured its response ETag, that marker pins the + // EXACT revision we produced. A follow-up GET here could instead observe a THIRD + // device's write that landed in between and store ITS ETag as "ours" — making the + // next probe 304 against a revision we never applied (the bug C2 fixes). So skip + // the GET entirely when the write-response ETag is already in hand. + if (this.#haveFreshWriteEtag) { + this.#haveFreshWriteEtag = false; + return; + } + const {etag} = await this.#conditionalGet(this.#gistUrl, null, progressFunc); + if (etag) { + setStoredEtag(this.#gistId, etag); + } + } catch { + // ignore: a missing ETag only costs one extra full fetch next cycle (fail-safe). + } + } + + /** + * Low-level conditional GET that, unlike {@link GithubGist#getInfo}, reads the response + * HEADERS (for the ETag) and tolerates an empty `304` body. Bypasses `#progressFetch`'s + * streaming JSON wrapper (which would throw on a bodyless 304) and `#request` (which + * discards headers). Sends `If-None-Match` when `etag` is provided. Returns the status + * + the response ETag; never parses the body (we only need "changed or not" + the tag). + * @param {string} url - the gist API url. + * @param {?string} etag - prior ETag to send as `If-None-Match`, or null for an unconditional read. + * @param {?function} progressFunc + * @returns {Promise<{status: number, etag: ?string}>} + */ + async #conditionalGet(url, etag, progressFunc = null) { + const headers = { + ...GithubGist.defaultHeaders, + Authorization: `Bearer ${this.#token}`, + }; + if (etag) { + headers['If-None-Match'] = etag; + } + + // no-store so the browser HTTP cache can't shortcut the conditional request before + // it reaches GitHub (mirrors `#request`'s GET cache policy). + const response = await fetch(url, {method: 'GET', headers, cache: 'no-store'}); + + this.#captureServerTime(response); + + progressFunc?.(100); + + return { + status: response.status, + etag: response.headers.get('etag'), + }; + } + + /** + * Record the SERVER time from a response's `Date` header (RFC 1123) into + * `#lastServerTimeMs`. Best-effort: a missing/unparseable header leaves the prior value + * untouched. The advisory lock reads it via {@link GithubGist#getServerTimeMs}. + * @param {Response} response + */ + #captureServerTime(response) { + const dateHeader = response?.headers?.get?.('date'); + if (!dateHeader) { + return; + } + const ms = Date.parse(dateHeader); + if (Number.isFinite(ms)) { + this.#lastServerTimeMs = ms; + } + } + + /** + * The SERVER time (ms) most recently observed from a gist response's `Date` header, or + * null if no response has carried one yet this session. Used by the advisory lock to + * compute / judge TTLs against GitHub's clock rather than this device's (which may be + * skewed). Callers fall back to `Date.now()` when this is null (see acquireLock). + * @returns {?number} + */ + getServerTimeMs() { + return this.#lastServerTimeMs; + } + + // Read+parse one gist file object, transparently following `raw_url` for files + // GitHub truncates (mirrors getContent's large-file handling). Returns the + // parsed JSON content. Caller maps a SyntaxError to 'githubInvalidGistContent'. + async #readFileContent(file, progressFunc = null) { + if (file.truncated) { + return this.#request('GET', file.raw_url, undefined, undefined, progressFunc); + } + + const content = JSON.parse(file.content); + progressFunc?.(100); + return content; + } + + /** + * List the names of every file currently in the (delta) gist. + * @param {?function} progressFunc + * @returns {Promise} file names ([] if the gist does not exist yet). + */ + async listFiles(progressFunc = null) { + await this.#findDeltaGist(); + + if (!this.hasGist) { + return []; + } + + const gist = await this.getInfo(undefined, progressFunc); + + return Object.keys(gist.files); + } + + /** + * Read and parse a single named file from the (delta) gist. + * @param {string} name - file name (e.g. SNAPSHOT_FILE_NAME). + * @param {?function} progressFunc + * @returns {Promise} parsed content, or null if the file is absent. + */ + async readFile(name, progressFunc = null) { + try { + const progressApiFunc = this.#createProgress(0, 50, progressFunc); + const progressRawFunc = this.#createProgress(50, 100, progressFunc); + + await this.#findDeltaGist(); + + if (!this.hasGist) { + return null; + } + + const gist = await this.getInfo(undefined, progressApiFunc); + const file = gist.files[name]; + + if (!file) { + progressRawFunc(100); + return null; + } + + return await this.#readFileContent(file, progressRawFunc); + } catch (e) { + if (e instanceof SyntaxError) { + throw new Error('githubInvalidGistContent', {cause: e}); + } + + throw e; + } + } + + /** + * Read+parse every gist file whose name starts with `prefix` (e.g. the + * DELTA_FILE_PREFIX to fetch all per-device delta logs). Truncated files are + * followed via `raw_url` like {@link readFile}. + * @param {string} prefix + * @param {?function} progressFunc + * @returns {Promise>} ([] if no gist). + */ + async readAllMatching(prefix, progressFunc = null) { + try { + const progressApiFunc = this.#createProgress(0, 30, progressFunc); + + await this.#findDeltaGist(); + + if (!this.hasGist) { + return []; + } + + const gist = await this.getInfo(undefined, progressApiFunc); + + const matching = Object.entries(gist.files).filter(([name]) => name.startsWith(prefix)); + + const results = []; + for (const [index, [name, file]] of matching.entries()) { + // spread each file's read across the remaining 30..100 progress band + const from = 30 + Math.floor((index / matching.length) * 70); + const to = 30 + Math.floor(((index + 1) / matching.length) * 70); + const content = await this.#readFileContent(file, this.#createProgress(from, to, progressFunc)); + results.push({name, content}); + } + + return results; + } catch (e) { + if (e instanceof SyntaxError) { + throw new Error('githubInvalidGistContent', {cause: e}); + } + + throw e; + } + } + + /** + * Write multiple files in a single PATCH (or POST to create the gist on first + * write). `contents` is a `{ [fileName]: contentObject }` map; each value is + * JSON-stringified by the request machinery. Per-device delta files mean + * concurrent writers touch different files and never clobber each other. + * @param {Object} contents + * @param {string} [description] + * @param {?function} progressFunc + * @returns {Promise} the refreshed gist info (incl. `lastUpdate`). + */ + async writeFiles(contents, description = '', progressFunc = null) { + await this.#findDeltaGist(); + + const files = {}; + for (const [name, content] of Object.entries(contents)) { + files[name] = {content}; + } + + const progressSend = this.#createProgress(0, 70, progressFunc); + const progressGet = this.#createProgress(70, 100, progressFunc); + + // No If-Match guard: GitHub does not support conditional requests on unsafe methods + // (PATCH/POST/DELETE) and rejects an If-Match'd gist PATCH with a bare 400 (the ETag it + // returns is weak, which If-Match forbids anyway). Snapshot clobbering between two + // simultaneously-compacting devices is harmless — per-device delta files use disjoint + // keys and are never clobbered, so the next compaction simply re-folds them; the sync + // re-pulls + re-merges every cycle. The write-response ETag is still captured (C2) for + // the conditional PULL fast path (If-None-Match on GET, which GitHub DOES support). + await this.#patchOrCreate({files}, description, progressSend); + + // refresh info after write (GitHub sometimes back-dates updated_at by 1s) + return await this.getInfo(undefined, progressGet); + } + + /** + * Execute one PATCH-or-POST write to the gist and capture the resulting ETag (C2): the + * ETag stored as "our last-known" must pin EXACTLY the revision we produced, so we read + * it from the write RESPONSE — never from a follow-up GET, which a third device could + * have advanced between our write and that GET. Sets the gist id on first create. + * + * @param {Object} body - the request body (`{files, ...}`). + * @param {string} description - gist description (create only). + * @param {?function} progressSend + * @returns {Promise} + */ + async #patchOrCreate(body, description, progressSend) { + let response; + + if (this.hasGist) { + // No If-Match: GitHub rejects conditional requests on unsafe methods (PATCH) with a + // bare 400 Bad Request, and the ETag it returns is weak (invalid in If-Match anyway). + response = await this.#requestRaw('PATCH', this.#gistUrl, body, undefined, progressSend); + } else { + // first write: no gist yet ⇒ unconditional create (nothing to guard against). + response = await this.#requestRaw('POST', this.#mainUrl, { + public: false, + description, + ...body, + }, undefined, progressSend); + + const gist = await response.clone().json(); + this.#gistId = gist.id; + } + + // C2: pin the stored conditional-pull marker to the exact revision THIS write produced. + const etag = response.headers.get('etag'); + if (etag) { + setStoredEtag(this.#gistId, etag); + this.#haveFreshWriteEtag = true; // suppress the redundant refreshEtagFromWrite GET + } + } + + /** + * Delete one file from the gist (PATCH with the file set to `null`, GitHub's + * delete primitive). Used by P4 compaction to trim folded delta files. + * @param {string} name + * @param {?function} progressFunc + * @returns {Promise} the refreshed gist info. + */ + async deleteFile(name, progressFunc = null) { + await this.#findDeltaGist(); + + if (!this.hasGist) { + throw new Error('githubNotFound'); + } + + const progressSend = this.#createProgress(0, 70, progressFunc); + const progressGet = this.#createProgress(70, 100, progressFunc); + + // Compaction GC only deletes per-device delta files (never the shared snapshot), so + // no If-Match guard is needed; still capture the write-response ETag (C2) so the next + // conditional probe pins the post-delete revision. + await this.#patchOrCreate({ + files: { + [name]: null, + }, + }, '', progressSend); + + return await this.getInfo(undefined, progressGet); + } + + // ------------------------------------------------------------------------- + // Advisory distributed lock (Part A) — serialize sync cycles across devices. + // Best-effort, NOT compare-and-set (GitHub has no conditional write): acquire = + // write-our-stamp THEN re-read to confirm we won any race; a crashed holder's stamp is + // reclaimed via the server-clock TTL. Deferred self-truncation (compaction.js) is the + // data-safety backstop. The lock file holds `{deviceId, expiresAt}` (server-clock ms). + // See ../delta/lock.js for the pure decision helpers used here. + // ------------------------------------------------------------------------- + + /** + * Try to ACQUIRE the advisory sync lock for `deviceId`. Protocol: + * 1. Read the lock file (this refreshes the server clock from the response `Date`). + * 2. If it is absent / stale / already ours → write our stamp `{deviceId, expiresAt}` + * with `expiresAt = serverNow + LOCK_TTL_MS`. If it is held, fresh, and another + * device's → DO NOT acquire (return false) without writing. + * 3. After writing, wait a short confirm delay then RE-READ. We won iff the re-read + * stamp is still ours (a peer that wrote last in the race wins instead). + * + * Best-effort: ANY transport error returns false (caller skips this cycle and retries), + * never throwing — the advisory lock must never be the thing that breaks a sync. + * + * @param {string} deviceId - this device's id (getDeviceId()). + * @param {?function} progressFunc + * @returns {Promise} true iff this device now holds the lock. + */ + async acquireLock(deviceId, progressFunc = null) { + try { + const lock = await this.readFile(LOCK_FILE_NAME, progressFunc); + const serverNow = this.getServerTimeMs() ?? Date.now(); + + if (!canWriteLock(lock, deviceId, serverNow)) { + return false; // held, fresh, and another device's ⇒ back off + } + + await this.writeFiles({ + [LOCK_FILE_NAME]: makeLockStamp(deviceId, serverNow), + }); + + // short confirm delay so a racing peer's write becomes observable, then re-read: + // exactly one stamp survives the last write, and the read-back reveals the winner. + await Utils.wait(LOCK_CONFIRM_DELAY_MS); + + const confirmed = await this.readFile(LOCK_FILE_NAME, progressFunc); + return didWinLock(confirmed, deviceId); + } catch { + // any failure ⇒ treat as not-acquired (fail-safe: skip the cycle, retry later). + return false; + } + } + + /** + * RELEASE the advisory sync lock by deleting the lock file (GitHub's `{file: null}` + * delete primitive, via {@link GithubGist#deleteFile}). Idempotent + best-effort: a + * missing file or any transport error is swallowed, since a stale lock is reclaimed by + * the TTL anyway. Always call this in a `finally` covering success, error, and the + * apply-watchdog path. + * @param {?function} progressFunc + * @returns {Promise} + */ + async releaseLock(progressFunc = null) { + try { + await this.deleteFile(LOCK_FILE_NAME, progressFunc); + } catch { + // ignore: best-effort. The TTL reclaims a lock we failed to delete. + } + } + async #request(method, url, body = null, options = {}, progressFunc = null) { + const response = await this.#requestRaw(method, url, body, options, progressFunc); + return response.json(); + } + + /** + * Like {@link GithubGist#request} but resolves with the validated `Response` + * object instead of its parsed JSON body, so a caller can read response HEADERS + * (notably the `ETag` produced by a write — see C2). All status / rate-limit / + * scope error mapping runs BEFORE returning, identical to `#request`, so a non-ok + * response throws the same provider error it always did. + * + * @returns {Promise} the ok Response (body not yet consumed). + */ + async #requestRaw(method, url, body = null, options = {}, progressFunc = null) { const isApi = url.startsWith(GithubGist.apiUrl); options.method = method; @@ -199,7 +687,8 @@ export default class GithubGist { } else if (body) { if (body.files) { for (const file of Object.values(body.files)) { - if (file.content && typeof file.content !== 'string') { + // a null file is GitHub's delete primitive (see deleteFile) - leave it + if (file && file.content && typeof file.content !== 'string') { file.content = JSON.stringify(file.content, null, 2); } } @@ -211,8 +700,12 @@ export default class GithubGist { const response = await this.#progressFetch(url, options, progressFunc); + // Capture the SERVER clock from every response (ok or not) so the advisory lock's TTL + // is judged against GitHub's clock, immune to this device's local clock skew. + this.#captureServerTime(response); + if (response.ok) { - return response.json(); + return response; } if (isApi) { @@ -231,12 +724,34 @@ export default class GithubGist { throw new Error('githubInvalidToken'); } - if (response.status === 403) { - if (response.headers.get('x-ratelimit-remaining') === '0') { + // C3: rate limiting. GitHub signals it three ways, only the first of which was + // handled before: + // 1. PRIMARY limit: 403 with `x-ratelimit-remaining: 0` + `x-ratelimit-reset`. + // 2. SECONDARY/abuse limit: 403 or 429 with a `Retry-After` header (seconds) and + // NO `x-ratelimit-remaining: 0` — previously mis-mapped to `githubTokenNoAccess` + // (a non-retryable auth error). + // 3. 429 generally. + // All map to a `githubRateLimit:` error the retry classifier treats as + // retryable with backoff (respecting Retry-After when present). + if (response.status === 403 || response.status === 429) { + const retryAfter = response.headers.get('retry-after'); + const remaining = response.headers.get('x-ratelimit-remaining'); + + if (remaining === '0') { const unix = response.headers.get('x-ratelimit-reset'); throw new Error(`githubRateLimit:${unix}000`); } + if (retryAfter !== null || response.status === 429) { + // Retry-After is delta-seconds; convert to an absolute unix-ms "reset" the + // CloudError formatter + retry classifier understand. Default to ~60s when the + // header is absent (a 429 with no Retry-After). + const seconds = Number(retryAfter); + const delayMs = Number.isFinite(seconds) && seconds >= 0 ? seconds * 1000 : 60_000; + throw new Error(`githubRateLimit:${Date.now() + delayMs}`); + } + + // a genuine 403 (forbidden / no gist scope) with no rate-limit signal throw new Error('githubTokenNoAccess'); } @@ -244,6 +759,22 @@ export default class GithubGist { throw new Error('githubNotFound'); } + // C1: `If-Match` precondition failed — a concurrent writer advanced the gist since + // the ETag we sent. Surface a distinct marker so the snapshot-write caller can + // re-pull the current revision and retry once (instead of clobbering the peer). + if (response.status === 412) { + throw new Error('githubPreconditionFailed', {cause: response}); + } + + // C4: a non-API host (e.g. a truncated file's raw_url on gist.githubusercontent.com) + // never carries the JSON `{message, errors}` error envelope and may return an HTML + // 5xx body. Parsing it as JSON below would throw a SyntaxError that the read callers + // mis-map to `githubInvalidGistContent` (corruption) instead of a retryable transport + // error. Surface the raw status so the failure stays retryable, not "corrupt". + if (!isApi) { + throw new Error(`${response.status}: github raw request failed`, {cause: response}); + } + const result = await response.clone().json(); const errors = result.errors?.map(err => err.message) ?? []; const errorsMessage = errors.length ? `. Errors: ${errors.join(', ')}` : ''; @@ -251,7 +782,7 @@ export default class GithubGist { if (response.status === 422) { if (['contents', 'large'].every(s => errorsMessage.includes(s))) { const bytes = Object.values(body.files) - .map(file => Utils.encodeToBytes(file.content).length) + .map(file => file?.content ? Utils.encodeToBytes(file.content).length : 0) .reduce((acc, fSize) => acc + fSize, 0); throw new Error(`githubContentsTooLarge:${bytes}`); @@ -350,7 +881,13 @@ export default class GithubGist { cache.responseLength = xhr.responseText.length; } - const headers = xhr.getAllResponseHeaders().trim().split(/[\r\n]+/).map(header => header.split(': ')); + const headers = xhr.getAllResponseHeaders().trim().split(/[\r\n]+/).reduce((acc, line) => { + const sep = line.indexOf(': '); + if (sep !== -1) { + acc.push([line.slice(0, sep), line.slice(sep + 2)]); + } + return acc; + }, []); return new Response(xhr.responseText, { status: xhr.status, diff --git a/addon/src/js/sync/cloud/provider.js b/addon/src/js/sync/cloud/provider.js new file mode 100644 index 00000000..47305e63 --- /dev/null +++ b/addon/src/js/sync/cloud/provider.js @@ -0,0 +1,150 @@ + +import GithubGist from './githubgist.js'; + +/** + * Provider-type constants. These identify which cloud backend the sync engine + * talks to. The selected value is stored as the local (per-device) option + * `syncProvider` (see `DEFAULT_OPTIONS` in constants.js) and is NOT part of the + * synced payload. + */ +export const PROVIDER_GITHUB_GIST = 'github-gist'; +export const PROVIDER_GOOGLE_DRIVE = 'google-drive'; + +/** + * CloudProvider contract. + * + * The sync engine (`cloud.js`) is provider-agnostic: it only depends on the + * methods documented below. Any new backend must implement this same contract + * so it can be returned by `createCloudProvider()` without further changes to + * the engine. + * + * @typedef {Object} CloudInfo + * @property {string} lastUpdate - ISO 8601 timestamp of the last cloud update + * (e.g. `"2024-01-01T00:00:00Z"`). Used by the engine to decide the source + * of truth (local vs cloud). + * + * @typedef {Object} CloudProvider + * + * @property {(revision: ?string, progressFunc: ?function) => Promise} getInfo + * Fetch metadata about the current cloud content (optionally for a given + * revision). Resolves with a `CloudInfo`. Throws `Error('githubNotFound')` + * (provider-specific "not found" marker) when there is no cloud content yet. + * + * @property {(revision: ?string, withInfo: ?boolean, progressFunc: ?function) => Promise<(Object|[Object, CloudInfo])>} getContent + * Fetch the stored backup content. When `withInfo` is falsy, resolves with + * the parsed content object. When `withInfo` is truthy, resolves with the + * tuple `[content, info]` where `info` is a `CloudInfo`. Throws + * `Error('githubNotFound')` when there is no cloud content yet (the engine + * treats this as "first sync"). + * + * @property {(content: Object, description: ?string, progressFunc: ?function) => Promise} setContent + * Upload/replace the backup content. Resolves with the resulting + * `CloudInfo` (notably `info.lastUpdate`, which the engine writes back to + * `syncLastUpdate`). + * + * @property {() => Promise<(boolean|undefined)>} checkToken + * Validate the provider credentials. Used by the options UI. Resolves + * (truthy/undefined) when valid; throws a provider-specific error otherwise. + * + * --- Multi-file (delta-era) methods (Phase P3a) ------------------------------- + * These back the hybrid snapshot + delta layout (one container holding + * `STG-snapshot.json` + per-device `STG-delta-.json`; see + * `.project/DESIGN_DELTA_SYNC.md` and `../delta/layout.js`). They are ADDITIVE: + * the single-file methods above remain the contract the current `cloud.js` sync + * flow uses. A multi-file container is located by the presence of the snapshot + * file, so absence of the container resolves to "empty" rather than throwing. + * + * @property {(name: string, progressFunc: ?function) => Promise} readFile + * Read+parse one named file from the container. Resolves with the parsed + * content, or `null` if the file (or the whole container) is absent. Large + * files are fetched transparently. A malformed JSON file throws + * `Error('githubInvalidGistContent')`. + * + * @property {(progressFunc: ?function) => Promise} listFiles + * List the names of every file in the container (`[]` if no container yet). + * + * @property {(prefix: string, progressFunc: ?function) => Promise>} readAllMatching + * Read+parse every file whose name starts with `prefix` (e.g. all per-device + * delta logs). Resolves with `[{name, content}, ...]` (`[]` if no container). + * + * @property {(contents: Object, description: ?string, progressFunc: ?function) => Promise} writeFiles + * Write multiple files in a single atomic request (creating the container on + * first write). `contents` maps file name → content object. Resolves with the + * resulting `CloudInfo`. Per-file granularity lets concurrent devices write + * their own delta files without clobbering each other. + * + * @property {(name: string, progressFunc: ?function) => Promise} deleteFile + * Delete a single named file from the container (compaction primitive, P4). + * Resolves with the resulting `CloudInfo`. Throws `Error('githubNotFound')` + * if there is no container. + * + * --- Conditional-pull fast path (optimization, optional) ---------------------- + * Lets the sync engine skip the download + replay/apply when the remote container is + * byte-for-byte unchanged since the last cycle, using the backend's native conditional + * request (e.g. HTTP ETag / If-None-Match). Both methods are OPTIONAL and FAIL-SAFE: a + * provider that doesn't implement them simply omits them, and the engine falls back to an + * unconditional full fetch (correctness is identical either way). + * + * @property {(progressFunc: ?function) => Promise} [isUnchangedSince] + * Probe whether the container changed since the last pull. Resolves `true` ONLY when + * the backend positively confirms "unchanged" (so the engine may skip pull+apply this + * cycle); resolves `false` on ANY doubt (no prior marker, first sync, discovery, or + * transport error) so the engine does a full fetch. MUST NOT throw. + * + * @property {(progressFunc: ?function) => Promise} [refreshEtagFromWrite] + * Refresh the provider's stored conditional-request marker from the container's + * current state after a successful write/push, so the NEXT `isUnchangedSince` probe + * compares against the just-pushed revision. Best-effort; MUST NOT throw. + * + * --- Advisory distributed lock (Part A, optional) ----------------------------- + * Serializes sync cycles across devices so two don't write the snapshot concurrently. Since + * the backend has no atomic compare-and-set (a gist `If-Match` PATCH returns a bare 400), the + * lock is ADVISORY: acquire = write-then-read-back-to-confirm; a crashed holder is reclaimed + * via a server-clock TTL. All three methods are OPTIONAL: a provider omitting them simply runs + * unserialized (deferred self-truncation in delta-sync.js is the data-safety backstop). + * + * @property {(deviceId: string, progressFunc: ?function) => Promise} [acquireLock] + * Try to acquire the lock for `deviceId`. Resolves `true` iff this device now holds it, + * `false` if a peer holds a fresh lock or on ANY error (caller skips + retries). MUST NOT throw. + * + * @property {(progressFunc: ?function) => Promise} [releaseLock] + * Release the lock (delete the lock file). Idempotent + best-effort; MUST NOT throw. + * Always call in a `finally` (success, error, and watchdog paths). + * + * @property {() => ?number} [getServerTimeMs] + * The SERVER time (ms) most recently observed from a response `Date` header, or null if + * none seen yet. Lets the lock judge TTLs against the backend clock, not the device's. + * + * Notes: + * - `progressFunc` is an optional `(percent: number) => void` callback the + * provider may call to report transfer progress. + * - Errors thrown by providers use short language-id messages (see `Lang`), + * which `cloud.js` wraps in `CloudError`. + */ + +/** + * Factory that returns the cloud provider instance for the given type. + * + * Existing GitHub Gist users (no `syncProvider`, or `syncProvider === 'github-gist'`) + * get exactly the same `GithubGist` instance as before, so behavior is preserved. + * + * @param {string} providerType - one of the `PROVIDER_*` constants. + * @param {Object} syncOptions - the resolved sync options (token, file name, ...). + * @returns {CloudProvider} + */ +export function createCloudProvider(providerType, syncOptions) { + switch (providerType) { + case PROVIDER_GOOGLE_DRIVE: + // Extension point: a later branch will return a GoogleDrive provider + // instance here, implementing the CloudProvider contract above. + throw new Error('cloudProviderNotImplemented'); + + case PROVIDER_GITHUB_GIST: + default: + // Default/fallback so existing users (and unset option) keep working. + return new GithubGist( + syncOptions.githubGistToken, + syncOptions.githubGistFileName + ); + } +} diff --git a/addon/src/js/sync/delta/applied-nav-echo.js b/addon/src/js/sync/delta/applied-nav-echo.js new file mode 100644 index 00000000..be26f108 --- /dev/null +++ b/addon/src/js/sync/delta/applied-nav-echo.js @@ -0,0 +1,59 @@ +/** + * Pure decision for the sync-applied navigation ECHO guard (see {@link module:sync/delta/delta-capture}). + * + * The transport's content-update path navigates a live tab via `browser.tabs.update({url})`, + * which resolves as soon as the navigation STARTS; the resulting `onUpdated` (and any server + * redirect it triggers) fires ASYNCHRONOUSLY, AFTER the apply's `endApply()` has run. Without a + * guard that delayed echo is re-captured as a fresh `tab.modify`/`pinned.modify` and pushed next + * cycle — and on a redirect the captured live url never equals the cloud url, so the planner + * re-emits the update EVERY cycle (perpetual churn). This module holds the import-free decision + * for "is this content change a sync-applied echo (suppress) or a genuine user navigation (sync)?" + * so the echo-vs-user-nav seam is unit-testable without the browser/cache dependencies of the + * capture layer. + * + * @module sync/delta/applied-nav-echo + */ + +/** + * Is a content change a sync-APPLIED navigation echo that must be suppressed (vs a genuine user + * navigation — or a server REDIRECT — that must sync)? + * + * - TRUE while a sync apply is in progress (`applying`): the in-apply synchronous suppression. + * Every change landing during the apply is the transport's own write, so it is always an echo. + * - When the tab carries a live applied-navigation mark (`now < markExpiry`), the decision is + * NARROWED BY URL — this is the convergence fix for "loads infinitely": + * · if the observed url EQUALS the url the apply navigated this tab to (`markUrl`), it is the + * plain settle echo of exactly what we applied ⇒ SUPPRESS (don't re-capture our own write); + * · if the observed url DIFFERS from `markUrl`, the page server-REDIRECTED (applied X → landed + * on Y). That Y is genuinely new information the cloud does not have: it MUST be captured and + * pushed so the cloud converges to Y (after which live==cloud and the planner stops + * re-emitting the update). So a redirect to a DIFFERENT url is NOT an echo ⇒ CAPTURE. + * · when `markUrl` is unknown (legacy mark without a url, or no observed url to compare), fall + * back to the old window-based suppression (suppress while the mark is live) — safe default. + * - FALSE otherwise — including an EXPIRED mark (`now >= markExpiry`) — so a user navigation made + * outside the apply's tight causal window syncs normally (preserves A6 user-nav capture). + * + * @param {object} args + * @param {boolean} args.applying - is a sync apply in progress right now. + * @param {number} [args.markExpiry] - this tab's applied-navigation mark expiry (epoch ms), or undefined. + * @param {string} [args.markUrl] - the EXACT url the apply navigated this tab to (the url whose echo + * must be suppressed). Undefined for a legacy/url-less mark. + * @param {string} [args.observedUrl] - the url of the content change being classified right now. + * @param {number} args.now - current epoch ms. + * @returns {boolean} true ⇒ suppress (echo); false ⇒ capture (user navigation / redirect). + */ +export function isAppliedNavigationEcho({applying, markExpiry, markUrl, observedUrl, now}) { + if (applying) { + return true; + } + if (!Number.isFinite(markExpiry) || now >= markExpiry) { + return false; // no live mark ⇒ not an echo (user nav outside the causal window syncs). + } + // Live mark. Narrow by url: suppress ONLY the echo of the exact applied url; let a redirect + // to a DIFFERENT url through so the cloud can converge to the redirect target. + if (typeof markUrl === 'string' && typeof observedUrl === 'string') { + return observedUrl === markUrl; + } + // url-less mark / no observed url to compare ⇒ window-based suppression (legacy-safe default). + return true; +} diff --git a/addon/src/js/sync/delta/compaction.js b/addon/src/js/sync/delta/compaction.js new file mode 100644 index 00000000..8a8347ba --- /dev/null +++ b/addon/src/js/sync/delta/compaction.js @@ -0,0 +1,259 @@ + +/** + * Pure compaction policy for hybrid snapshot + delta sync (Phase P4). + * + * The cloud holds a consolidated BASE snapshot (`STG-snapshot.json`) carrying a + * `watermark` map `{deviceId: seq}` — the highest delta `seq` from each device ALREADY + * folded into the base — plus per-device append-only `STG-delta-.json` logs. Replay + * (`replay.js`) computes the effective state as `base + every event with seq > watermark`, + * SKIPPING anything `seq <= watermark[device]` (replay.js dedup, the safety foundation). + * + * Two things grow unboundedly without compaction: the cloud rewrites the full snapshot + * every cycle (wasteful), and the per-device delta logs are never truncated. This module + * decides — PURELY — WHEN to compact and WHICH of THIS device's own events are safe to + * truncate, so the impure transport (`delta-sync.js`) can act on it. + * + * ## What "folding" is (no second fold path) + * The planner already replays `base + all pulled+pending events` into a RESOLVED snapshot + * each cycle, and `replay()` returns that snapshot's `watermark` advanced to the max applied + * `seq` per device. THAT resolved snapshot IS the folded base, and its watermark IS the + * advanced watermark — there is no separate folding step here. Compaction is simply the + * DECISION to (a) persist that resolved snapshot as the new base and (b) truncate the now- + * folded events from this device's OWN log. We never recompute the fold a second, divergent + * way; correctness rides entirely on the single pure replay engine. + * + * ## Trigger + * When the number of UNFOLDED events — events whose `seq` exceeds the BASE (pre-replay) + * watermark for their device, summed across ALL device delta logs this device pulled — + * exceeds {@link COMPACTION_THRESHOLD}, the next successful sync compacts. + * + * ## Non-blocking w.r.t. lagging / lost devices (hard requirement) + * The count and the fold use ONLY this device's pulled view. We do NOT gate on a + * min-watermark across devices and we NEVER wait for other devices to catch up. We fold + * ALL pulled events into the base, advancing EACH device's watermark to the highest folded + * seq for that device. A behind/lost device that later returns pulls the NEW base (which + * already contains the folded effect) plus whatever of its own deltas remain unfolded — + * nothing is lost. A permanently-lost device's stale delta file simply lingers in the gist + * and is skipped on replay (`seq <= watermark`); it is harmless, never a blocker. + * + * ## Own-log truncation only + * A device may rewrite ONLY its own delta file (the transport writes this device's delta + + * the snapshot, nothing else — confirmed in delta-sync.js). So here we compute the highest + * SELF seq that the new base has folded ({@link selfFoldedSeq}); the transport truncates the + * local {@link module:sync/delta/delta-log} up to it and writes back only the self events + * with `seq` beyond it. Other devices' stale events are NOT rewritten — the advanced + * watermark already makes replay skip them, and a returning device still owns its own file. + * + * ## Purity + * No `browser.*`, no network, no `constants.js`. Reads its inputs, returns plain data. + * + * @module sync/delta/compaction + */ + +/** + * Compact once the unfolded-event count EXCEEDS this many. A modest cap: large enough that + * a normal session's deltas accumulate cheaply between compactions, small enough that the + * logs never grow without bound. Compaction is the only time the full snapshot is rewritten. + * @type {number} + */ +export const COMPACTION_THRESHOLD = 100; + +/** + * Count the UNFOLDED events across all pulled device delta logs — events whose `seq` + * strictly exceeds the BASE (pre-replay) watermark for their device. This mirrors EXACTLY + * the replay dedup predicate (`event.seq > baseWatermark[deviceId]`, see replay.js), so the + * count equals the number of events replay will actually fold this cycle. Events at/below + * the watermark are already in the base and are not counted (nor folded, nor truncated). + * + * A null/`undefined` `seq` is treated as unfolded (counted) — it can never be `<=` a numeric + * watermark, matching replay, which only skips when `seq != null && seq <= folded`. + * + * @param {Array<{deviceId: string, events: object[]}>} pulledDeltaLogs - logs as pulled. + * @param {object} [baseWatermark] - the snapshot's watermark BEFORE this cycle's replay. + * @returns {number} count of events with seq beyond their device's base watermark. + */ +export function countUnfoldedEvents(pulledDeltaLogs, baseWatermark = {}) { + const wm = baseWatermark || {}; + let count = 0; + + for (const log of pulledDeltaLogs || []) { + const folded = wm[log?.deviceId] ?? 0; + for (const event of log?.events || []) { + if (event.seq == null || event.seq > folded) { + count += 1; + } + } + } + + return count; +} + +/** + * Decide whether THIS cycle should compact: true iff the unfolded-event count EXCEEDS + * {@link COMPACTION_THRESHOLD}. Pure predicate over the pulled logs + base watermark. + * + * @param {Array<{deviceId: string, events: object[]}>} pulledDeltaLogs + * @param {object} [baseWatermark] + * @param {number} [threshold] - override (tests); defaults to {@link COMPACTION_THRESHOLD}. + * @returns {{shouldCompact: boolean, unfoldedCount: number}} + */ +export function evaluateCompaction(pulledDeltaLogs, baseWatermark = {}, threshold = COMPACTION_THRESHOLD) { + const unfoldedCount = countUnfoldedEvents(pulledDeltaLogs, baseWatermark); + return {shouldCompact: unfoldedCount > threshold, unfoldedCount}; +} + +/** + * The highest SELF seq that the NEW base has folded — i.e. how far this device may safely + * truncate its OWN local log. It is the advanced watermark for `selfDeviceId` in the resolved + * snapshot (the new base), which `replay()` set to the max applied self `seq`. We additionally + * CLAMP it to `lastPushedSeq`: the local log may hold freshly-captured events whose seq is + * higher than anything that reached the cloud / the resolved snapshot this cycle, and those + * are NOT yet in the base — truncating them would lose them. Taking the MIN is the conservative + * choice ("never truncate an event whose effect isn't in the new base"): we trim only events + * that are BOTH folded into the new base AND already pushed. + * + * @param {object} newWatermark - the resolved snapshot's advanced watermark `{deviceId: seq}`. + * @param {string} selfDeviceId + * @param {number} [lastPushedSeq=0] - highest self seq already pushed to the cloud. + * @returns {number} the seq to pass to DeltaLog.clearUpTo (0 ⇒ nothing to truncate). + */ +export function selfFoldedSeq(newWatermark, selfDeviceId, lastPushedSeq = 0) { + const foldedSelf = (newWatermark || {})[selfDeviceId] ?? 0; + const pushed = Number.isFinite(lastPushedSeq) ? lastPushedSeq : 0; + return Math.min(foldedSelf, pushed); +} + +/** + * From the self events the planner would write (`deltaFileToWrite.events`, the FULL self log + * in portable form), keep only those with `seq` STRICTLY GREATER than `foldedSeq` — i.e. the + * events NOT yet folded into the new base. This is the cloud-side counterpart to the local + * {@link module:sync/delta/delta-log.clearUpTo}: after compaction the self delta file holds + * only the still-unfolded tail, so the gist log stops growing. Events with a null seq are + * KEPT (conservative: an un-sequenced event can't be proven folded). + * + * @param {object[]} selfEvents - the full self events the non-compacting path would write. + * @param {number} foldedSeq - from {@link selfFoldedSeq}. + * @returns {object[]} the retained (still-unfolded) self events, in order. + */ +export function truncateSelfEvents(selfEvents, foldedSeq) { + return (selfEvents || []).filter(event => event.seq == null || event.seq > foldedSeq); +} + +/** + * DEFERRED SELF-TRUNCATION reconciliation (Part C, the data-loss backstop). + * + * ## Why truncation is deferred + * The snapshot is the ONLY home of folded history — per-device delta files are truncated + * TAILS, not a full replay source. If a compaction cycle truncated its own log (local + * `clearUpTo` + a truncated cloud self-delta) in the SAME cycle it wrote the snapshot, and a + * peer then CLOBBERED that just-written snapshot with an older one (the advisory lock makes + * this rare but, lacking conditional writes, not impossible; a crash / expired lock can also + * allow it), the just-truncated events would live ONLY in the clobbered snapshot ⇒ PERMANENT + * LOSS. So a compaction cycle does NOT truncate; it records a pending marker = the self + * watermark seq it folded into the snapshot it wrote, and keeps the full self log in BOTH the + * local log and the cloud self-delta. + * + * ## The invariant + * An event ALWAYS lives in either (a) a cloud delta file, or (b) a cloud snapshot whose + * durability we have CONFIRMED by re-reading. We never truncate the only copy until confirmed. + * + * ## This function — the confirmation, run on a SUBSEQUENT cycle after pulling the snapshot + * Given the pending marker and the PULLED cloud snapshot's self watermark, decide whether the + * deferred truncation is now safe. It is safe iff the cloud snapshot durably carries those + * folded events: `cloudSelfWatermark >= pendingTruncateSeq` (our snapshot survived, or a later + * one supersedes it). Then truncate up to `pendingTruncateSeq` (local `clearUpTo` + drop + * seq <= it from the cloud self-delta) and clear the marker. If the snapshot was CLOBBERED + * (watermark rolled back below the marker), it is NOT safe: the events are still in the cloud + * self-delta (we deferred), so they get re-folded; leave the marker so it resolves once the + * cloud snapshot catches up. + * + * @param {?number} pendingTruncateSeq - the persisted pending marker (0/null ⇒ nothing pending). + * @param {object} [cloudSnapshotWatermark] - the PULLED snapshot's watermark `{deviceId: seq}`. + * @param {string} selfDeviceId + * @returns {{confirmed: boolean, truncateSeq: number}} `confirmed` true iff it is now safe to + * truncate; `truncateSeq` is the seq to truncate up to (only meaningful when confirmed). + */ +export function resolveDeferredTruncation(pendingTruncateSeq, cloudSnapshotWatermark = {}, selfDeviceId) { + const pending = Number(pendingTruncateSeq); + if (!Number.isFinite(pending) || pending <= 0) { + return {confirmed: false, truncateSeq: 0}; // nothing pending + } + const cloudSelfWatermark = Number((cloudSnapshotWatermark || {})[selfDeviceId]) || 0; + if (cloudSelfWatermark >= pending) { + return {confirmed: true, truncateSeq: pending}; // snapshot durably carries the folded events + } + return {confirmed: false, truncateSeq: 0}; // clobbered / not yet durable ⇒ keep deferring +} + +/** + * Is a single device's delta log FULLY FOLDED into the base — i.e. is EVERY event in it + * already represented in the snapshot (so the file may be safely deleted)? + * + * The predicate is the EXACT per-event inverse of {@link countUnfoldedEvents} / replay's + * dedup skip test (`event.seq != null && event.seq <= folded`, see replay.js): an event is + * folded iff it has a numeric `seq` that is `<= watermark[device]`. A log is fully folded + * iff EVERY event is folded. Equivalently it is NOT fully folded if ANY event is unfolded + * (`seq == null || seq > folded`) — a `null`/missing seq is treated as UNFOLDED (it can + * never be proven to be in the base, exactly as replay never skips it), so a file holding + * one is NEVER deletable. An EMPTY log counts as fully folded (no event is outside the base). + * + * @param {object[]} events - the device's delta events (as pulled). + * @param {number} [folded=0] - that device's watermark in the AUTHORITATIVE snapshot. + * @returns {boolean} true iff every event has a numeric seq <= folded. + */ +export function isLogFullyFolded(events, folded = 0) { + const wm = Number.isFinite(folded) ? folded : 0; + for (const event of events || []) { + if (event.seq == null || event.seq > wm) { + return false; // an unfolded event ⇒ NOT safe to delete (would lose data) + } + } + return true; +} + +/** + * Pure orphan-GC policy: from the pulled per-device delta files, select the names of files + * SAFE to delete because they are FULLY FOLDED into the AUTHORITATIVE base — every one of + * their events has `seq <= watermark[device]`, so the base already carries their effect and + * replay would skip them all (`seq <= watermark`, see replay.js). Deleting such a file loses + * NOTHING, and a device that later returns and re-pushes its full local log stays safe: those + * events (`seq <= watermark`) are skipped by replay's dedup — no double-apply, no + * resurrection. This relies on the watermark ENTRY being kept forever; this function NEVER + * mutates the watermark (it only reads it), so that invariant is preserved by construction. + * + * STRICT SAFETY (the task's rules 1-6): + * - The CURRENT device's own file is NEVER selected (rule 1): it manages its own log via + * own-log truncation. Matched by deviceId. + * - A file is selected ONLY when {@link isLogFullyFolded} holds against the AUTHORITATIVE + * watermark passed in (rules 2 & 4) — ANY unfolded event keeps the file. + * - Watermark ENTRIES are not touched (rule 3): this returns names, never mutates `watermark`. + * - Bias to KEEP (rule 6): a file with no resolvable name, a null deviceId, or any doubt is + * skipped (never returned for deletion). + * + * @param {Array<{name?: string, deviceId: string, events: object[]}>} pulledDeltaLogs - the + * per-device logs as pulled, each carrying the gist file `name` it was read from. + * @param {object} [watermark] - the AUTHORITATIVE (resolved/snapshot) watermark `{deviceId: seq}`. + * @param {?string} [selfDeviceId] - the current device; its own file is never selected. + * @returns {string[]} gist file names safe to delete (possibly empty), in input order. + */ +export function selectOrphanDeltaFilesToDelete(pulledDeltaLogs, watermark = {}, selfDeviceId = null) { + const wm = watermark || {}; + const toDelete = []; + + for (const log of pulledDeltaLogs || []) { + const deviceId = log?.deviceId; + const name = log?.name; + + // bias to keep: no usable file name, or no device identity, or it's our own file. + if (!name || deviceId == null || deviceId === selfDeviceId) { + continue; + } + + const folded = wm[deviceId] ?? 0; + if (isLogFullyFolded(log?.events, folded)) { + toDelete.push(name); + } + } + + return toDelete; +} diff --git a/addon/src/js/sync/delta/container-map.js b/addon/src/js/sync/delta/container-map.js new file mode 100644 index 00000000..1af81ae2 --- /dev/null +++ b/addon/src/js/sync/delta/container-map.js @@ -0,0 +1,279 @@ + +/** + * Portable container identity mapping for delta sync (Phase P3c — container parity). + * + * Firefox contextual identities ("containers" / Multi-Account Containers) are referenced + * everywhere by their `cookieStoreId` (e.g. `firefox-container-1`). That id is assigned + * per-install and is NOT stable across machines: the SAME logical "Work" container is + * `firefox-container-1` on one PC and `firefox-container-3` on another. Syncing the raw + * id would land a tab in the wrong (or a non-existent) container on the receiving side. + * + * So the synced model never carries a raw `cookieStoreId`. Instead every container- + * referencing field carries a PORTABLE KEY derived from the container's identity + * (`name + color + icon`, via {@link stringifyContainer} — exactly the legacy + * `cloud.js` `stringifyContainer`), and a per-snapshot registry maps that key back to + * the `{name, color, icon}` needed to find-or-create the matching local container on the + * receiving device. This is a direct port of the legacy `cloud.js` `syncContainers` / + * `mapContainers` / `mapDataContainers` / `eachGroupContainerKeyMap` approach into the + * delta model. + * + * Two reserved markers are NEVER stored as real containers in the registry: + * - {@link DEFAULT_MARKER} — the default (no) container; translated to the local + * default `cookieStoreId` on the receiving side (whose literal value can differ, + * e.g. `icecat-default` vs `firefox-default`). + * - {@link TEMPORARY_MARKER} — a temporary container; never a stable identity, so it is + * never recreated as a named container (the receiving side maps it to its own + * temporary/default per its `mapToLocal` callback). + * + * ## Purity (hard requirement) + * This module is PURE: no `browser.*`, no `constants.js` import. The two reserved + * marker literals mirror `Constants.DEFAULT_COOKIE_STORE_ID_FIREFOX` / + * `Constants.TEMPORARY_CONTAINER` but are local literals so the engine and its tests run + * under plain `node`. The IMPURE boundary (resolving a portable key to a real local + * `cookieStoreId`, find-or-creating the container) lives in `delta-sync.js`, which passes + * its resolver in as a callback. Inputs are mutated IN PLACE by the mappers (the caller + * owns fresh clones — `buildLocalState` / `deepClone`d events / `browserOps`), matching + * the legacy `eachGroupContainerKeyMap` contract. + * + * @module sync/delta/container-map + */ + +/** + * Reserved portable key for the DEFAULT (no) container. Mirrors + * `Constants.DEFAULT_COOKIE_STORE_ID_FIREFOX`; a literal here keeps the module pure. + * Use the canonical firefox value (not the per-browser `DEFAULT_COOKIE_STORE_ID`) so the + * marker is identical across IceCat/Firefox installs — the local literal value is + * substituted back only at the inbound boundary. + * @readonly + */ +export const DEFAULT_MARKER = 'firefox-default'; + +/** + * Reserved portable key for a TEMPORARY container. Mirrors `Constants.TEMPORARY_CONTAINER`. + * Never stored in the registry as a real container (temporary containers have no stable + * identity to recreate). + * @readonly + */ +export const TEMPORARY_MARKER = 'temporary-container'; + +/** + * Group (and `defaultGroupProps`) keys that reference a container. `newTabContainer` is a + * single `cookieStoreId`; `catchTabContainers` / `excludeContainersForReOpen` are arrays + * of them. Mirrors the legacy `cloud.js` `GROUP_CONTAINER_KEYS`. + * @readonly + */ +export const GROUP_CONTAINER_KEYS = Object.freeze([ + 'newTabContainer', + 'catchTabContainers', + 'excludeContainersForReOpen', +]); + +/** + * Build the portable key for a container identity — `name + color + icon` joined. A direct + * port of the legacy `cloud.js` `stringifyContainer`. Reserved markers (default/temporary) + * are passed through untouched so this is safe to call on an already-portable value. + * @param {{name?: string, color?: string, icon?: string}} container + * @returns {string} + */ +export function stringifyContainer({name, color, icon} = {}) { + return [name, color, icon].join(''); +} + +/** + * Walk every container-referencing field of a single group-shaped object (a group OR + * `defaultGroupProps`) and replace each `cookieStoreId` via `mapFn`. Mirrors the legacy + * `eachGroupContainerKeyMap`: a scalar field stays scalar, an array field stays an array. + * The group's `tabs` (when present) have each tab's `cookieStoreId` mapped too. Mutates + * `group` in place; absent fields are left untouched. + * + * @param {object} group - a group or defaultGroupProps object. + * @param {(cookieStoreId: string) => string} mapFn - portable<->local translator. + */ +export function mapGroupContainers(group, mapFn) { + if (!group || typeof group !== 'object') { + return; + } + + for (const key of GROUP_CONTAINER_KEYS) { + if (!Object.prototype.hasOwnProperty.call(group, key)) { + continue; + } + const value = group[key]; + if (Array.isArray(value)) { + group[key] = value.map(csId => mapFn(csId)); + } else if (value != null) { + group[key] = mapFn(value); + } + } + + if (Array.isArray(group.tabs)) { + for (const tab of group.tabs) { + if (tab && tab.cookieStoreId != null) { + tab.cookieStoreId = mapFn(tab.cookieStoreId); + } + } + } +} + +/** + * Map the container field(s) of a single delta EVENT in place (matches the event payload + * shapes in `replay.js`): `tab.add/modify` carry `event.tab.cookieStoreId`; `pinned.*` + * carry `event.tab.cookieStoreId`; `group.add/modify` carry `event.group` (a full group + * record with the group container keys, mapped via {@link mapGroupContainers}); and an + * `option.set` for `defaultGroupProps` carries its container keys in `event.value`. + * + * @param {object} event - a delta event (`{op, ...}`). + * @param {(cookieStoreId: string) => string} mapFn + */ +export function mapEventContainers(event, mapFn) { + if (!event || typeof event !== 'object') { + return; + } + + if (event.tab && event.tab.cookieStoreId != null) { + event.tab.cookieStoreId = mapFn(event.tab.cookieStoreId); + } + + if (event.group) { + mapGroupContainers(event.group, mapFn); + } + + // option.set for defaultGroupProps carries container keys inside its value. + if (event.op === 'option.set' && event.key === 'defaultGroupProps' && event.value) { + mapGroupContainers(event.value, mapFn); + } +} + +/** + * Map every container field across a whole snapshot-shaped state in place: each group + * (via {@link mapGroupContainers}, which also covers its tabs), each global pinned tab, + * and `options.defaultGroupProps` when present. Used for both the localState (outbound) + * and the resolved snapshot's container fields. + * + * @param {object} state - `{groups?, pinnedTabs?, options?, defaultGroupProps?}`. + * @param {(cookieStoreId: string) => string} mapFn + */ +export function mapStateContainers(state, mapFn) { + if (!state || typeof state !== 'object') { + return; + } + + for (const group of Array.isArray(state.groups) ? state.groups : []) { + mapGroupContainers(group, mapFn); + } + + for (const tab of Array.isArray(state.pinnedTabs) ? state.pinnedTabs : []) { + if (tab && tab.cookieStoreId != null) { + tab.cookieStoreId = mapFn(tab.cookieStoreId); + } + } + + // defaultGroupProps roams as a synced option value (a group-shaped object). + if (state.options && state.options.defaultGroupProps) { + mapGroupContainers(state.options.defaultGroupProps, mapFn); + } + // also support a top-level defaultGroupProps (e.g. an options bag passed directly). + if (state.defaultGroupProps) { + mapGroupContainers(state.defaultGroupProps, mapFn); + } +} + +/** + * Make the OUTBOUND local→portable translator for a given set of local containers and a + * registry to populate. Returns a `mapFn(cookieStoreId) => portableKey` that: + * - default container ⇒ {@link DEFAULT_MARKER}; + * - temporary container ⇒ {@link TEMPORARY_MARKER} (never registered as a real container); + * - any other known local container ⇒ its {@link stringifyContainer} key, AND records its + * `{name, color, icon}` into `registry[key]` so the receiving side can recreate it; + * - an unknown `cookieStoreId` (no local definition — e.g. a container removed out from + * under us) ⇒ {@link DEFAULT_MARKER} (conservative: never fail the sync, fall back to + * default — mirrors the legacy code's missing-definition handling). + * + * @param {object} localContainers - `{[cookieStoreId]: {name, color, icon}}` (real, non-temp). + * @param {object} registry - mutated in place: `{[portableKey]: {name, color, icon}}`. + * @param {(cookieStoreId: string) => boolean} isDefault - default-container predicate. + * @param {(cookieStoreId: string) => boolean} isTemporary - temporary-container predicate. + * @returns {(cookieStoreId: string) => string} + */ +export function makeOutboundMapper(localContainers, registry, isDefault, isTemporary) { + return cookieStoreId => { + if (cookieStoreId == null || isDefault(cookieStoreId)) { + return DEFAULT_MARKER; + } + if (isTemporary(cookieStoreId)) { + return TEMPORARY_MARKER; + } + + const container = localContainers[cookieStoreId]; + if (!container) { + // unknown / missing definition ⇒ fall back to default (never fail the sync). + return DEFAULT_MARKER; + } + + const key = stringifyContainer(container); + if (!registry[key]) { + registry[key] = { + name: container.name, + color: container.color, + icon: container.icon, + }; + } + return key; + }; +} + +/** + * Make the INBOUND portable→local translator for a given registry. Returns a + * `mapFn(portableKey) => cookieStoreId` that: + * - {@link DEFAULT_MARKER} (or null) ⇒ the local default `cookieStoreId` (`localDefault`); + * - {@link TEMPORARY_MARKER} ⇒ resolved by `resolveTemporary()` (caller decides: a fresh + * temporary container, or the local default — conservative); + * - a registered portable key ⇒ `findOrCreate({name, color, icon})`'s local cookieStoreId, + * CACHED per round so a container is created at most once; + * - an unknown key with no registry entry ⇒ the local default (missing definition ⇒ + * fall back to default; never fail the sync — mirrors the legacy code). + * + * `findOrCreate` and `resolveTemporary` are the IMPURE injection points (they touch + * `Containers.*`); this function itself stays pure aside from invoking them. + * + * @param {object} registry - `{[portableKey]: {name, color, icon}}` from the snapshot. + * @param {string} localDefault - this install's default `cookieStoreId`. + * @param {(identity: {name, color, icon}) => string} findOrCreate - resolve a portable + * identity to a local cookieStoreId (find existing match or create). + * @param {() => string} resolveTemporary - resolve the temporary marker to a local id. + * @returns {(portableKey: string) => string} + */ +export function makeInboundMapper(registry, localDefault, findOrCreate, resolveTemporary) { + const cache = new Map(); + + return portableKey => { + if (portableKey == null || portableKey === DEFAULT_MARKER) { + return localDefault; + } + if (portableKey === TEMPORARY_MARKER) { + if (!cache.has(portableKey)) { + cache.set(portableKey, resolveTemporary()); + } + return cache.get(portableKey); + } + + if (cache.has(portableKey)) { + return cache.get(portableKey); + } + + const identity = registry && registry[portableKey]; + if (!identity) { + // missing definition ⇒ fall back to default (never fail the sync). + cache.set(portableKey, localDefault); + return localDefault; + } + + const cookieStoreId = findOrCreate({ + name: identity.name, + color: identity.color, + icon: identity.icon, + }); + cache.set(portableKey, cookieStoreId); + return cookieStoreId; + }; +} diff --git a/addon/src/js/sync/delta/delta-capture.js b/addon/src/js/sync/delta/delta-capture.js new file mode 100644 index 00000000..7d418658 --- /dev/null +++ b/addon/src/js/sync/delta/delta-capture.js @@ -0,0 +1,743 @@ + +/** + * Capture layer: translates STG's existing tab/group lifecycle into delta events + * and appends them to the local {@link module:sync/delta/delta-log} (Phase P1). + * + * Design constraints (see `.project/DESIGN_DELTA_SYNC.md`): + * - HOOK existing handlers; never add raw `browser.tabs.on*` listeners. The tab + * functions here are invoked from `tabs.js` handlers AFTER STG's own skip checks + * (`skip.created/tracking/removed`, `skipTrackingWindows`), and the group + * functions from groups.js `send*` helpers. Sync-originated changes bypass both + * (cloud.js creates tabs with `skipListener`/`skip.*` and saves groups via + * `Groups.save` directly, not `Groups.add/update/remove`), so they are NOT + * recorded as fresh deltas. See "Sync-origination guard" note below. + * - Tab identity is the stable `uid` (cache.js). GROUPED tabs (a groupId in cache) fold + * into `groups`; GLOBAL PINNED tabs (window-global, no groupId) fold into the separate + * `pinnedTabs` snapshot section via the `pinned.*` ops below. Untracked tabs that are + * neither (e.g. transient unsync tabs) are still ignored, mirroring backups. The + * groupId is the discriminator: a per-group pinned tab (one that HAS a groupId) is NOT + * a global pinned tab and is never captured here — only via the group `tab.*` ops. + * - modify/add carry the FULL record so a future resurrection on replay is faithful. + * - tab/pinned capture gates URL on the SYNCABLE allow-list ({@link module:sync/delta/url-sync} + * `isUrlSyncable`): everything `Utils.isUrlAllowToCreate` admits PLUS non-trivial + * `about:` URLs (which the receiving machine renders as the "unsupported URL" stub), + * minus trivial new-tab states. A tab whose LIVE url is that stub page is decoded back + * to its embedded original (`unwrapStubUrl`) so it keeps its original identity. + * + * Inert in P1: nothing reads these events yet; failures are swallowed (logged) so + * capture can never affect current behaviour. + * + * ### Sync-origination guard + * P1 relies on STG's existing skip flags: the tab handlers return early for any tab + * STG is itself manipulating during sync, so this layer is never invoked for those. + * P3b (the delta transport, `delta-sync.js`) routes its applies through those same + * skip flags — BUT not every capture entry point sits behind one: notably the + * `onUpdated` url/title `tab.modify` path fires when a freshly sync-created tab loads + * its url, which no skip flag covers. To close that residual feedback loop, the + * transport brackets its whole apply with {@link beginApply}/{@link endApply}, which + * sets a module-level flag so EVERY capture function below early-returns while a sync + * apply is in progress. The flag is a depth counter so nested/overlapping applies are + * safe, and it is reset defensively by `endApply` even on error. + * + * @module sync/delta/delta-capture + */ + +import Logger from '/js/logger.js'; +import * as Cache from '/js/cache.js'; +import * as Constants from '/js/constants.js'; +import * as DeltaLog from './delta-log.js'; +import {syncedOptionKeys} from './option-keys.js'; +import {isUrlSyncable, unwrapStubUrl, sanitizeFavIconUrl} from './url-sync.js'; +import {computeGroupRelativeIndex} from './group-relative-index.js'; +import {isAppliedNavigationEcho} from './applied-nav-echo.js'; + +const logger = new Logger('DeltaCapture'); + +// Depth of in-progress sync applies. While > 0, every capture function early-returns +// so transport-applied changes are never recorded as fresh local deltas (feedback +// loop). A counter (not a bool) tolerates overlapping begin/end pairs. +let applyDepth = 0; + +/** Suppress capture while a sync apply runs. Pair with {@link endApply}. */ +export function beginApply() { + applyDepth++; +} + +/** End the suppression started by {@link beginApply}. Never drops below zero. */ +export function endApply() { + if (applyDepth > 0) { + applyDepth--; + if (applyDepth === 0) { + // remember when the OUTERMOST apply finished: the navigation it just issued via + // `browser.tabs.update` settles ASYNCHRONOUSLY, so its onUpdated echo arrives AFTER + // this point and must still be recognised as an apply echo (see the guard below). + lastApplyEndedAt = Date.now(); + } + } +} + +/** @returns {boolean} true while a sync apply is suppressing capture. */ +export function isApplying() { + return applyDepth > 0; +} + +// --------------------------------------------------------------------------- +// Post-apply navigation-echo guard (feedback-loop protection for sync-APPLIED url changes). +// +// The synchronous {@link isApplying} counter only suppresses capture for changes that land +// WHILE an apply runs. But the transport's content-update path +// (`delta-sync.applyTabContentUpdate`) navigates a LOADED tab via +// `browser.tabs.update(liveId, {url})`, which resolves as soon as the navigation STARTS — the +// resulting `onUpdated` (status/url) fires ASYNCHRONOUSLY, and `tabs.js onUpdated` itself +// `await`s ~70ms before reaching the capture fn. By then `endApply()` has already run, so the +// applied navigation surfaces as a "fresh" url `tab.modify`/`pinned.modify`. +// +// URL-NARROWED suppression (the convergence fix for "loads infinitely"): the mark records the +// EXACT url the apply navigated the tab to. While the mark is live: +// · a settle whose url EQUALS the applied url is the plain echo of our own write ⇒ SUPPRESS +// (re-capturing it would push it back and grow the log for no reason); +// · a settle whose url DIFFERS from the applied url is a server REDIRECT (applied X → landed on +// Y: http→https, trailing slash, login bounce, SPA fragment) ⇒ CAPTURE + push. This is the +// crucial change from the old window-based guard, which dropped the redirect too: when the +// redirect is dropped, the cloud stays at X while live is Y forever, so the planner re-emits +// `tabsToUpdate{url:X}` and the apply re-navigates X→Y EVERY cycle (the perpetual spinner). +// Capturing Y lets the cloud CONVERGE to Y; then resolved==live and the planner emits nothing. +// (The no-op guard in `applyTabContentUpdate` is the complementary half: once converged, an +// update whose url already matches the live url issues ZERO browser ops.) +// +// SCOPING — echo vs USER navigation: a tab is armed ONLY when its content change is observed +// WHILE a sync apply is in progress, OR within a tight trailing window after the apply ended +// (`APPLIED_NAV_WINDOW_MS`). A user navigation of a grouped/pinned tab made OUTSIDE that causal +// window is never marked and syncs normally (the whole point of the A6 url-capture fix). tabs.js +// arms the mark via {@link markAppliedNavigation} when {@link shouldArmAppliedNavigation} holds; +// capture suppresses only the exact-url echo while the mark is live. The synchronous `isApplying()` +// in-apply suppression is left fully intact. +const appliedNavTabs = new Map(); // tabId -> {expiry: epoch ms, url: applied url} (suppression mark) +const APPLIED_NAV_WINDOW_MS = 4_000; // trailing causal window after endApply for the async echo +let lastApplyEndedAt = 0; // epoch ms when the outermost apply last finished + +/** + * Should a content change observed RIGHT NOW (in `tabs.js onUpdated`) be armed as a sync-applied + * navigation? True while an apply is in progress, or within the trailing causal window after the + * last apply ended (covering the async `onUpdated` that fires after `endApply`). When this holds, + * tabs.js calls {@link markAppliedNavigation} so the echo of the exact applied url is suppressed + * (a redirect to a different url is still captured — see the module guard note above). + * @returns {boolean} + */ +export function shouldArmAppliedNavigation() { + return isApplying() || (Date.now() - lastApplyEndedAt) < APPLIED_NAV_WINDOW_MS; +} + +/** + * Mark a tab id as having just been navigated by a sync apply (or as having a content change + * observed inside the apply's causal window), recording the APPLIED url so the capture layer can + * suppress the echo of EXACTLY that url while still letting a server-redirect to a DIFFERENT url + * through (so the cloud converges to the redirect target). No-op for a non-finite id. + * + * The url is captured ONCE — on the first arm of a fresh mark — and PRESERVED across subsequent + * arms within the window: a redirect-chain hop re-arms the mark (refreshing its expiry so the + * whole chain stays inside one causal window) but must NOT overwrite the recorded applied url + * with the redirect's url, or the redirect would compare equal to itself and be suppressed + * instead of captured. The url is decoded through STG's "unsupported URL" stub so a stub-rendered + * about: tab is marked by its embedded original identity (matching how the observed url is + * compared + how the record url is stored). + * + * @param {number} tabId + * @param {string} [url] - the url the apply navigated this tab to (recorded only on first arm). + */ +export function markAppliedNavigation(tabId, url) { + if (!Number.isFinite(tabId)) { + return; + } + const existing = appliedNavTabs.get(tabId); + const expiry = Date.now() + APPLIED_NAV_WINDOW_MS; + // preserve the FIRST recorded url across re-arms (redirect hops keep the applied url). + const markUrl = (existing && existing.url != null) + ? existing.url + : (typeof url === 'string' ? unwrapStubUrl(url) : undefined); + appliedNavTabs.set(tabId, {expiry, url: markUrl}); +} + +/** Drop an applied-navigation mark (e.g. on tab removal, or once consumed). */ +export function clearAppliedNavigation(tabId) { + appliedNavTabs.delete(tabId); +} + +/** + * Is the content change for `tabId` (now at `observedUrl`) a sync-applied navigation echo right + * now? Reads the live {@link isApplying} state and the tab's mark, delegating to + * {@link isAppliedNavigationEcho}. + * + * Suppression is NARROWED BY URL (the convergence fix): while the mark is live, ONLY the echo of + * the EXACT applied url is dropped; a settle whose url DIFFERS from the applied url is a server + * REDIRECT and is CAPTURED so the cloud can converge to the redirect target (and then live==cloud, + * ending the perpetual re-navigation). The applied url and the observed url are both compared + * stub-decoded so a stub-rendered about: tab matches by its embedded identity. An EXPIRED mark is + * pruned (returns false) so the next user navigation of the same tab is captured normally. + * + * @param {number} tabId + * @param {string} [observedUrl] - the url of the content change being classified. + * @returns {boolean} + */ +function consumeAppliedNavigationEcho(tabId, observedUrl) { + const mark = appliedNavTabs.get(tabId); + const now = Date.now(); + const echo = isAppliedNavigationEcho({ + applying: isApplying(), + markExpiry: mark?.expiry, + markUrl: mark?.url, + observedUrl: typeof observedUrl === 'string' ? unwrapStubUrl(observedUrl) : observedUrl, + now, + }); + if (mark != null && now >= mark.expiry) { + appliedNavTabs.delete(tabId); // prune the stale mark so later user navs aren't blocked. + } + return echo; +} + +/** + * Resolve the stable uid for a tracked tab, assigning + persisting one if absent + * (mirrors cache.js lazy backfill). Returns null if it can't be resolved. + * @param {number} tabId + * @returns {Promise} + */ +async function resolveUid(tabId) { + const uid = Cache.getTabUid(tabId); + if (uid) { + return uid; + } + try { + return await Cache.setTabUid(tabId); + } catch { + return null; + } +} + +/** + * Compute a tab's GROUP-RELATIVE index: its 0-based position among the tabs of the + * SAME group within the same window, ordered by browser index. + * + * The delta/replay model treats a tab's `index` as the position WITHIN its group's + * ordered tab list (0..n-1), NOT the browser-window-absolute `tab.index`. The browser + * index is shifted by pinned tabs and by other groups' tabs sharing the window, and it + * differs per machine — replaying by it shuffles the group's tab order on every other + * device. So at capture time we derive the position within the group instead. + * + * Cheap and side-effect-light: one `browser.tabs.query` for the tab's window, filtered + * to the same group via the cache. Returns null (⇒ caller omits `index` ⇒ replay + * appends) if the position can't be determined — better an append than a wrong slot. The + * pure positional math lives in {@link computeGroupRelativeIndex} (unit-tested). + * + * @param {number} tabId + * @param {number} windowId + * @param {string} groupId + * @returns {Promise} + */ +async function getGroupRelativeIndex(tabId, windowId, groupId) { + try { + if (!Number.isFinite(windowId) || !groupId) { + return null; + } + + const windowTabs = await browser.tabs.query({windowId}); + + return computeGroupRelativeIndex(windowTabs, Cache.getTabGroup, tabId, groupId); + } catch { + return null; + } +} + +/** + * Build the full tab record an add/modify event carries. + * + * `index` is GROUP-RELATIVE (0-based position within the group's ordered tab list), + * resolved from the live browser via {@link getGroupRelativeIndex}. When it can't be + * resolved the field is OMITTED, which the replay engine treats as append-at-end. + * + * @param {object} tab - browser tab merged with cache session. + * @param {string} uid + * @param {number|null} groupRelativeIndex - precomputed group-relative position, or null. + * @param {object} [snapshot] - synchronously-read cache fields pinned at the call site (A4), + * used instead of re-reading the (possibly torn-down) cache. Recognised keys: + * `lastModified`, `groupPinned`, `favIconUrl`. + * @returns {object} + */ +function buildTabRecord(tab, uid, groupRelativeIndex, snapshot = null) { + const record = { + uid, + // decode STG's "unsupported URL" stub page back to the original about: url it + // embeds, so a tab the receiving machine rendered as the stub keeps its original + // identity instead of syncing the moz-extension stub url (feedback/divergence loop). + url: unwrapStubUrl(tab.url), + title: tab.title, + cookieStoreId: tab.cookieStoreId, + // KEEP the favicon (incl. small `data:` PNGs) so the synced/sleeping tab shows its + // icon. sanitizeFavIconUrl only drops a PATHOLOGICALLY large favicon (>~50 KB). The + // favicon is just a field that RIDES ALONG in this record (written for a real url/ + // title change); there is no favicon-only event, so it can never be duplicated across + // hundreds of thousands of events again (the 5 GB bloat). undefined ⇒ field omitted. + // A4: prefer the live tab favicon, fall back to the snapshot taken at the call site. + favIconUrl: sanitizeFavIconUrl(tab.favIconUrl ?? snapshot?.favIconUrl), + // A4: prefer the snapshot's lastModified (pinned at the call site, after the bump); + // fall back to a live cache read for callers that pass no snapshot. + lastModified: snapshot?.lastModified ?? Cache.getTabLastModified(tab.id), + }; + if (Number.isInteger(groupRelativeIndex)) { + record.index = groupRelativeIndex; + } + // group-scoped pin flag: a tab pinned WITHIN its group (pinned only while the group + // is active). Rides the group tab.add/tab.modify record (NEVER a global pinned.* op — + // a group-pinned tab still has a groupId, so the pinned.* discriminator excludes it). + // ALWAYS emitted explicitly (true or false): replay clobber-safety preserves an OMITTED + // flag (legacy records carried it only when true), so emitting `false` is what lets a + // genuine un-pin actually clear a previously-synced pinned:true. See replay.js. + // A4: prefer the snapshot's value (pinned at the call site) over a live cache read. + record.pinned = (snapshot ? snapshot.groupPinned : Cache.getTabGroupPinned(tab.id)) === true; + // source loaded-state: was this tab loaded (NOT discarded) on THIS machine when + // captured? Carried so a receiving device with `syncActivatePreviouslyActiveTabs` + // on can re-activate the tabs the user had open here. ALWAYS emitted explicitly so a + // genuine un-load clears it (same reasoning as `pinned`). A tab whose discarded state + // is unknown (undefined) is treated as not-loaded (safe: replays as asleep). + record.loaded = tab.discarded === false; + return record; +} + +/** + * Record a tab addition (op `tab.add`). Called from tabs.js `onCreated` after skip + * checks. Ignores pinned / ungrouped tabs (no groupId ⇒ STG doesn't track it). + * @param {object} tab + */ +export async function tabAdded(tab) { + try { + if (isApplying()) { + return; + } + + const groupId = Cache.getTabGroup(tab.id); + if (!groupId) { + return; + } + + // gate on the SYNCABLE allow-list (the unwrapped url so a stub-rendered tab is + // judged by its original about: url). Non-syncable urls (about:blank/newtab/…) + // are noise and never enter the log. + if (!isUrlSyncable(unwrapStubUrl(tab.url))) { + return; + } + + const uid = await resolveUid(tab.id); + if (!uid) { + return; + } + + const index = await getGroupRelativeIndex(tab.id, tab.windowId, groupId); + + await DeltaLog.append(DeltaLog.OPS.TAB_ADD, { + groupId, + tab: buildTabRecord(tab, uid, index), + }); + } catch (e) { + logger.onCatch('tabAdded', false)(e); + } +} + +/** + * Record a tab content change (op `tab.modify`) - url/title/favIcon. Called from + * tabs.js `onUpdated` after skip checks. Ignores ungrouped tabs. + * @param {object} tab + * @param {object} [snapshot] - A4: cache fields read SYNCHRONOUSLY at the call site + * (`{groupId, uid, lastModified, groupPinned, favIconUrl}`), pinned before this + * fire-and-forget fn awaits. Used in place of re-reading the cache, which may be torn + * down if the tab was removed in the meantime → garbled/empty record. Omitting it keeps + * the legacy live-read behaviour. + */ +export async function tabModified(tab, snapshot = null) { + try { + if (isApplying()) { + return; + } + + // suppress the async echo of a sync-APPLIED navigation: a tab the apply just navigated is + // marked with the applied url (see markAppliedNavigation); a settle whose url MATCHES that + // applied url is dropped so it is not re-captured and pushed back (perpetual churn). A + // server REDIRECT to a DIFFERENT url is intentionally let through so the cloud converges to + // the redirect target. A genuine later user navigation is NOT marked → still syncs. + if (consumeAppliedNavigationEcho(tab.id, tab.url)) { + return; + } + + const groupId = snapshot?.groupId ?? Cache.getTabGroup(tab.id); + if (!groupId) { + return; + } + + // see tabAdded: gate on the syncable allow-list of the unwrapped url. + if (!isUrlSyncable(unwrapStubUrl(tab.url))) { + return; + } + + // prefer the uid pinned at the call site; fall back to resolve/mint if absent. + const uid = snapshot?.uid || await resolveUid(tab.id); + if (!uid) { + return; + } + + const index = await getGroupRelativeIndex(tab.id, tab.windowId, groupId); + + await DeltaLog.append(DeltaLog.OPS.TAB_MODIFY, { + groupId, + tab: buildTabRecord(tab, uid, index, snapshot), + }); + } catch (e) { + logger.onCatch('tabModified', false)(e); + } +} + +/** + * Record a tab move (op `tab.move`). Called from tabs.js `onMoved`/`onAttached` + * after skip checks. Carries uid + the GROUP-RELATIVE target index. + * + * The browser `toIndex` passed by the handler is the window-absolute destination, + * which is meaningless to the other devices (see {@link getGroupRelativeIndex}). The + * move has already landed by the time the handler fires, so we re-derive the tab's + * 0-based position WITHIN its group from the live browser. If it can't be resolved we + * OMIT `toIndex`, which replay treats as append-at-end (better than a wrong slot). + * + * @param {number} tabId + * @param {number} [toIndex] - browser window-absolute index (informational only). + */ +export async function tabMoved(tabId, toIndex) { + void toIndex; + try { + if (isApplying()) { + return; + } + + const groupId = Cache.getTabGroup(tabId); + if (!groupId) { + return; + } + + const uid = await resolveUid(tabId); + if (!uid) { + return; + } + + const windowId = Cache.getWindowId(groupId); + const groupRelativeIndex = await getGroupRelativeIndex(tabId, windowId, groupId); + + const payload = {groupId, uid}; + if (Number.isInteger(groupRelativeIndex)) { + payload.toIndex = groupRelativeIndex; + } + + await DeltaLog.append(DeltaLog.OPS.TAB_MOVE, payload); + } catch (e) { + logger.onCatch('tabMoved', false)(e); + } +} + +/** + * Record a tab removal (op `tab.remove`). Called from tabs.js `onRemoved` for + * tracked (grouped) tabs only. The caller reads uid/groupId from the cache BEFORE + * the cache entry is dropped and passes them in (cache is gone by send time). + * @param {string} uid + * @param {string} groupId + */ +export async function tabRemoved(uid, groupId) { + try { + if (isApplying()) { + return; + } + + if (!uid || !groupId) { + return; + } + + await DeltaLog.append(DeltaLog.OPS.TAB_REMOVE, { + groupId, + uid, + }); + } catch (e) { + logger.onCatch('tabRemoved', false)(e); + } +} + +/** + * Record a group addition (op `group.add`). Called from groups.js `sendAdded`. + * @param {object} group - full group record. + */ +export async function groupAdded(group) { + try { + if (isApplying()) { + return; + } + + await DeltaLog.append(DeltaLog.OPS.GROUP_ADD, {group}); + } catch (e) { + logger.onCatch('groupAdded', false)(e); + } +} + +/** + * Record a group change (op `group.modify`). Called from groups.js `sendUpdated`. + * Carries the full group record (not just the changed keys) so replay can faithfully + * resurrect a group deleted elsewhere. + * @param {object} fullGroup - the full, post-update group record (incl. id). + */ +export async function groupModified(fullGroup) { + try { + if (isApplying()) { + return; + } + + await DeltaLog.append(DeltaLog.OPS.GROUP_MODIFY, {group: fullGroup}); + } catch (e) { + logger.onCatch('groupModified', false)(e); + } +} + +/** + * Record a group reorder (op `group.move`). Called from groups.js `move`. + * + * Group order is the snapshot's group-array position, so we capture the group's final + * 0-based index in the persisted groups list. Replay applies last-writer-wins by event + * order (see {@link module:sync/delta/replay}). Without this op a local reorder produced + * no delta, so the cloud kept the stale order and apply reverted the local change. + * + * @param {string} groupId + * @param {number} toIndex - final 0-based position of the group in the groups list. + */ +export async function groupMoved(groupId, toIndex) { + try { + if (isApplying()) { + return; + } + + if (groupId == null || !Number.isInteger(toIndex)) { + return; + } + + await DeltaLog.append(DeltaLog.OPS.GROUP_MOVE, {groupId, toIndex}); + } catch (e) { + logger.onCatch('groupMoved', false)(e); + } +} + +/** + * Record a group removal (op `group.remove`). Called from groups.js `sendRemoved`. + * @param {string} groupId + */ +export async function groupRemoved(groupId) { + try { + if (isApplying()) { + return; + } + + await DeltaLog.append(DeltaLog.OPS.GROUP_REMOVE, {groupId}); + } catch (e) { + logger.onCatch('groupRemoved', false)(e); + } +} + +/** The concrete set of option keys that roam (derived from constants via the predicate). */ +const SYNCED_OPTION_KEYS = new Set(syncedOptionKeys(Constants.ALL_OPTION_KEYS)); + +/** + * Record changes to global STG option values (op `option.set`, one event per changed + * SYNCED key). Called from the single option-save choke point — background.js + * `saveOptions(...)` — AFTER the values have been persisted, with the just-saved + * key/value bag. Mirrors the tab/group capture entry points: + * - early-returns while a sync apply runs ({@link isApplying}) so the transport's own + * option writes (which go through that same `saveOptions`) are NOT re-captured into a + * feedback loop; + * - only synced keys are recorded — per-device/local keys (`sync*`, `autoBackup*`, see + * {@link module:sync/delta/option-keys}) never enter the log; + * - one `option.set` per key so replay resolves each key independently (per-key LWW). + * + * @param {object} savedOptions - the {key: value} bag that was just persisted. + */ +export async function optionsChanged(savedOptions) { + try { + if (isApplying()) { + return; + } + + for (const [key, value] of Object.entries(savedOptions || {})) { + if (!SYNCED_OPTION_KEYS.has(key)) { + continue; + } + await DeltaLog.append(DeltaLog.OPS.OPTION_SET, {key, value}); + } + } catch (e) { + logger.onCatch('optionsChanged', false)(e); + } +} + +// --------------------------------------------------------------------------- +// Global pinned tabs (window-global, NOT in any STG group). Captured into the separate +// `pinnedTabs` snapshot section via the `pinned.*` ops. Identity is the tab `uid` +// (assigned/backfilled here, since pinned tabs don't get one from the group machinery). +// URL is gated by `isUrlSyncable` (see module header) — a pinned tab whose url is pure +// noise (about:blank/newtab/…) is not worth syncing; non-trivial about: urls DO sync. +// +// DISCRIMINATOR: these entry points are only reached from tabs.js for tabs with NO +// groupId in the cache. A per-group pinned tab (one that HAS a groupId) is a group tab +// and is captured via the `tab.*` ops instead — the two never collide. +// --------------------------------------------------------------------------- + +/** + * Build the full pinned-tab record a pinned.add/modify event carries. Like + * {@link buildTabRecord} but carries NO groupId (pinned is window-global) and NO + * group-relative index — pinned tabs are first in the window, so their browser `index` + * already equals their pinned-relative position. Replay re-stamps index from array order. + * @param {object} tab - browser tab. + * @param {string} uid + * @param {object} [snapshot] - A4: synchronously-read cache fields pinned at the call site + * (`lastModified`, `favIconUrl`), used instead of re-reading the torn-down cache. + * @returns {object} + */ +function buildPinnedRecord(tab, uid, snapshot = null) { + const record = { + uid, + // decode the "unsupported URL" stub page back to the embedded original (see + // buildTabRecord) — keeps a synced about: pinned tab on its original identity. + url: unwrapStubUrl(tab.url), + title: tab.title, + cookieStoreId: tab.cookieStoreId, + // see buildTabRecord: KEEP the favicon (incl. small data:); only a >~50 KB blob is + // dropped. Bloat is bounded by the emit throttle + latest-wins snapshot. + // A4: prefer the live tab favicon, fall back to the call-site snapshot. + favIconUrl: sanitizeFavIconUrl(tab.favIconUrl ?? snapshot?.favIconUrl), + // A4: prefer the snapshot's lastModified (pinned at the call site), else live cache. + lastModified: snapshot?.lastModified ?? Cache.getTabLastModified(tab.id), + }; + if (Number.isInteger(tab.index)) { + record.index = tab.index; + } + // source loaded-state (see buildTabRecord): pinned tabs are usually loaded by + // Firefox, but we still record the live signal so `syncActivatePreviouslyActiveTabs` + // can honor it. ALWAYS emitted explicitly (true or false) so a genuine un-load clears + // it and the replay clobber-safety has an unambiguous value to work with. + record.loaded = tab.discarded === false; + return record; +} + +/** + * Record a pinned-tab addition (op `pinned.add`). Called from tabs.js when a tab is + * created already pinned, or when an existing tab transitions to pinned (`onUpdated` + * with `changeInfo.pinned === true`). Only URLs that {@link isUrlSyncable} + * are recorded. + * @param {object} tab + */ +export async function pinnedAdded(tab) { + try { + if (isApplying()) { + return; + } + + if (!isUrlSyncable(unwrapStubUrl(tab.url))) { + return; + } + + const uid = await resolveUid(tab.id); + if (!uid) { + return; + } + + await DeltaLog.append(DeltaLog.OPS.PINNED_ADD, { + tab: buildPinnedRecord(tab, uid), + }); + } catch (e) { + logger.onCatch('pinnedAdded', false)(e); + } +} + +/** + * Record a pinned-tab content change (op `pinned.modify`) - url/title/favIcon. Called + * from tabs.js `onUpdated` for an already-pinned tab whose content changed. + * @param {object} tab + * @param {object} [snapshot] - A4: cache fields read synchronously at the call site, + * used in place of re-reading the (possibly torn-down) cache. See {@link tabModified}. + */ +export async function pinnedModified(tab, snapshot = null) { + try { + if (isApplying()) { + return; + } + + // suppress the async echo of a sync-APPLIED navigation (see tabModified): the echo of the + // EXACT applied pinned url is dropped, while a redirect to a DIFFERENT url is captured so + // the cloud converges to it (rather than re-navigating the pinned tab every cycle). + if (consumeAppliedNavigationEcho(tab.id, tab.url)) { + return; + } + + if (!isUrlSyncable(unwrapStubUrl(tab.url))) { + return; + } + + const uid = snapshot?.uid || await resolveUid(tab.id); + if (!uid) { + return; + } + + await DeltaLog.append(DeltaLog.OPS.PINNED_MODIFY, { + tab: buildPinnedRecord(tab, uid, snapshot), + }); + } catch (e) { + logger.onCatch('pinnedModified', false)(e); + } +} + +/** + * Record a pinned-tab reorder (op `pinned.move`). Called from tabs.js `onMoved` for a + * pinned tab. Carries only uid + target index per the schema. Pinned tabs are first in + * the window, so the browser `toIndex` is already the pinned-relative position. + * @param {number} tabId + * @param {number} [toIndex] + */ +export async function pinnedMoved(tabId, toIndex) { + try { + if (isApplying()) { + return; + } + + const uid = await resolveUid(tabId); + if (!uid) { + return; + } + + const payload = {uid}; + if (Number.isInteger(toIndex)) { + payload.toIndex = toIndex; + } + + await DeltaLog.append(DeltaLog.OPS.PINNED_MOVE, payload); + } catch (e) { + logger.onCatch('pinnedMoved', false)(e); + } +} + +/** + * Record a pinned-tab removal (op `pinned.remove`). Called from tabs.js `onRemoved` for + * a pinned tab, AND on the unpin transition (a tab becoming unpinned leaves the global + * pinned set). The caller passes the uid (read from cache before it is dropped on + * removal, or resolved live on unpin). + * @param {string} uid + */ +export async function pinnedRemoved(uid) { + try { + if (isApplying()) { + return; + } + + if (!uid) { + return; + } + + await DeltaLog.append(DeltaLog.OPS.PINNED_REMOVE, {uid}); + } catch (e) { + logger.onCatch('pinnedRemoved', false)(e); + } +} diff --git a/addon/src/js/sync/delta/delta-log.js b/addon/src/js/sync/delta/delta-log.js new file mode 100644 index 00000000..20a0d5bb --- /dev/null +++ b/addon/src/js/sync/delta/delta-log.js @@ -0,0 +1,346 @@ + +/** + * Local append-only delta event log for hybrid snapshot + delta sync (Phase P1). + * + * This module records the changes the user makes to tabs and groups as an ordered + * stream of events for THIS device. It is purely additive and inert in P1: events + * accumulate locally and nothing consumes them yet. The replay engine (P2) and the + * gist transport (P3) build on top of this; see `.project/DESIGN_DELTA_SYNC.md`. + * + * ## Event schema (per DESIGN_DELTA_SYNC.md "Event schema") + * Each event: + * { seq, ts, op, ...payload } + * where + * - `seq` — per-device monotonic integer, assigned on append. Disambiguates + * same-device ordering when wall-clock `ts` ties. + * - `ts` — `Utils.unixNowMs()` at append time (best-effort wall clock). + * - `op` — one of the OPS below. + * + * Payloads by op (the writer always has the full local record, so modify/add carry + * it in full so a later resurrection during replay is faithful): + * tab.add { groupId, tab: } + * tab.modify { groupId, tab: } + * tab.move { groupId, uid, toIndex } + * tab.remove { groupId, uid } + * group.add { group: } + * group.modify { group: } + * group.move { groupId, toIndex } // group reordered to 0-based position in groups list + * group.remove { groupId } + * option.set { key, value } // one global STG option key changed to `value` + * pinned.add { tab: } + * pinned.modify { tab: } + * pinned.move { uid, toIndex } + * pinned.remove { uid } + * + * Global pinned tabs are window-global in Firefox (NOT in any STG group), so their + * events carry NO groupId — identity is the tab `uid` and they fold into a separate + * ordered `pinnedTabs` array in the snapshot (mirrors the legacy backup field + * `data.pinnedTabs`), rather than into `groups`. See `sync/delta/replay.js` for how + * they replay. A tab that becomes pinned emits `pinned.add` (and a `tab.remove` for the + * group it left); a tab that becomes unpinned emits `pinned.remove`. + * + * The persisted file shape (for the future transport) is: + * { v: SCHEMA_VERSION, deviceId, events: [ ...event ] } + * + * ## Backing store + * `browser.storage.local` (the same async store the snapshot uses, see storage.js), + * NOT the synchronous localStorage proxy. An append-only log must stay cheap to + * append to and survive restarts; the async storage handles larger payloads and + * does not block, whereas the localStorage proxy re-stringifies a whole value per + * key write and is synchronous. An in-memory mirror keeps `seq` monotonic and makes + * reads cheap; appends serialize through a write chain so concurrent appends keep + * their order and never lose an event. + * + * @module sync/delta/delta-log + */ + +import '/js/prefixed-storage.js'; +import * as Utils from '/js/utils.js'; +import Logger from '/js/logger.js'; +import {getDeviceId} from './device-id.js'; + +const logger = new Logger('DeltaLog'); + +/** Event schema version (the `v` field in the persisted file). */ +export const SCHEMA_VERSION = 1; + +/** browser.storage.local key holding this device's event log. */ +const STORAGE_KEY = 'syncDeltaLog'; + +/** + * Supported operations. Frozen so callers can reference them without typos. + * @readonly + */ +export const OPS = Object.freeze({ + TAB_ADD: 'tab.add', + TAB_MODIFY: 'tab.modify', + TAB_MOVE: 'tab.move', + TAB_REMOVE: 'tab.remove', + GROUP_ADD: 'group.add', + GROUP_MODIFY: 'group.modify', + GROUP_MOVE: 'group.move', + GROUP_REMOVE: 'group.remove', + // one global STG option key set to a value; per-key last-writer-wins on replay + OPTION_SET: 'option.set', + // global pinned tabs (window-global, no groupId); fold into `snapshot.pinnedTabs` + PINNED_ADD: 'pinned.add', + PINNED_MODIFY: 'pinned.modify', + PINNED_MOVE: 'pinned.move', + PINNED_REMOVE: 'pinned.remove', +}); + +const VALID_OPS = new Set(Object.values(OPS)); + +// in-memory mirror of the persisted log; lazily hydrated on first access +let events = null; +let lastSeq = 0; + +// memoizes the single in-flight first hydration so concurrent first-touch callers all +// await ONE `browser.storage.local.get(...)` + migration instead of each running their own +// (which would clobber `events`/`lastSeq` and lose an event + collide its seq). Reset to +// null by clear() so a post-reset hydration can re-run. See `ensureLoaded`. +let loadingPromise = null; + +// serializes persistence so overlapping appends keep order and don't clobber +let writeChain = Promise.resolve(); + +/** + * Strip the favicon from the tab record a HISTORICAL event carries, in place (migration). + * + * Recovery for the multi-GB `syncDeltaLog` bloat: a single favicon-only event was duplicated + * across hundreds of thousands of already-stored events. Those historical favicons are + * REDUNDANT — the snapshot keeps each tab's latest favicon and the live tabs re-capture it + * within a sync or two — so on hydrate we drop EVERY event's `favIconUrl` unconditionally + * (not just data:/oversized ones). This collapses the existing bloat without a manual reset, + * favicons re-establish from live state, and it's idempotent: a log with no favicons left + * triggers no rewrite. Favicons are cosmetic, so identity (url/title/group/pinned) is + * untouched. Going forward there are NO favicon-only events (the favicon just rides along as + * one field of a record written for a real change), so the log stays small. + * + * @param {object} event - a single delta event (mutated in place). + * @returns {boolean} true if the event was modified. + */ +function stripEventFavicon(event) { + // tab.add / tab.modify / pinned.add / pinned.modify all carry the record under `tab`. + const tab = event?.tab; + if (tab && typeof tab === 'object' && Object.hasOwn(tab, 'favIconUrl')) { + delete tab.favIconUrl; + return true; + } + return false; +} + +/** + * Hydrate the in-memory mirror from storage exactly once. + * + * On hydrate we run a ONE-TIME MIGRATION over the stored log: every historical event has + * its `favIconUrl` stripped and the log is rewritten ONCE. This is the recovery path for + * existing users whose `syncDeltaLog` had grown to gigabytes of duplicated base64 favicons + * — it reclaims that RAM without a manual reset. Safe + idempotent: the stripped favicons + * are redundant (the snapshot holds each tab's latest favicon and live tabs re-capture it), + * identity/url/title/group/pinned are untouched, and a log already free of favicons triggers + * no rewrite. + * + * ## First-hydration race + * The hydration is memoized as a SINGLE in-flight promise (`loadingPromise`). A bare + * `if (events !== null) return` guard is not enough: `events` is only assigned AFTER the + * `await browser.storage.local.get(...)`, so two concurrent first-touch callers (e.g. a + * capture `append` racing the transport's `getEvents`/`append` at background startup) would + * BOTH observe `events === null`, BOTH await the get, and the second would overwrite `events` + * with a fresh stored array — dropping any event the first already pushed and recomputing + * `lastSeq` from storage, so the next append reuses a collided seq (data loss). Memoizing the + * promise makes every caller await the same hydration; the body runs at most once per reset. + * If the hydration FAILS, `loadingPromise` is cleared so a later call can retry (preserving + * the pre-fix behaviour of leaving `events === null` on a failed get) rather than caching the + * rejection forever. + * @returns {Promise} + */ +function ensureLoaded() { + return loadingPromise ??= (async () => { + const stored = await browser.storage.local.get(STORAGE_KEY); + const log = stored[STORAGE_KEY]; + + events = Array.isArray(log?.events) ? log.events : []; + lastSeq = events.length ? events[events.length - 1].seq : 0; + + // one-time recovery: strip favicons from already-stored events and rewrite if any were + // found. Done after `events`/`lastSeq` are set so persist() writes the cleaned log. + let changed = false; + for (const event of events) { + if (stripEventFavicon(event)) { + changed = true; + } + } + if (changed) { + logger.info('migrated stored delta log: stripped historical favicons', {events: events.length}); + await persist(); + } + })().catch(err => { + // let a later first touch retry instead of permanently caching the rejection. + loadingPromise = null; + throw err; + }); +} + +/** + * Persist the current in-memory log. Serialized through `writeChain`. + * @returns {Promise} + */ +function persist() { + writeChain = writeChain.then(() => browser.storage.local.set({ + [STORAGE_KEY]: { + v: SCHEMA_VERSION, + deviceId: getDeviceId(), + events, + }, + })); + + return writeChain; +} + +/** + * Append one event to this device's log. Assigns the next `seq`, stamps `ts`, + * and persists. Unknown ops are rejected (logged + ignored) so a caller typo can + * never corrupt the stream. + * + * @param {string} op - one of {@link OPS}. + * @param {object} [payload={}] - op-specific payload (see module docs). + * @returns {Promise} the appended event, or undefined if ignored. + */ +export async function append(op, payload = {}) { + if (!VALID_OPS.has(op)) { + logger.error('append: unknown op', op); + return; + } + + await ensureLoaded(); + + const event = { + seq: ++lastSeq, + ts: Utils.unixNowMs(), + op, + ...payload, + }; + + events.push(event); + + await persist(); + + return event; +} + +/** + * Returns a shallow copy of all events currently in this device's log. + * @returns {Promise} + */ +export async function getEvents() { + await ensureLoaded(); + return events.slice(); +} + +/** + * Returns events with `seq` strictly greater than `seq` (in order). + * Used by the future transport to push only not-yet-synced events. + * @param {number} seq + * @returns {Promise} + */ +export async function getEventsSince(seq) { + await ensureLoaded(); + return events.filter(event => event.seq > seq); +} + +/** + * Drops events with `seq` less than or equal to `seq` (compaction support, P4). + * Events newer than `seq` are kept; `lastSeq` is unchanged so future appends stay + * monotonic even after the head is trimmed. + * @param {number} seq + * @returns {Promise} + */ +export async function clearUpTo(seq) { + await ensureLoaded(); + events = events.filter(event => event.seq > seq); + await persist(); +} + +/** + * Returns the highest assigned seq (0 if empty). Useful for watermark math (P3/P4). + * @returns {Promise} + */ +export async function getLastSeq() { + await ensureLoaded(); + return lastSeq; +} + +/** + * Fully reset this device's local delta log: empties the persisted events, resets the + * `seq` counter to 0, and refreshes the in-memory mirror. Unlike {@link clearUpTo} + * (which trims the head but keeps `lastSeq` monotonic for ongoing compaction), this is + * a hard reset for the recovery flow — after it the next append starts from `seq` 1, + * matching a fresh install. Local only: it never touches the cloud delta file (the + * stale cloud file is reconciled/compacted on the next sync, or deleted manually). + * @returns {Promise} + */ +export async function clear() { + await ensureLoaded(); + events = []; + lastSeq = 0; + // Reset the memoized hydration too: the in-memory mirror above is now the authoritative + // (empty) state and is persisted below, but dropping `loadingPromise` lets a later first + // touch re-run the memoized hydration cleanly instead of being short-circuited by the + // already-resolved promise from before the reset. + loadingPromise = null; + await persist(); +} + +/** + * RESET/WATERMARK TRAP recovery (E2). After a local {@link resetSyncState}, this + * device's `lastSeq` is rewound to 0 so fresh appends start at seq 1 — but the + * CLOUD snapshot still carries `watermark[thisDevice] = N` from before the reset, + * which reset cannot clear. replay() dedups every event with `seq <= watermark` + * (replay.js rule 4), so those re-issued low-seq events would be SILENTLY SKIPPED + * and the post-reset local changes lost until seq organically climbs past N. + * + * This fast-forwards THIS device's log so EVERY event sits strictly above `minSeq` + * (the stale cloud watermark): it shifts every event's `seq` up by a SINGLE constant + * offset chosen so the LOWEST event seq lands at `minSeq + 1` (preserving relative + * order and any gaps), and bumps `lastSeq` to match, so future appends also stay above + * the watermark. For an EMPTY log there is no event to anchor on, so it just advances + * `lastSeq` to `minSeq + 1`. A no-op when nothing is at risk — i.e. the lowest event + * seq already exceeds `minSeq` (and, for an empty log, `lastSeq` already does) — which + * is the normal, non-reset case. + * + * Safe to call on every sync: it only acts when the trap is actually present, never + * lowers a seq, and keeps the log monotonic. The caller must re-read pending events + * afterwards (their seqs may have changed). + * + * @param {number} minSeq - the highest seq already folded into the cloud base for this + * device (its cloud watermark). Events must end up strictly greater than this. + * @returns {Promise} true iff a fast-forward was applied (seqs shifted). + */ +export async function fastForwardSeqsAbove(minSeq) { + await ensureLoaded(); + + if (!Number.isFinite(minSeq)) { + return false; + } + + // anchor on the LOWEST event seq (the one most at risk of dedup); for an empty log + // anchor on lastSeq + 1 so the counter alone advances. A single constant offset + // preserves relative order and any pre-existing gaps. + const lowest = events.length + ? events.reduce((min, e) => (e.seq < min ? e.seq : min), Infinity) + : lastSeq + 1; + + if (lowest > minSeq) { + return false; // normal path: nothing sits at/below the cloud watermark + } + + const offset = (minSeq + 1) - lowest; + for (const event of events) { + event.seq += offset; + } + lastSeq += offset; + + await persist(); + return true; +} diff --git a/addon/src/js/sync/delta/delta-sync.js b/addon/src/js/sync/delta/delta-sync.js new file mode 100644 index 00000000..02f05e21 --- /dev/null +++ b/addon/src/js/sync/delta/delta-sync.js @@ -0,0 +1,2346 @@ + +/** + * Live delta-sync orchestrator (Phase P3b). + * + * Wires the pure delta pipeline (`plan-sync.js` → `replay.js`) into the real sync + * flow: pull the cloud snapshot + every device's delta log, compute the resolved + * effective state and a declarative `browserOps` diff (PURE, via {@link planSync}), + * then APPLY that diff to the live browser through STG's existing tab/group apply + * paths, and finally PUSH this device's own delta file back. + * + * This module is the impure transport half of the contract documented in + * `plan-sync.js`. It deliberately reuses `cloud.js`'s scaffolding rather than + * duplicating it: the provider factory, the `inProgress` guard, the + * `send('sync-*')` broadcast, and `CloudError` are all imported from there. The + * old URL-based `synchronization()` in `cloud.js` is left intact; the manual button + * and the alarm route here instead (see `cloud.js` `USE_DELTA_SYNC` / background.js). + * + * ## Feedback-loop guard (critical) + * Applying remote changes must NOT be re-captured by `delta-capture.js` as fresh + * local deltas, or every sync would grow the log and duplicate tabs forever. Two + * layers protect against this: + * 1. We route every apply through STG's existing skip machinery — `createMultiple` + * / `moveNative` with `skipTrackingFlag=true`, `Tabs.remove(arr, true)` with + * `silentRemove`, and `Groups.save(...)` directly (which never fires the + * `sendAdded/sendUpdated/sendRemoved` helpers the capture layer hooks). Those + * paths make the `tabs.js` `onCreated/onMoved/onRemoved` handlers early-return, + * so capture is never invoked. + * 2. Belt-and-suspenders: we wrap the whole apply in + * {@link module:sync/delta/delta-capture.beginApply}/`endApply`, which sets a + * module-level flag so that even capture entry points NOT covered by a skip + * flag (notably the `onUpdated` url/title `tab.modify` path, which a freshly + * created sync tab triggers when it loads) early-return during apply. + * + * @module sync/delta/delta-sync + */ + +import * as Constants from '/js/constants.js'; +import * as Storage from '/js/storage.js'; +import * as Tabs from '/js/tabs.js'; +import * as Groups from '/js/groups.js'; +import * as MenusMain from '/js/menus-main.js'; +import * as Windows from '/js/windows.js'; +import * as Cache from '/js/cache.js'; +import * as Containers from '/js/containers.js'; +import backgroundSelf from '/js/background.js'; +import Logger from '/js/logger.js'; +import {createCloudProvider} from '../cloud/provider.js'; +import * as SyncStorage from '../sync-storage.js'; +// Same `?can-do-synchronization` instance the background page uses, so `send` +// broadcasts on the one cloud channel and there is no duplicate cloud.js module. +import {CloudError, send, ALARM_NAME_RETRY} from '../cloud/cloud.js?can-do-synchronization'; +import {runSyncApply, isUserActive, DEFAULT_SYNC_APPLY_WATCHDOG_MS} from './user-priority-lock.js'; +import * as DeltaLog from './delta-log.js'; +import * as DeltaCapture from './delta-capture.js'; +import {getDeviceId} from './device-id.js'; +import {planSync, computeBootstrapEvents, baselineFromSnapshot} from './plan-sync.js'; +import { + evaluateCompaction, + selfFoldedSeq, + truncateSelfEvents, + resolveDeferredTruncation, +} from './compaction.js'; +import { + makeOutboundMapper, + makeInboundMapper, + mapStateContainers, + mapEventContainers, +} from './container-map.js'; +import {syncedOptionKeys} from './option-keys.js'; +import {shouldSleepSyncedTab, SLEEP_OPTION_KEYS} from './tab-sleep.js'; +import {isUrlSyncable, unwrapStubUrl, sanitizeFavIconUrl, liveUrlMatchesSource, shouldNavigateLiveTabUrl} from './url-sync.js'; +import {seedSnapshotFromLegacyBackup} from './seed.js'; +import { + SNAPSHOT_FILE_NAME, + DELTA_FILE_PREFIX, + LEGACY_BACKUP_FILE_NAME, + deltaFileName, + deviceIdFromDeltaFileName, +} from './layout.js'; + +const logger = new Logger('DeltaSync'); + +const storage = localStorage.create(Constants.MODULES.CLOUD); + +// --- Apply-phase diagnostics + completion watchdog -------------------------- +// The sync apply runs as the critical section of the user-priority mutex (every user +// group/tab mutation queues behind it on the same promise tail). If any await inside the +// apply never settles, the lock never releases and EVERY future user action hangs forever +// — the reported post-sync UI freeze. To both DIAGNOSE and BOUND that, we (a) time each +// major apply phase and record the most-recently-STARTED one in `currentApplyPhase`, and +// (b) wrap the apply in a completion watchdog (see runSyncApply): if it trips, the WARN it +// logs names exactly the phase that stalled. +// +// `currentApplyPhase` is module-level (single bg-page realm; one apply at a time under the +// mutex) so the watchdog callback can read the stuck phase without threading it through. +let currentApplyPhase = null; + +// How long the apply may HOLD the user-priority lock before the completion watchdog frees +// it (see runSyncApply in user-priority-lock.js). Generous — well above any legitimate +// apply (a real apply of dozens of tabs completes in a few seconds per the debug logs) — +// so it only ever fires on a genuine never-settling stall, never on a slow-but-fine apply. +const SYNC_APPLY_WATCHDOG_MS = DEFAULT_SYNC_APPLY_WATCHDOG_MS; + +/** + * Begin one apply phase: record it as the current phase (overwriting any prior phase, so a + * stall leaves `currentApplyPhase` holding the phase that hung) and return an `endPhase()` + * that logs the phase duration. Phase-level only — no per-tab spam, no blobs. Call the + * returned `endPhase()` when the phase finishes; it is a no-op if a later phase already + * started (so an early-return that forgets to call it can't mislabel the next phase's timing). + * @param {string} name phase name (shows in the debug log + the watchdog WARN). + * @param {object} log parent logger context. + * @returns {() => void} endPhase + */ +function beginApplyPhase(name, log) { + currentApplyPhase = name; + const startedAt = Date.now(); + return function endPhase() { + log.log('apply phase done', {phase: name, ms: Date.now() - startedAt}); + }; +} + +// Per-device, per-install persisted high-water marks for the delta pipeline. Kept +// in the synchronous CLOUD localStorage (same store as deviceId/githubGistFileName) +// so they survive restarts and never need an await. Keyed by selfDeviceId so a +// shared profile copied between installs can't cross its marks. +const LAST_PUSHED_SEQ_PREFIX = 'deltaLastPushedSeq:'; +const WATERMARK_PREFIX = 'deltaBaseWatermark:'; +// Per-device "a local reset happened; reconcile against the stale CLOUD watermark on +// the next sync" flag (E2). resetSyncState rewinds local lastSeq to 0 but CANNOT clear +// the cloud snapshot's watermark[self]=N, so re-issued seq 1..N would be silently +// dedup-skipped by replay. While this flag is set we force a FULL pull (bypassing the +// conditional fast path, which can't see the cloud watermark) so we can fast-forward +// this device's log above that watermark, then clear the flag. Keyed by selfDeviceId. +const RESET_PENDING_PREFIX = 'deltaResetPending:'; +// Per-device DEFERRED SELF-TRUNCATION marker (Part C, the data-loss backstop). When a +// compaction cycle writes a new snapshot, it does NOT truncate its own log in the same cycle +// (a peer could clobber that just-written snapshot before its durability is confirmed, and +// the snapshot is the ONLY home of folded history — see compaction.js). Instead it records +// here the self watermark seq it folded into the snapshot it wrote. On a SUBSEQUENT cycle, +// once the PULLED cloud snapshot's watermark[self] proves >= this seq (our snapshot survived +// or a later one supersedes it), the truncation is confirmed safe: we clearUpTo it locally +// and drop seq <= it from the cloud self-delta, then clear the marker. If the snapshot was +// clobbered (watermark below the marker), we keep deferring (events stay in the cloud delta, +// nothing lost). Stored as a number string in the synchronous CLOUD localStorage, keyed by +// selfDeviceId. +const PENDING_TRUNCATE_PREFIX = 'deltaPendingTruncateSeq:'; +// Per-device durable baseline = the set of group ids + tab uids this device last +// reconciled as synced (the ids/uids in the resolved snapshot at the end of its last +// successful sync). It is the authoritative "did THIS device know this as synced" +// signal that gates removals: it survives cloud compaction (which prunes the delete +// events the old cloud-known gate relied on), so a delete still propagates to a +// device that was offline across a compaction window, while never deleting a +// never-synced local item. Stored as JSON {tabUids, groupIds} in the synchronous +// CLOUD localStorage, keyed by selfDeviceId. +const BASELINE_PREFIX = 'deltaBaseline:'; + +// --- Pre-apply safety backup (Feature 1) ----------------------------------- +// Number of rolling backup slots kept on disk. Each pre-apply backup overwrites +// the next slot (round-robin), so disk use is bounded to exactly this many files +// instead of growing unbounded. 5 mirrors a "last few" retention window. +const PRE_APPLY_BACKUP_SLOTS = 5; +// CLOUD-localStorage key holding the next slot index to (over)write. Synchronous, +// survives restarts. A missing/corrupt value restarts from slot 0 (harmless). +const PRE_APPLY_BACKUP_SLOT_KEY = 'deltaPreApplyBackupSlot'; +// Rolling file path written via STG's existing backup serializer. `{ff-version}` is +// expanded by File.save's getFilePathVariables(); the trailing `.json` is appended by +// File.saveBackup. The slot index makes the round-robin set overwrite in place. +function preApplyBackupFilePath(slot) { + return `STG-sync-pre-apply-backups-FF-{ff-version}/STG-sync-pre-apply-backup-${slot}`; +} + +function lastPushedSeqKey(deviceId) { + return LAST_PUSHED_SEQ_PREFIX + deviceId; +} + +function watermarkKey(deviceId) { + return WATERMARK_PREFIX + deviceId; +} + +function baselineKey(deviceId) { + return BASELINE_PREFIX + deviceId; +} + +function resetPendingKey(deviceId) { + return RESET_PENDING_PREFIX + deviceId; +} + +function pendingTruncateKey(deviceId) { + return PENDING_TRUNCATE_PREFIX + deviceId; +} + +/** + * Load this device's persisted baseline. Returns empty sets on first run (or if the + * stored value is missing/corrupt) — the conservative default: an empty baseline + * authorizes NO removals, so a lost/cleared baseline can never cause data loss, only + * a transient failure to propagate a delete (re-established on the next sync once the + * item is reconciled again). + * @param {string} deviceId + * @returns {{tabUids: Set<*>, groupIds: Set<*>, optionKeys: Set<*>, pinnedUids: Set<*>}} + */ +function loadBaseline(deviceId) { + const raw = storage[baselineKey(deviceId)]; + if (!raw) { + return {tabUids: new Set(), groupIds: new Set(), optionKeys: new Set(), pinnedUids: new Set()}; + } + try { + const parsed = JSON.parse(raw); + return { + tabUids: new Set(parsed.tabUids || []), + groupIds: new Set(parsed.groupIds || []), + optionKeys: new Set(parsed.optionKeys || []), + pinnedUids: new Set(parsed.pinnedUids || []), + }; + } catch { + return {tabUids: new Set(), groupIds: new Set(), optionKeys: new Set(), pinnedUids: new Set()}; + } +} + +/** + * Persist this device's baseline (arrays of ids/uids + synced option keys + global + * pinned uids). Called ONLY after a successful sync, with the ids/uids/keys of the + * resolved snapshot. `pinnedUids` MUST round-trip: it is what gates global-pinned-tab + * removal (see {@link module:sync/delta/plan-sync.diffToBrowserOps}) — without it a + * synced pinned tab deleted on another device would never be removed here, because the + * removal gate `in baseline AND absent from resolved` could never see it as synced. + * @param {string} deviceId + * @param {{tabUids: Array<*>, groupIds: Array<*>, optionKeys: Array<*>, pinnedUids: Array<*>}} baseline + */ +function saveBaseline(deviceId, baseline) { + storage[baselineKey(deviceId)] = JSON.stringify({ + tabUids: baseline.tabUids || [], + groupIds: baseline.groupIds || [], + optionKeys: baseline.optionKeys || [], + pinnedUids: baseline.pinnedUids || [], + }); +} + +/** + * Map STG's loaded groups into the snapshot shape the planner consumes: + * `{groups: [{id, ...props, tabs: [{uid, url, title, cookieStoreId, index, ...}]}]}`. + * + * PURE: no `browser.*`, reads the input, returns a fresh structure. Archived groups + * keep their stored `tabs` as-is (they already carry uid via `prepareForSave`); live + * groups carry the session-hydrated tabs (uid/lastModified applied by `Tabs.get`). + * Tabs without a `uid` are dropped from localState: the planner keys tabs by uid, and + * an un-uided tab can't be reconciled — leaving it out means the planner treats it as + * "not present locally" rather than mis-removing it (bias to keep, never to lose). + * + * Global pinned tabs (window-global, not in any group) are passed in separately and + * Tab URLs are decoded back from STG's "unsupported URL" stub page to their embedded + * original (see {@link module:sync/delta/url-sync} `unwrapStubUrl`) so a tab a previous + * sync rendered as the stub keeps its original about: identity in this snapshot. + * + * mapped into a flat `pinnedTabs` array — the planner diffs them as a parallel section + * keyed by uid. Same uid-required filter as group tabs (an un-uided pinned tab can't be + * reconciled, so leaving it out biases to keep rather than mis-remove). + * + * @param {object[]} loadedGroups - result of `Groups.load(null, true).groups`. + * @param {object} [syncedOptions] - this device's current values for the SYNCED option + * keys (already filtered to the synced subset by the caller). Folded into + * `localState.options` so the planner can diff resolved-vs-local options. Defaults to + * `{}` (so existing callers/tests that pass only groups keep working). + * @param {object[]} [livePinnedTabs] - live global pinned tabs (uid-hydrated) for this device. + * @returns {{groups: object[], pinnedTabs: object[], options: object}} + */ +export function buildLocalState(loadedGroups, syncedOptions = {}, livePinnedTabs = []) { + const groups = (loadedGroups || []).map(group => { + const {tabs, ...props} = group; + + const mappedTabs = (Array.isArray(tabs) ? tabs : []) + .filter(tab => tab && tab.uid != null) + .map((tab, index) => ({ + uid: tab.uid, + url: unwrapStubUrl(tab.url), + title: tab.title, + cookieStoreId: tab.cookieStoreId, + // GROUP-RELATIVE position: `Groups.load` returns `group.tabs` already in + // group order (sorted by browser index), so the local ARRAY POSITION is + // the correct 0-based order WITHIN the group. The raw `tab.index` is the + // browser-window-absolute index (shifted by pinned tabs and other groups' + // tabs sharing the window, and different per machine) — using it would + // shuffle tab order on the receiving side. See replay.js for the contract. + index, + lastModified: tab.lastModified, + // KEEP the CURRENT favicon (incl. small data:) so the synced/sleeping tab + // shows its icon. Read live here, so the compaction snapshot always carries + // the tab's current favicon — this is the propagation path for a favicon that + // changed without a title/url change (no favicon-only event exists). One + // favicon per tab ⇒ no bloat. sanitizeFavIconUrl drops only a >~50 KB blob. + favIconUrl: sanitizeFavIconUrl(tab.favIconUrl), + // group-scoped pin flag (pinned within an active group). Mirrors the + // delta-capture record field; only present when true so it round-trips + // through replay/plan without perturbing other tabs. + ...(tab.groupPinned ? {pinned: true} : {}), + // source loaded-state (mirrors delta-capture buildTabRecord): true when the + // tab is loaded (not discarded) here, so a receiving device with + // `syncActivatePreviouslyActiveTabs` on can re-activate it. Additive/true-only. + ...(tab.discarded === false ? {loaded: true} : {}), + // keep the live browser id so the apply step can target real tabs + id: tab.id, + })); + + // NO-SILENT-DROP SAFEGUARD: a non-archived group's tabs are LIVE-enumerated (see + // Groups.load: `group.tabs` is rebuilt from the browser query, not from storage), so + // a normal grouped tab that fails to enumerate (or whose uid never committed) silently + // vanishes from the pushed snapshot — the exact shape of the group-normal-tabs sync + // data loss this guards against. If a non-archived group HAD input tabs but every one + // was dropped (no uid), or fewer survived than came in, surface it in the debug log so + // this class of loss is visible rather than silent. Archived groups carry stored tabs + // (already uid'd) and aren't live-enumerated, so they're exempt. Warn only, never throw + // — biasing to keep what we can over failing the whole sync. + const inputTabCount = Array.isArray(tabs) ? tabs.length : 0; + if (!group.isArchive && inputTabCount > mappedTabs.length) { + logger.warn('buildLocalState: non-archived group dropped tabs from local snapshot', { + groupId: group.id, + inputTabCount, + mappedTabCount: mappedTabs.length, + droppedCount: inputTabCount - mappedTabs.length, + }); + } + + return {...props, tabs: mappedTabs}; + }); + + const pinnedTabs = (Array.isArray(livePinnedTabs) ? livePinnedTabs : []) + .filter(tab => tab && tab.uid != null) + .map((tab, index) => ({ + uid: tab.uid, + url: tab.url, + title: tab.title, + cookieStoreId: tab.cookieStoreId, + // pinned tabs are first in the window, so the live browser `index` is already + // the pinned-relative position; fall back to array order if it's not finite. + index: Number.isFinite(tab.index) ? tab.index : index, + lastModified: tab.lastModified, + // KEEP the favicon (incl. small data:) so the synced pinned tab shows its icon. + // The snapshot holds ONE favicon per tab (bounded by tab count), so this can't + // bloat. sanitizeFavIconUrl only drops a >~50 KB blob ⇒ undefined ⇒ field omitted. + favIconUrl: sanitizeFavIconUrl(tab.favIconUrl), + // source loaded-state (see grouped tabs above): pinned tabs are usually loaded, + // but we record the live signal so `syncActivatePreviouslyActiveTabs` can honor + // it on the receiving side. Additive/true-only. + ...(tab.discarded === false ? {loaded: true} : {}), + // keep the live browser id so the apply step can target real tabs + id: tab.id, + })); + + return {groups, pinnedTabs, options: {...syncedOptions}}; +} + +/** + * Resolve the base snapshot to plan against: prefer the delta-era snapshot file; if + * absent, seed from the legacy single-file backup (a pre-delta user's first delta + * sync); else an empty snapshot. Never throws on "absent" — `readFile` resolves null. + * + * `snapshotExists` reports whether the `STG-snapshot.json` FILE was actually present in + * the gist (true) or had to be synthesized from the legacy backup / empty default (false). + * The push step uses it to FIRST-CREATE the snapshot when it does not yet exist, even on a + * cycle that did not compact (see the snapshot-write gate in {@link deltaSynchronization}). + * @param {object} Cloud - provider instance. + * @returns {Promise<{snapshot: {groups: object[], watermark?: object}, snapshotExists: boolean}>} + */ +async function resolveBaseSnapshot(Cloud) { + const snapshot = await Cloud.readFile(SNAPSHOT_FILE_NAME); + if (snapshot) { + return {snapshot, snapshotExists: true}; + } + + const legacy = await Cloud.readFile(LEGACY_BACKUP_FILE_NAME); + if (legacy) { + logger.info('seeding snapshot from legacy backup'); + return {snapshot: seedSnapshotFromLegacyBackup(legacy), snapshotExists: false}; + } + + return {snapshot: {groups: [], watermark: {}}, snapshotExists: false}; +} + +/** + * Read every per-device delta log from the cloud and normalize to the planner shape + * `[{deviceId, events}]`. The deviceId comes from the file content when present, + * else is parsed from the file name (resilient to a hand-edited file). The gist file + * `name` it was read from is carried through too (used when rewriting a device's own + * delta file, rather than reconstructing the name and risking a mismatch between a + * hand-edited `content.deviceId` and the filename). + * @param {object} Cloud - provider instance. + * @returns {Promise>} + */ +async function resolvePulledDeltaLogs(Cloud) { + const files = await Cloud.readAllMatching(DELTA_FILE_PREFIX); + + return (files || []).map(({name, content}) => ({ + name, + deviceId: content?.deviceId ?? deviceIdFromDeltaFileName(name), + events: Array.isArray(content?.events) ? content.events : [], + })); +} + +/** + * Apply the planner's declarative `browserOps` to the live browser through STG's + * existing apply paths, using skip/silent flags so the capture layer does not record + * these as new deltas. Conservative ordering: create groups first (so tab targets + * exist), then create/move tabs, then remove tabs, then remove groups last. + * + * Removal is the resolved-says-gone set ONLY (the planner derives it by uid/id + * difference); when in doubt the pipeline biases to keeping/creating a tab (replay + * rule 1), so a removal here genuinely means the resolved state dropped it. + * + * @param {object} browserOps - from {@link planSync}. + * @param {object} [resolvedSnapshot] - from {@link planSync} (`plan.resolvedSnapshot`). + * Its per-group tab arrays are the AUTHORITATIVE order; after creates/moves we reconcile + * each touched group's live tab order to it (see {@link reconcileGroupTabOrders}). + * @returns {Promise} + */ +async function applyBrowserOps(browserOps, resolvedSnapshot) { + const log = logger.start('applyBrowserOps', { + groupsToCreate: browserOps.groupsToCreate.length, + groupsToUpdate: browserOps.groupsToUpdate.length, + groupsToRemove: browserOps.groupsToRemove.length, + groupsReorder: browserOps.groupsOrder ? browserOps.groupsOrder.length : 0, + tabsToCreate: browserOps.tabsToCreate.length, + tabsToMove: browserOps.tabsToMove.length, + tabsToUpdate: browserOps.tabsToUpdate?.length || 0, + tabsToRemove: browserOps.tabsToRemove.length, + pinnedToCreate: browserOps.pinnedToCreate?.length || 0, + pinnedToMove: browserOps.pinnedToMove?.length || 0, + pinnedToUpdate: browserOps.pinnedToUpdate?.length || 0, + pinnedToRemove: browserOps.pinnedToRemove?.length || 0, + }); + + // belt-and-suspenders: suppress capture for anything we apply (see module docs) + DeltaCapture.beginApply(); + + // LOCAL-ONLY sleep/activate options that decide whether each sync-created tab is + // created asleep (discarded) or loaded. Read once here and threaded into both the + // grouped-create and pinned-create paths via shouldSleepSyncedTab (see tab-sleep.js). + const sleepOptions = await Storage.get(SLEEP_OPTION_KEYS); + + try { + // --- groups create/update: persist via Groups.save (bypasses send* helpers + // the capture layer hooks). We load current groups, fold the changes, save once. + const needGroupWrite = browserOps.groupsToCreate.length + || browserOps.groupsToUpdate.length + || browserOps.groupsToRemove.length + || browserOps.groupsOrder; + + if (needGroupWrite) { + const endPhase = beginApplyPhase('groups-write', log); + const {groups} = await Groups.load(null, false); + const byId = new Map(groups.map(g => [g.id, g])); + + for (const props of browserOps.groupsToCreate) { + if (!byId.has(props.id)) { + const created = {...Groups.create(props.id, props.title), ...props, tabs: []}; + groups.push(created); + byId.set(created.id, created); + } + } + + for (const props of browserOps.groupsToUpdate) { + const existing = byId.get(props.id); + if (existing) { + Object.assign(existing, props); + } + } + + // unload (move out of windows) any group we're about to remove, then drop it + const removeIds = new Set(browserOps.groupsToRemove.map(g => g.id)); + for (const id of removeIds) { + if (Groups.isLoaded(id)) { + await Groups.unload(id).catch(log.onCatch(['cant unload group', id], false)); + } + } + + let nextGroups = groups.filter(g => !removeIds.has(g.id)); + + // GROUP ORDER: `Groups.save` persists the array order = the order STG shows, + // so reorder to match the resolved (authoritative) group order. Groups present + // only locally (not in the resolved order) are KEPT, appended after the ordered + // ones — never dropped. + if (browserOps.groupsOrder) { + nextGroups = reorderGroups(nextGroups, browserOps.groupsOrder); + } + + await Groups.save(nextGroups); + endPhase(); + } + + // --- tabs create: group by target group so each batch gets the right + // groupId/windowId (mirrors cloud.js:316-324). Stamp the REMOTE uid onto each + // created tab afterwards so its identity is stable across the next sync round + // (otherwise apply→re-pull would churn: remove + recreate the same tab forever). + // + // IDEMPOTENT CREATE (anti-duplication): a tab whose uid is ALREADY live must never + // be re-created. The planner emits a create when a resolved uid is absent from the + // snapshot localState, but uid↔cloud correspondence can drift (partial reset, or a + // Firefox session restore minting fresh uids while the cloud still holds old-uid + // records) so that the live cache here disagrees with that snapshot — re-creating + // then puts a second copy next to the existing tab (the duplicate-in-active-group + // bug). Build the live-by-uid index ONCE and skip any create whose uid is already + // live; a tab that already exists at most needs a move/update, which other ops do. + const endCreatePhase = browserOps.tabsToCreate.length ? beginApplyPhase('tabs-create', log) : null; + const liveByUidForCreate = await buildLiveTabIndexByUid(); + + const createsByGroup = new Map(); + for (const tab of browserOps.tabsToCreate) { + const groupId = tab.target?.groupId; + if (groupId == null) { + continue; + } + if (tab.uid != null && liveByUidForCreate.has(tab.uid)) { + log.log('idempotent create: skip already-live tab uid', tab.uid); + continue; + } + if (!createsByGroup.has(groupId)) { + createsByGroup.set(groupId, []); + } + createsByGroup.get(groupId).push(tab); + } + + for (const [groupId, allTabs] of createsByGroup) { + const groupWindowId = Cache.getWindowId(groupId); // null if group not loaded + + // URL-LESS / BLANK GUARD (anti-freeze): never create a bare tab from a url-less or + // trivial-blank sync record. A record with no syncable url would yield a loading + // about:blank tab (Tabs.create with no url) that carries a uid the cloud never had — + // re-created every cycle (the unstamped-create flood). The cloud is structurally + // clean of such records (verified against the gist), so this is a belt-and-suspenders + // guard against a malformed/partial record rather than an expected case; a dropped + // create is re-attempted on the next sync if the record gains a real url. + const tabs = allTabs.filter(tab => { + if (isUrlSyncable(unwrapStubUrl(tab.url))) { + return true; + } + log.log('skip create of url-less/blank tab record', tab.uid, tab.url); + return false; + }); + if (!tabs.length) { + continue; + } + + const toCreate = tabs.map(tab => ({ + url: tab.url, + title: tab.title, + cookieStoreId: tab.cookieStoreId, + index: tab.target?.index, + groupId, + windowId: groupWindowId, + favIconUrl: tab.favIconUrl, + // group-scoped pin flag rides the create; persisted as a session value so + // the group's next apply pins it in the right slot. We do NOT create it + // browser-pinned here: a not-loaded group keeps its tabs hidden, and a + // pinned tab can't be hidden — apply does the live pin when the group shows. + groupPinned: tab.pinned === true, + // sleep-by-default: create the tab asleep (discarded) unless the user's + // activate-options say otherwise (see tab-sleep.js). STG's create path + // already discards inactive-group tabs and refuses to discard the active + // group's foreground tab, so this only widens "asleep" to cases STG would + // otherwise force-load; it never force-loads the active group's tab. + discarded: shouldSleepSyncedTab(tab, false, sleepOptions), + })); + + const created = await Tabs.createMultiple(toCreate, true); + + // Stamp the REMOTE uid (+ lastModified) onto each created tab so identity is + // stable across the next sync round; otherwise apply→re-pull would churn + // (remove + recreate the same tab forever — the unstamped-create flood that left the + // M2 device with re-created tabs whose uids the cloud never had). + // + // createMultiple may re-order / drop tabs (per-window grouping, index re-sort), so we + // match in precision order: + // (1) STUB-AWARE url match: a created tab's url equals the source url, OR — for a + // privileged about: url that STG rendered as the moz-extension "unsupported URL" + // stub — its DECODED url equals the source url. This is the key fix: without the + // decode, an about:debugging/about:memory source (created.url = stub) never + // matched its source.url = about:… → left UNSTAMPED → re-created next cycle. + // (2) ORDER-STABLE positional fallback for anything still unmatched: `toCreate` was + // built in `tabs` order and createMultiple returns each window's tabs sorted by + // index (the order we requested), so pairing the remaining created tabs to the + // remaining sources IN ORDER reliably stamps tabs whose live url differs from + // both source and stub (e.g. a tab still at about:blank mid-navigation). + // A source we still can't pair (more sources than created tabs — a create failed) is + // left unstamped, but the idempotent-by-uid create guard above will NOT re-create it + // while a live same-url tab exists, so it cannot flood. + const createdPool = created.filter(Boolean); + const usedCreated = new Set(); + + const stamp = async (newTab, source) => { + await Cache.setTabUid(newTab.id, source.uid).catch(log.onCatch(['cant set uid', newTab.id], false)); + if (source.lastModified != null) { + await Cache.setTabLastModified(newTab.id, source.lastModified) + .catch(log.onCatch(['cant set lastModified', newTab.id], false)); + } + }; + + const unmatchedSources = []; + for (const source of tabs) { + const match = createdPool.find(t => !usedCreated.has(t.id) + && liveUrlMatchesSource(t.url, source.url)); + if (match) { + usedCreated.add(match.id); + await stamp(match, source); + } else { + unmatchedSources.push(source); + } + } + + const remainingCreated = createdPool.filter(t => !usedCreated.has(t.id)); + for (let k = 0; k < unmatchedSources.length && k < remainingCreated.length; k++) { + await stamp(remainingCreated[k], unmatchedSources[k]); + } + + // if the group isn't shown in a window, keep the created tabs hidden + if (!Groups.isLoaded(groupId)) { + await Tabs.hide(created, true).catch(log.onCatch(['cant hide tabs for group', groupId], false)); + } else if (tabs.some(tab => tab.pinned === true)) { + // the group is shown and at least one created tab is group-pinned → + // re-apply pin ordering so it lands after the global pinned region. + await Groups.applyGroupPinnedOrder(groupId) + .catch(log.onCatch(['cant apply group-pinned order', groupId], false)); + } + } + endCreatePhase?.(); + + // --- tabs move: resolve live tab ids by uid, move with skipTrackingFlag + if (browserOps.tabsToMove.length) { + const endPhase = beginApplyPhase('tabs-move', log); + const liveByUid = await buildLiveTabIndexByUid(); + + for (const move of browserOps.tabsToMove) { + const tabId = liveByUid.get(move.uid); + if (tabId == null) { + continue; // tab not live (resurrected ones surface as create, not move) + } + const windowId = Cache.getWindowId(move.target?.groupId); + const moveProps = {index: move.target?.index}; + if (Number.isFinite(windowId)) { + moveProps.windowId = windowId; + } + await Tabs.moveNative([{id: tabId}], moveProps, true) + .catch(log.onCatch(['cant move tab', move.uid], false)); + } + endPhase(); + } + + // --- tabs update: CONTENT changes (url/title/favIcon/group-pin/…) to a tab that + // ALREADY exists on this device. The per-tab analogue of groupsToUpdate; without it a + // content change to an existing peer tab never landed (the "half-wired" sync bug). + // Applied via the create path's mechanisms (Cache session values + a guarded live url + // navigation); see applyTabContentUpdate. Runs after move so it lands on the final tab. + if (browserOps.tabsToUpdate?.length) { + const endPhase = beginApplyPhase('tabs-update', log); + const liveByUid = await buildLiveTabRecordByUid(); + + for (const update of browserOps.tabsToUpdate) { + const liveTab = liveByUid.get(update.uid); + if (liveTab == null) { + continue; // not live (a resurrected tab surfaces as create, not update) + } + await applyTabContentUpdate(liveTab, update.target || {}, log); + } + endPhase(); + } + + // --- tabs remove: resolved-says-gone set ONLY. Silent so the removal isn't + // captured and so the UI reloads in one pass. + if (browserOps.tabsToRemove.length) { + const endPhase = beginApplyPhase('tabs-remove', log); + const liveByUid = await buildLiveTabIndexByUid(); + const removeIds = browserOps.tabsToRemove + .map(t => liveByUid.get(t.uid)) + .filter(id => id != null); + + if (removeIds.length) { + await Tabs.remove(removeIds.map(id => ({id})), true) + .catch(log.onCatch('cant remove tabs', false)); + } + endPhase(); + } + + // --- realize the RESOLVED group tab order. createMultiple may place created + // tabs out of order and the planner emits no `tabsToMove` for not-yet-local + // creates, so the live order can diverge from the resolved (authoritative) order + // for created and/or unloaded-group tabs. Reconcile every group the resolved + // state knows, matching live tabs to resolved tabs by uid. Runs AFTER creates + + // moves + removes so it orders the group's FINAL membership. + const endReconcile = beginApplyPhase('reconcile-group-tab-orders', log); + await reconcileGroupTabOrders(resolvedSnapshot, log); + endReconcile(); + + const endPinned = beginApplyPhase('pinned-ops', log); + await applyPinnedOps(browserOps, log, sleepOptions); + endPinned(); + + log.stop(); + } finally { + DeltaCapture.endApply(); + } +} + +/** + * Reconcile every resolved group's LIVE browser tab order to its RESOLVED (authoritative) + * order, for BOTH loaded and unloaded groups. The bug this fixes: on a clean sync the + * receiving machine creates a group's tabs via {@link Tabs.createMultiple}, which re-sorts + * by absolute window index (and, for an unloaded group, drops them into a shared hidden + * pool) — so the group can show its tabs SHUFFLED, while the planner emits no `tabsToMove` + * to fix it (the created tabs weren't "local" at plan time). Since a non-archived group's + * order lives ONLY in the browser tab indices (`Storage.set` wipes the stored `tabs`), the + * robust fix is to physically `moveNative` the group's tabs into the resolved sequence. + * + * We move the group's tabs AS AN ORDERED ARRAY to the group's current minimum index in one + * `moveNative` call (the same pattern `Groups.showGroup` uses): Firefox places an array of + * ids contiguously starting at that index, so the global-pinned / group-pinned / other-group + * offset ahead of the group is preserved and the tabs just permute among the slots they + * already occupy. Match is by `uid` (see {@link orderedGroupTabIds}). Conservative on + * failure: logs and never removes/loses a tab. + * + * @param {object} [resolvedSnapshot] - from {@link planSync} (`plan.resolvedSnapshot`). + * @param {object} log - parent logger context. + * @returns {Promise} + */ +async function reconcileGroupTabOrders(resolvedSnapshot, log) { + const resolvedGroups = resolvedSnapshot?.groups; + if (!Array.isArray(resolvedGroups) || !resolvedGroups.length) { + return; + } + + // resolved uid order per group id + const resolvedOrderByGroupId = new Map(); + for (const group of resolvedGroups) { + const uids = (Array.isArray(group.tabs) ? group.tabs : []) + .map(tab => tab?.uid) + .filter(uid => uid != null); + if (uids.length) { + resolvedOrderByGroupId.set(group.id, uids); + } + } + if (!resolvedOrderByGroupId.size) { + return; + } + + // live tabs per group (with their current browser index, sorted by it via Groups.load). + const {groups: liveGroups} = await Groups.load(null, true); + + for (const group of liveGroups) { + if (group.isArchive || !Array.isArray(group.tabs)) { + continue; + } + const resolvedUidOrder = resolvedOrderByGroupId.get(group.id); + if (!resolvedUidOrder) { + continue; + } + + const orderedIds = orderedGroupTabIds(resolvedUidOrder, group.tabs); + if (!orderedIds.length) { + continue; // <2 tabs or already in order + } + + // anchor: the smallest index this group's tabs currently occupy, so the move + // preserves the offset of anything ahead of the group (pinned / other groups). + const minIndex = Math.min(...group.tabs.map(tab => tab.index).filter(Number.isFinite)); + if (!Number.isFinite(minIndex)) { + continue; + } + + await Tabs.moveNative(orderedIds.map(id => ({id})), {index: minIndex}, true) + .catch(log.onCatch(['cant reorder group tabs to resolved order', group.id], false)); + + // a LOADED group with group-pinned tabs: the moveNative above ignores the pin + // boundary, so re-settle the group-pinned region (pins flagged tabs right after the + // global pinned tabs, before this group's normal tabs). No-op when not loaded / none pinned. + if (Groups.isLoaded(group.id) && group.tabs.some(tab => tab.groupPinned)) { + await Groups.applyGroupPinnedOrder(group.id) + .catch(log.onCatch(['cant re-apply group-pinned order after reorder', group.id], false)); + } + } +} + +/** + * Apply the planner's pinned `browserOps` (`pinnedToCreate/Move/Remove`) to the live + * browser through STG's skip machinery. Global pinned tabs are window-global; created + * ones go into the last-focused NORMAL window (reusing {@link Windows.getLastFocusedNormalWindow}), + * with `pinned:true` and `skipTrackingFlag` so capture does not re-record them. STG group + * tabs are never created here — the planner only emits pinned ops for the global pinned + * section. Order: create, then update (content), then move (reorder), then remove (silent). + * + * @param {object} browserOps - from {@link planSync}. + * @param {object} log - the parent logger context. + * @param {object} sleepOptions - the LOCAL-ONLY sleep/activate option bag (read by + * {@link applyBrowserOps}); decides whether created pinned tabs are asleep or loaded. + * @returns {Promise} + */ +async function applyPinnedOps(browserOps, log, sleepOptions = {}) { + const toCreate = browserOps.pinnedToCreate || []; + const toMove = browserOps.pinnedToMove || []; + const toRemove = browserOps.pinnedToRemove || []; + const toUpdate = browserOps.pinnedToUpdate || []; + + if (!toCreate.length && !toMove.length && !toRemove.length && !toUpdate.length) { + return; + } + + // --- pinned create: window-global ⇒ last-focused normal window. Stamp the REMOTE + // uid (+ lastModified) onto each created tab so identity is stable across rounds + // (otherwise apply→re-pull would churn: remove + recreate forever). Same matching + // strategy as the grouped create path. + if (toCreate.length) { + // IDEMPOTENT CREATE (anti-duplication): skip creating a pinned tab whose uid is + // already live. Mirrors the grouped create guard — uid drift must never spawn a + // second copy of a tab that already exists; at worst it needs a move/update. + const liveByUidForCreate = new Map( + (await getLivePinnedTabs()).filter(t => t.uid != null).map(t => [t.uid, t.id]) + ); + const pinnedToActuallyCreate = toCreate.filter(tab => { + if (tab.uid != null && liveByUidForCreate.has(tab.uid)) { + log.log('idempotent pinned create: skip already-live pinned uid', tab.uid); + return false; + } + // URL-LESS / BLANK GUARD (anti-freeze): never create a bare pinned tab from a + // url-less / trivial-blank record. Pinned tabs are created LOADED (Firefox refuses a + // discarded pinned tab), so a url-less record would spawn a loading about:blank pinned + // tab carrying a uid the cloud never had — re-created every cycle (the about:blank + // pinned-tab flood seen on the M2 device). The cloud is verified clean of such records. + if (!isUrlSyncable(unwrapStubUrl(tab.url))) { + log.log('skip create of url-less/blank pinned record', tab.uid, tab.url); + return false; + } + return true; + }); + + const windowId = await Windows.getLastFocusedNormalWindow() + .catch(log.onCatch('cant resolve normal window for pinned create', false)); + + const createProps = pinnedToActuallyCreate.map(tab => ({ + url: tab.url, + title: tab.title, + cookieStoreId: tab.cookieStoreId, + index: tab.target?.index, + pinned: true, + // Sleep-by-default: create the synced pinned tab ASLEEP (discarded) so it shows + // title+favicon but never navigates/loads until the user activates it — unless + // the user's activate-options say otherwise (see tab-sleep.js). Besides the UX + // win, an asleep pinned tab can't redirect (e.g. a logged-out + // meet.google.com → accounts.google.com) and so can't be re-captured as a + // pinned.modify that pollutes the cloud URL. Tabs.create only honors `discarded` + // for tabs with a real, restorable URL (about:/long urls still load). + discarded: shouldSleepSyncedTab(tab, true, sleepOptions), + windowId: Number.isFinite(windowId) ? windowId : undefined, + favIconUrl: tab.favIconUrl, + })); + + const created = await Tabs.createMultiple(createProps, true); + const createdPool = (created || []).filter(Boolean); + const usedCreated = new Set(); + + const stamp = async (newTab, source) => { + await Cache.setTabUid(newTab.id, source.uid).catch(log.onCatch(['cant set pinned uid', newTab.id], false)); + if (source.lastModified != null) { + await Cache.setTabLastModified(newTab.id, source.lastModified) + .catch(log.onCatch(['cant set pinned lastModified', newTab.id], false)); + } + }; + + // UID-STABLE MATCHING (anti-duplication + anti-flood): stamp each source uid onto the + // created tab that actually IS that source. `createdPool` is NOT in source order — Tabs + // .createMultiple regroups/re-sorts by window+index — and a freshly-created LOADED pinned + // tab's `url` is UNRELIABLE: Firefox creates pinned tabs loaded (it refuses a discarded + // pinned tab), so while the tab is still navigating, `created.url` is `about:blank`, not the + // target url. That is exactly why URL-only matching left the M2 device's claude.ai/gmail + // pinned tabs UNSTAMPED → re-created every cycle with fresh uids the cloud never had (the + // about:blank pinned-tab flood). So we match in precision order, ending in an order-stable + // index fallback that does NOT depend on the (possibly about:blank) live url: + // (1) STUB-AWARE url + matching index — unambiguous even when the url repeats; the decode + // lets a privileged about: source (created.url = stub) match its source.url = about:… + // (2) STUB-AWARE url alone, only when that url is unique among the create sources; + // (3) ORDER-STABLE index fallback: pair the still-unmatched sources to the still-unmatched + // created tabs IN ASCENDING-INDEX ORDER. createMultiple returns the window's pinned + // tabs sorted by index, and each source was requested at its own target.index, so this + // reliably stamps a tab whose live url is still about:blank mid-navigation. + // Only if a source STILL can't be paired (fewer created tabs than sources — a create failed) + // is the stamp skipped; the idempotent-by-uid guard above then prevents a re-create flood. + const indexOf = source => source.target?.index; + const matchesUrl = (t, source) => liveUrlMatchesSource(t.url, source.url); + const urlCount = new Map(); + for (const source of pinnedToActuallyCreate) { + urlCount.set(source.url, (urlCount.get(source.url) || 0) + 1); + } + + const stillUnmatched = []; + for (const source of pinnedToActuallyCreate) { + // (1) stub-aware url + matching index — unambiguous even when the url repeats. + let match = Number.isInteger(indexOf(source)) + ? createdPool.find(t => !usedCreated.has(t.id) && matchesUrl(t, source) && t.index === indexOf(source)) + : undefined; + + // (2) stub-aware url alone, only if this url is not shared by another create source. + if (!match && urlCount.get(source.url) === 1) { + match = createdPool.find(t => !usedCreated.has(t.id) && matchesUrl(t, source)); + } + + if (match) { + usedCreated.add(match.id); + await stamp(match, source); + } else { + stillUnmatched.push(source); + } + } + + // (3) ORDER-STABLE index fallback for anything URL matching couldn't pair (e.g. a tab still + // at about:blank mid-load). Sort both remaining lists by index and pair positionally — this + // is what stamps the navigating pinned tabs that URL matching used to skip → no more flood. + if (stillUnmatched.length) { + const remainingCreated = createdPool + .filter(t => !usedCreated.has(t.id)) + .sort((a, b) => (a.index ?? 0) - (b.index ?? 0)); + const remainingSources = stillUnmatched + .slice() + .sort((a, b) => (indexOf(a) ?? 0) - (indexOf(b) ?? 0)); + for (let k = 0; k < remainingSources.length && k < remainingCreated.length; k++) { + usedCreated.add(remainingCreated[k].id); + await stamp(remainingCreated[k], remainingSources[k]); + } + const leftover = remainingSources.length - remainingCreated.length; + if (leftover > 0) { + log.log('pinned create: more sources than created tabs, left unstamped', leftover); + } + } + } + + // --- pinned update / move / remove: resolve live pinned tabs by uid. + if (toUpdate.length || toMove.length || toRemove.length) { + const livePinned = await getLivePinnedTabs(); + const idByUid = new Map(livePinned.filter(t => t.uid != null).map(t => [t.uid, t.id])); + const tabByUid = new Map(livePinned.filter(t => t.uid != null).map(t => [t.uid, t])); + + // --- pinned update: CONTENT change (url/title/favIcon) on an existing global pinned + // tab. Same mechanism as the grouped update path (Cache for title/favicon, a guarded + // tabs.update for url that never wakes a discarded tab). `pinned`/`loaded` are filtered + // out by the planner for global pinned tabs, so only url/title/favIcon arrive here. + for (const update of toUpdate) { + const liveTab = tabByUid.get(update.uid); + if (liveTab == null) { + continue; // not live (a resurrected pinned tab surfaces as create, not update) + } + await applyTabContentUpdate(liveTab, update.target || {}, log); + } + + for (const move of toMove) { + const tabId = idByUid.get(move.uid); + if (tabId == null) { + continue; // not live (a resurrected pinned tab surfaces as create, not move) + } + await Tabs.moveNative([{id: tabId}], {index: move.target?.index}, true) + .catch(log.onCatch(['cant move pinned tab', move.uid], false)); + } + + const removeIds = toRemove + .map(t => idByUid.get(t.uid)) + .filter(id => id != null); + + if (removeIds.length) { + await Tabs.remove(removeIds.map(id => ({id})), true) + .catch(log.onCatch('cant remove pinned tabs', false)); + } + } +} + +/** + * Query the live GLOBAL pinned tabs (across all normal windows) and hydrate each with + * its `uid`/`lastModified` session value, backfilling a uid where missing (pinned tabs + * don't get one from the group machinery). Mirrors `background.js` createBackup: only + * tabs whose URL passes the syncable allow-list ({@link module:sync/delta/url-sync} + * `isUrlSyncable`) are considered — non-trivial about: urls DO sync (rendered as the stub + * on apply), trivial new-tab states do not. Used for both `localState.pinnedTabs` and as + * the `uid -> live tabId` index for pinned move/remove. + * + * CRITICAL — GLOBAL vs GROUP-SCOPED pinned: `browser.tabs.query({pinned:true})` returns + * BOTH truly-global pinned tabs (no STG group) AND the group-pinned tabs of whatever group + * is currently ACTIVE (while a group is active, its group-pinned tabs are genuinely + * browser-pinned). Only the GLOBAL ones belong to `localState.pinnedTabs` / the pinned + * `OPS.PINNED_*` section — group-pinned tabs sync as NORMAL group tabs (+ groupPinned flag) + * through the groups path. So we EXCLUDE any tab STG tracks in a group (`Cache.getTabGroup`), + * exactly as the capture path discriminates (see delta-capture.js `tabAdded`). Without this + * filter an active group's pinned tabs leak into the global pinned set, which (a) lets the + * planner emit `pinnedToRemove` against a uid that is really a live GROUP tab — and the + * uid→tabId index built from this set would then resolve+DELETE that real group tab; (b) + * double-counts a uid as both a group `tab.add` and a global `pinned.add` on bootstrap → + * duplicate pinned copies on the peer; (c) shifts globals' live pinned indices → a + * never-converging `pinnedToMove` churn. Filtering here fixes all three at the source. + * + * @returns {Promise} live GLOBAL (groupless) pinned tabs (each with id, uid, url, title, ...). + */ +async function getLivePinnedTabs() { + // windowId=null ⇒ all normal windows; pinned=true ⇒ only pinned tabs; hidden=null ⇒ any. + // (Tabs.get skips session hydration for pinned, so we hydrate uid/lastModified below.) + let pinnedTabs = await Tabs.get(null, true, null).catch(() => []); + + // decode the "unsupported URL" stub page back to its embedded original (so a + // stub-rendered about: tab keeps its original identity instead of bootstrapping the + // moz-extension stub url), then gate on the syncable allow-list. See url-sync.js. + return Promise.all( + pinnedTabs + // keep ONLY truly-global pinned tabs: drop any tab STG tracks in a group (an + // active group's group-pinned tabs are browser-pinned but belong to the groups + // path, not the global pinned section). See the doc comment above. + .filter(tab => !Cache.getTabGroup(tab.id)) + .map(tab => { + tab.url = unwrapStubUrl(tab.url); + return tab; + }) + .filter(tab => isUrlSyncable(tab.url)) + .map(async tab => { + const uid = Cache.getTabUid(tab.id) || await Cache.setTabUid(tab.id).catch(() => null); + tab.uid = uid; + tab.lastModified = Cache.getTabLastModified(tab.id); + return tab; + }) + ); +} + +/** + * Apply the planner's resolved option diff (`optionsToApply`, a `{key: value}` bag) to + * this device through STG's REAL save path — background.js `saveOptions(...)` — so all + * side effects run (it persists via `Storage.set`, refreshes `options`, re-broadcasts + * hotkeys to content scripts, resets the backup/sync alarms, updates the temporary + * container title). We bracket the call with {@link module:sync/delta/delta-capture.beginApply}/ + * `endApply` so the `option.set` deltas that `saveOptions` would otherwise re-capture + * (it calls `DeltaCapture.optionsChanged`) are suppressed — no feedback loop. Nothing to + * do when the diff is empty. + * + * @param {object} optionsToApply - from {@link planSync}. + * @returns {Promise} + */ +async function applyOptions(optionsToApply) { + const keys = Object.keys(optionsToApply || {}); + if (!keys.length) { + return; + } + + const log = logger.start('applyOptions', {keys}); + + DeltaCapture.beginApply(); + try { + // real save path → persists + runs all side effects (alarms/hotkeys/container); + // suppressed from re-capture by the beginApply bracket. + await backgroundSelf.saveOptions(optionsToApply); + log.stop(); + } catch (e) { + log.logError('cant apply options', e); + } finally { + DeltaCapture.endApply(); + } +} + +/** + * Build the OUTBOUND local→portable container mapper + the registry it populates, from + * the live local containers. The registry ({portableKey: {name,color,icon}}) is what + * travels in the snapshot so a receiving device can find-or-create the matching container. + * + * Seeds from the pulled snapshot's registry so container definitions contributed by OTHER + * devices survive into the next write even if this device has no local copy of them. + * + * @param {object} [pulledContainers] - the pulled snapshot's `containers` registry. + * @returns {{registry: object, mapToPortable: (cookieStoreId: string) => string}} + */ +function buildOutboundContainerMapping(pulledContainers) { + // real, non-temporary local containers keyed by cookieStoreId. + const localContainers = Containers.query({temporaryContainers: false}); + + const registry = {...(pulledContainers || {})}; + + const mapToPortable = makeOutboundMapper( + localContainers, + registry, + Containers.isDefault, + Containers.isTemporary, + ); + + return {registry, mapToPortable}; +} + +/** + * Build the INBOUND portable→local container mapper for a resolved snapshot's registry. + * The find-or-create is the IMPURE boundary: it reuses {@link module:containers.findExistOrCreateSimilar} + * (find a local container whose name+color+icon match, else create one), caching the + * cookieStoreId per portable identity for the round via its own `storageMap`. The default + * marker maps to this install's default `cookieStoreId`; the temporary marker maps to the + * local default too (conservative: a synced reference to a temporary container has no stable + * identity to recreate, so a stable, never-failing default is safer than spinning up a + * throwaway temporary container on every apply). + * + * @param {object} [registry] - resolved snapshot's `containers` registry. + * @returns {{mapper: (portableKey: string) => string, findOrCreateMap: Map}} + */ +function buildInboundContainerMapper(registry) { + // a single shared map so identical identities resolve to one created container per round. + const findOrCreateMap = new Map(); + + const findOrCreate = identity => { + // resolving synchronously is not possible (Containers.findExistOrCreateSimilar is + // async), so all identities are pre-resolved up-front in translateInboundContainers. + // This mapper is only used by the synchronous mapStateContainers/mapEventContainers + // walk AFTER the identities have been resolved into findOrCreateMap (see + // resolveInboundContainers); an unresolved key conservatively falls back to default. + const key = identity.name + identity.color + identity.icon; + return findOrCreateMap.has(key) ? findOrCreateMap.get(key) : Constants.DEFAULT_COOKIE_STORE_ID; + }; + + const mapper = makeInboundMapper( + registry || {}, + Constants.DEFAULT_COOKIE_STORE_ID, + findOrCreate, + () => Constants.DEFAULT_COOKIE_STORE_ID, + ); + + return {mapper, findOrCreateMap}; +} + +/** + * Pre-resolve (find-or-create) a local `cookieStoreId` for every portable identity in a + * resolved snapshot's container registry, populating `findOrCreateMap` so the synchronous + * inbound mapper can translate without awaiting. This is the impure half: it calls + * {@link module:containers.findExistOrCreateSimilar} per identity (which finds a name+color+icon + * match or creates the container). Conservative: a create failure leaves the identity + * unmapped, so the mapper falls back to the local default (never fails the sync). + * + * @param {object} registry - resolved snapshot's `containers` registry. + * @param {Map} findOrCreateMap - mutated in place ({identityKey: cookieStoreId}). + * @param {object} log - parent logger context. + * @returns {Promise} + */ +async function resolveInboundContainers(registry, findOrCreateMap, log) { + // Containers.findExistOrCreateSimilar(cookieStoreId, data, map) finds a local container + // whose name+color+icon match `data`, else creates one, caching into `map` keyed by the + // first arg. We pass a SYNTHETIC first arg (prefixed so it can never be read as a default + // cookieStoreId by its internal `isDefault` check, nor collide with a real cookieStoreId) + // and reuse one shared map so identical identities resolve to a single created container. + const containerStorageMap = new Map(); + + for (const [, identity] of Object.entries(registry || {})) { + const identityKey = identity.name + identity.color + identity.icon; + if (findOrCreateMap.has(identityKey)) { + continue; + } + const syntheticId = 'sync-container:' + identityKey; + const cookieStoreId = await Containers.findExistOrCreateSimilar(syntheticId, identity, containerStorageMap) + .catch(log.onCatch(['cant find-or-create container', identityKey], false)); + if (cookieStoreId) { + findOrCreateMap.set(identityKey, cookieStoreId); + } + } +} + +/** + * Translate every portable container key in the planner's `browserOps` + resolved option + * diff back to a real local `cookieStoreId`, IN PLACE. The impure boundary: it first + * find-or-creates the local containers for the resolved registry, then walks the ops with + * the synchronous inbound mapper. Tab creates/pinned creates carry `cookieStoreId`; group + * create/update carry the group container keys; `optionsToApply.defaultGroupProps` carries + * them inside its value. + * + * @param {object} browserOps - from {@link planSync} (mutated in place). + * @param {object} optionsToApply - from {@link planSync} (mutated in place). + * @param {object} containerRegistry - resolved snapshot's `containers` registry. + * @param {object} log - parent logger context. + * @returns {Promise} + */ +async function translateInboundContainers(browserOps, optionsToApply, containerRegistry, log) { + const {mapper, findOrCreateMap} = buildInboundContainerMapper(containerRegistry); + + await resolveInboundContainers(containerRegistry, findOrCreateMap, log); + + // groups: create + update carry the group container keys. + for (const props of browserOps.groupsToCreate || []) { + mapEventContainers({group: props}, mapper); + } + for (const props of browserOps.groupsToUpdate || []) { + mapEventContainers({group: props}, mapper); + } + + // tabs / pinned create carry cookieStoreId directly. + for (const tab of browserOps.tabsToCreate || []) { + if (tab.cookieStoreId != null) { + tab.cookieStoreId = mapper(tab.cookieStoreId); + } + } + for (const tab of browserOps.pinnedToCreate || []) { + if (tab.cookieStoreId != null) { + tab.cookieStoreId = mapper(tab.cookieStoreId); + } + } + + // resolved options: only defaultGroupProps carries container keys. + if (optionsToApply && optionsToApply.defaultGroupProps) { + mapStateContainers({options: optionsToApply}, mapper); + } +} + +/** + * Reorder a groups array to match the resolved group-id order. Groups whose id is in + * `order` are placed in that order (skipping ids that aren't present locally); any + * remaining local-only group keeps its relative order and is appended at the end so a + * group that exists only on this device is NEVER dropped by the reorder. + * + * @param {object[]} groups - current local groups (post create/update/remove). + * @param {Array<*>} order - resolved group-id order from the planner. + * @returns {object[]} a reordered shallow copy (same group object references). + */ +function reorderGroups(groups, order) { + const byId = new Map(groups.map(g => [g.id, g])); + const placed = new Set(); + const result = []; + + for (const id of order) { + const group = byId.get(id); + if (group && !placed.has(id)) { + result.push(group); + placed.add(id); + } + } + + for (const group of groups) { + if (!placed.has(group.id)) { + result.push(group); + placed.add(group.id); + } + } + + return result; +} + +/** + * PURE: compute the ordered live-tab-id sequence to realize a group's RESOLVED tab + * order in the browser. + * + * The receiving side's create path ({@link Tabs.createMultiple}) may place created tabs + * in the WRONG ORDER for a group — it re-sorts by ABSOLUTE window index per window and, + * for an UNLOADED group, the created tabs land hidden in a shared pool whose absolute + * indices don't encode the group-relative order. `Storage.set` then wipes a non-archived + * group's stored `tabs` (it is re-derived from the live browser indices on every + * `Groups.load`), so the only durable source of a group's order is the BROWSER INDEX of + * its tabs. The planner's `tabsToMove` never fixes this because freshly created tabs + * aren't "local" yet at plan time, so no move is emitted for them. We therefore reconcile + * the live order to the resolved order explicitly, AFTER apply, by `moveNative`-ing the + * group's tabs into the resolved sequence (works for loaded AND unloaded groups, since + * `Groups.load` sorts by browser index either way). + * + * Match is by `uid` (createMultiple may reorder/drop tabs, so creation order is not + * trustworthy — we stamp the remote uid onto each created tab and key off that). Resolved + * uids with no live tab are skipped; live tabs whose uid is NOT in the resolved order keep + * their relative order and are appended after the resolved ones (never dropped). + * + * @param {Array<*>} resolvedUidOrder - the resolved group's tab uids, in authoritative order. + * @param {object[]} liveTabs - the group's live tabs `{id, uid}` (any order). + * @returns {number[]} live tab ids in the order they should appear in the group, or `[]` + * when there is nothing to do (fewer than 2 tabs, or the live order already matches). + */ +export function orderedGroupTabIds(resolvedUidOrder, liveTabs) { + const live = (Array.isArray(liveTabs) ? liveTabs : []).filter(t => t && t.id != null); + if (live.length < 2) { + return []; // 0/1 tabs ⇒ no ordering to realize + } + + const byUid = new Map(); + for (const t of live) { + if (t.uid != null && !byUid.has(t.uid)) { + byUid.set(t.uid, t); + } + } + + const placed = new Set(); + const ordered = []; + + for (const uid of (Array.isArray(resolvedUidOrder) ? resolvedUidOrder : [])) { + const t = byUid.get(uid); + if (t && !placed.has(t.id)) { + ordered.push(t); + placed.add(t.id); + } + } + // local-only tabs (uid not in resolved order) keep their current relative order, appended. + for (const t of live) { + if (!placed.has(t.id)) { + ordered.push(t); + placed.add(t.id); + } + } + + const orderedIds = ordered.map(t => t.id); + + // no-op if the live order already equals the desired order (avoid needless moves). + const sameOrder = orderedIds.length === live.length + && orderedIds.every((id, i) => id === live[i].id); + + return sameOrder ? [] : orderedIds; +} + +/** + * Build a `uid -> live tabId` index from the current cache. Used to translate the + * planner's uid-keyed ops into real browser tab ids at apply time. + * @returns {Promise>} + */ +async function buildLiveTabIndexByUid() { + const {groups} = await Groups.load(null, true); + const byUid = new Map(); + for (const group of groups) { + if (group.isArchive || !Array.isArray(group.tabs)) { + continue; + } + for (const tab of group.tabs) { + if (tab.uid != null && tab.id != null) { + byUid.set(tab.uid, tab.id); + } + } + } + return byUid; +} + +/** + * Build a `uid -> live group tab` index (the FULL hydrated browser tab, carrying `id`, + * `groupId`, `discarded`, `cookieStoreId`, …) for the `tabsToUpdate` apply path, which + * needs more than the bare id `buildLiveTabIndexByUid` returns (e.g. to avoid waking a + * discarded tab when applying a url change). Archived groups are skipped. + * @returns {Promise>} + */ +async function buildLiveTabRecordByUid() { + const {groups} = await Groups.load(null, true); + const byUid = new Map(); + for (const group of groups) { + if (group.isArchive || !Array.isArray(group.tabs)) { + continue; + } + for (const tab of group.tabs) { + if (tab.uid != null && tab.id != null) { + byUid.set(tab.uid, tab); + } + } + } + return byUid; +} + +/** + * Apply ONE `tabsToUpdate` entry's content changes to a live tab through the SAME + * mechanism the create path uses (Cache session values for STG-display fields; a guarded + * `tabs.update` only for the genuinely live-settable `url`). All session writes funnel + * through `Cache.*` so nothing here is captured as a fresh delta (and we are already inside + * the `beginApply` bracket regardless). Conservative throughout — a single field failing is + * logged and never aborts the rest. + * + * - `title` is NOT settable via `tabs.update` (Firefox only honors `title` on a discarded + * tab at CREATE time). So we update STG's cached title via {@link Cache.setTab}, which is + * what STG's UI / a discarded tab's display reads. + * - `favIconUrl` is likewise a session/cache field: {@link Cache.setTabFavIcon} (which, as + * on the create path, persists `data:` favicons to sessions and updates the in-memory + * cache). Non-`data:` favicons refresh from the live page on load, same as create. + * - `url`: settable live, but we MUST NOT force-load a discarded/unloaded tab. So for a + * discarded tab we update the cached url only (mirrors how STG stores an unloaded tab's + * url; it navigates there when the user wakes it); a loaded tab is navigated via + * `tabs.update` (the content genuinely changed elsewhere — same as a created tab's url). + * - `cookieStoreId`: changing a live tab's container requires recreating the tab (tabs.update + * can't move containers); doing that here risks losing tab state, so we DO NOT reopen — + * the cached value is refreshed and the container converges on the next create/reload. + * - `pinned` (group-pin): {@link Cache.setTabGroupPinned}; if the group is loaded, reflect + * it live via {@link Groups.applyGroupPinnedOrder} (pin) / a Groups re-settle (unpin). + * - `loaded`: a hint only; we never force-load (sleep-by-default UX, see tab-sleep.js). + * + * @param {object} liveTab - the hydrated live tab (from {@link buildLiveTabRecordByUid}). + * @param {object} target - the changed-fields bag from the planner's `tabsToUpdate` entry. + * @param {object} log - parent logger context. + * @returns {Promise} + */ +async function applyTabContentUpdate(liveTab, target, log) { + const liveId = liveTab.id; + + // url/title/favicon → STG cache (display) + a guarded live navigation for url only. + if (Object.hasOwn(target, 'url') || Object.hasOwn(target, 'title') || Object.hasOwn(target, 'favIconUrl')) { + const nextUrl = Object.hasOwn(target, 'url') ? target.url : liveTab.url; + const nextTitle = Object.hasOwn(target, 'title') ? target.title : liveTab.title; + + // refresh STG's cached url/title (display + identity); never wakes the tab. + Cache.setTab({ + id: liveId, + url: nextUrl, + title: nextTitle, + favIconUrl: Object.hasOwn(target, 'favIconUrl') ? target.favIconUrl : liveTab.favIconUrl, + cookieStoreId: liveTab.cookieStoreId, + status: liveTab.status, + }); + + if (Object.hasOwn(target, 'favIconUrl')) { + await Cache.setTabFavIcon(liveId, target.favIconUrl) + .catch(log.onCatch(['cant set favIcon (update)', liveId], false)); + } + + // a URL change is settable LIVE, but only navigate a LOADED tab — navigating a + // discarded tab would force it to load (defeats sleep-by-default and can trigger + // a redirect that re-pollutes the synced url). A discarded tab keeps the cached + // url set above and navigates there when the user wakes it. + // + // NO-OP CONVERGENCE GUARD (fix for "loads infinitely"): only navigate when the live + // tab's CURRENT url actually differs from the target. The planner can re-emit a + // `tabsToUpdate{url}` for a tab that is ALREADY at that url (e.g. the cloud url X was + // applied, the page server-redirected to Y, and until the cloud learns Y the planner + // keeps diffing cloud-X vs live-Y) — re-issuing `tabs.update(url:X)` every cycle then + // re-navigates the tab perpetually (the spinner the user sees). Comparing first makes a + // repeat-apply a true no-op, so a stable tab incurs ZERO browser ops. We unwrap the + // live url through the "unsupported URL" stub so a stub-rendered about: tab compares by + // its embedded original identity (which is what `target.url` carries) and is not + // needlessly re-navigated. The redirect case itself converges via the narrowed echo + // guard (the redirect target is now captured + pushed, so the cloud learns Y). + if (Object.hasOwn(target, 'url') && liveTab.discarded !== true + && shouldNavigateLiveTabUrl(liveTab.url, target.url)) { + await browser.tabs.update(liveId, {url: target.url}) + .catch(log.onCatch(['cant update tab url', liveId], false)); + } + } + + // group-pin flag (the per-group pin, NOT global pinned). Delegate to the canonical + // toggle: it persists the cache flag AND — when the group is loaded — pins (placing the + // tab at the front of the group's pinned block) or unpins+re-settles it live, the exact + // same path the user's toggle uses. Its internal pin/unpin uses Tabs.skipTracking and the + // delta it would emit is suppressed by the surrounding beginApply() bracket, so this does + // not re-capture. When the group is NOT loaded it just sets the flag. We compare against + // the current cache value to avoid a redundant toggle (idempotent re-apply). + if (Object.hasOwn(target, 'pinned') && Cache.getTabGroupPinned(liveId) !== (target.pinned === true)) { + await Groups.setTabGroupPinned(liveId, target.pinned === true) + .catch(log.onCatch(['cant set group-pin (update)', liveId], false)); + } + + // `cookieStoreId`/`loaded`: intentionally NOT applied to the live tab (see fn docs). +} + +/** + * The delta-era replacement for `cloud.js` `synchronization()`. Mirrors its + * scaffolding: the same `send('sync-*')` broadcast (imported from cloud.js so the UI + * / background progress wiring is identical) and the same `CloudError` wrapping. It + * carries its OWN `inProgress` guard (the legacy `synchronization()` guard belongs to + * a path we no longer call) so a second trigger while a delta sync runs is a no-op. + * + * @returns {Promise} syncResult ({ok, progress, ...}). + */ +let inProgress = false; + +// How soon (in minutes) to reschedule a sync cycle that DEFERRED because the user was +// mutating groups/tabs at apply time. Short — the work (pull/plan) was already done and is +// cheap to repeat; we just want to retry once the user's burst settles. Reuses the existing +// `cloud-retry` alarm (background.js `onAlarm` → cloudSync TRIGGER_RETRY) so no new wiring. +const USER_DEFER_RESCHEDULE_MINUTES = 0.2; // ~12s + +// How soon (in minutes) to reschedule a sync cycle that could NOT acquire the advisory lock +// because a peer holds it. Short (~30s per the design): the peer's cycle finishes well under +// the lock TTL, so we just want to retry shortly after it releases. Reuses the existing +// `cloud-retry` alarm, exactly like the user-defer reschedule above (no new scheduler). +const LOCK_CONTENDED_RESCHEDULE_MINUTES = 0.5; // ~30s + +/** + * Reschedule a soon sync after a user-active DEFER, reusing the existing retry alarm. Best + * effort: a failure to (re)create the alarm only delays the next sync to the regular + * periodic alarm — never an error. Not awaited on the hot path's correctness. + * @param {object} log - parent logger context. + */ +async function rescheduleSoonAfterDefer(log) { + try { + await browser.alarms.create(ALARM_NAME_RETRY, { + delayInMinutes: USER_DEFER_RESCHEDULE_MINUTES, + }); + } catch (e) { + log.warn('cant reschedule deferred sync; will run on next periodic alarm', String(e)); + } +} + +/** + * Reschedule a soon sync after we could NOT acquire the advisory lock (a peer holds it), + * reusing the same `cloud-retry` alarm. Best effort — a failure only delays the next sync to + * the regular periodic alarm. + * @param {object} log - parent logger context. + */ +async function rescheduleSoonAfterLockContention(log) { + try { + await browser.alarms.create(ALARM_NAME_RETRY, { + delayInMinutes: LOCK_CONTENDED_RESCHEDULE_MINUTES, + }); + } catch (e) { + log.warn('cant reschedule lock-contended sync; will run on next periodic alarm', String(e)); + } +} + +/** + * Reset THIS device's local delta-sync state so the next sync behaves like a first + * sync. A recovery action for a bad/inconsistent local state that lets a user (or + * tester) recover WITHOUT making a fresh Firefox profile. + * + * Clears, for `selfDeviceId = getDeviceId()`: + * - the durable baseline (`deltaBaseline:`) — so removals are re-gated from an + * EMPTY baseline, which authorizes NO removals (a delete propagates only once an + * item is reconciled as synced again). This is the data-loss-safe default. + * - the last-pushed-seq mark (`deltaLastPushedSeq:`) — so the local log is + * considered fully un-pushed and gets re-uploaded. + * - the base watermark (`deltaBaseWatermark:`). + * - the local event log (`DeltaLog.clear()`) — events + seq reset. + * + * The deviceId itself is PRESERVED (kept in the same store but under a different key). + * Cloud is untouched: the gist/snapshot/delta files are not modified or deleted here + * (delete those manually on GitHub if a full reset is wanted). After reset, the next + * sync re-establishes the baseline from the resolved snapshot and bootstrap-uploads + * the current local groups/tabs. + * + * E2 — RESET vs CLOUD-WATERMARK TRAP. Clearing local lastSeq (via DeltaLog.clear) means + * the next appends re-issue seq 1..N, but the AUTHORITATIVE `watermark[self]=N` lives in + * the CLOUD snapshot, which reset cannot touch. replay() dedups every event with + * `seq <= watermark[self]` (replay.js rule 4), so without intervention every re-issued + * event would be SILENTLY SKIPPED until seq organically climbs past N — losing the + * post-reset local changes. We fix this with the LEAST-SURPRISING, data-preserving option + * (keeping the deviceId stable, as documented above, rather than minting a new one and + * orphaning the cloud delta file): set a durable per-device "reset pending" flag here, and + * on the NEXT sync force a full pull (the conditional fast path can't see the cloud + * watermark) so the flow can fast-forward this device's log strictly above the stale cloud + * watermark[self] BEFORE pushing (see DeltaLog.fastForwardSeqsAbove + the deltaSynchronization + * full path). The fast-forward never lowers a seq and is a no-op on the normal path, so the + * non-reset flow is unaffected, with no double-apply and no data loss. + * + * Refuses while a sync is in progress so it can't race the live pipeline. + * @returns {Promise<{ok: boolean, inProgress?: boolean}>} + */ +export async function resetSyncState() { + if (inProgress) { + return {ok: false, inProgress: true}; + } + + const log = logger.start(resetSyncState); + + const selfDeviceId = getDeviceId(); + + delete storage[baselineKey(selfDeviceId)]; + delete storage[lastPushedSeqKey(selfDeviceId)]; + delete storage[watermarkKey(selfDeviceId)]; + // a hard reset clears the whole local log, so any deferred-truncation marker (a seq into + // that log) is moot — drop it so a stale seq can't trim the post-reset re-issued log. + delete storage[pendingTruncateKey(selfDeviceId)]; + + await DeltaLog.clear(); + + // arm the E2 reconciliation for the next sync (see the doc-comment above and the + // RESET_PENDING_PREFIX flag handling in deltaSynchronization). + storage[resetPendingKey(selfDeviceId)] = '1'; + + log.stop('reset local delta-sync state (cloud untouched)', {selfDeviceId}); + + return {ok: true}; +} + +/** + * SINGLE SOURCE OF TRUTH (D-3) for "what kind of change does this plan carry?". + * + * The "is there any browser op" question was previously open-coded in THREE places + * with subtly DIFFERENT op subsets — the pre-apply backup gate, the menu-rebuild gate, + * and the `syncResult.changes.local` flag. That drift is exactly what caused B1/B2: + * two of those sites OMITTED `tabsToUpdate` / `pinnedToUpdate` (the only ops that + * overwrite an existing live tab's url/title/favicon/group-pin), so a content-only sync + * mutated the browser yet (B1) took NO pre-apply safety backup and (B2) reported + * `changes.local = false` so the UI never refreshed. + * + * Every caller now derives its boolean from THIS pure helper so they can never diverge + * again. The granular booleans returned: + * - `anyBrowserOp` — any live browser mutation: group/tab/pinned create/move/remove/ + * update/reorder, INCLUDING the previously-omitted tabsToUpdate / pinnedToUpdate. + * - `anyOption` — at least one resolved global option to apply. + * - `groupsChanged` — only the group-set ops (create/update/remove/reorder); used by + * the per-group menu rebuild, which cares ONLY about the group set, not tabs. + * - `mutatesBrowser` — `anyBrowserOp || anyOption`: the predicate the pre-apply backup + * and `changes.local` consume. + * + * Pure (no browser deref); exported for unit testing. + * + * @param {object} [browserOps] - `plan.browserOps`. + * @param {object} [optionsToApply] - `plan.optionsToApply`. + * @returns {{anyBrowserOp: boolean, anyOption: boolean, groupsChanged: boolean, mutatesBrowser: boolean}} + */ +export function summarizeOps(browserOps, optionsToApply) { + const ops = browserOps || {}; + const len = arr => (Array.isArray(arr) ? arr.length : 0); + + const groupsChanged = !!( + len(ops.groupsToCreate) || len(ops.groupsToUpdate) + || len(ops.groupsToRemove) || ops.groupsOrder + ); + + const anyBrowserOp = !!( + groupsChanged + || len(ops.tabsToCreate) || len(ops.tabsToMove) || len(ops.tabsToRemove) || len(ops.tabsToUpdate) + || len(ops.pinnedToCreate) || len(ops.pinnedToMove) || len(ops.pinnedToRemove) || len(ops.pinnedToUpdate) + ); + + const anyOption = Object.keys(optionsToApply || {}).length > 0; + + return { + anyBrowserOp, + anyOption, + groupsChanged, + mutatesBrowser: anyBrowserOp || anyOption, + }; +} + +/** + * Does this plan actually MUTATE the live browser? True iff any browser op is a + * create/move/remove/update/reorder OR there is at least one resolved option to apply. + * An idle/no-op sync (the common 5-min alarm case) returns false, so we never take a + * backup for it. Thin wrapper over {@link summarizeOps} — same source of truth as the + * `syncResult.changes.local` predicate and the menu-rebuild gate below. + * @param {object} plan - from {@link planSync}. + * @returns {boolean} + */ +function planMutatesBrowser(plan) { + return summarizeOps(plan.browserOps, plan.optionsToApply).mutatesBrowser; +} + +/** + * SAFETY NET (Feature 1): take a full local STG backup BEFORE delta-sync applies any + * browser-mutating change, so a bad sync can be rolled back via STG's normal restore. + * + * Reuses STG's existing backup serializer end-to-end — `background.js` `createBackup` + * (the exact code path the auto-backup alarm and the manual "create backup" button + * use) — invoked here with a rolling override path so it writes a BOUNDED round-robin + * set ({@link PRE_APPLY_BACKUP_SLOTS} slots) instead of growing unbounded. It runs in + * the background page (no DOM / no user gesture needed) and honors the user's + * configured backup location (downloads vs native host). + * + * Gated by the LOCAL-ONLY `syncBackupBeforeApply` option (default ON). Only fires when + * the plan actually mutates the browser (see {@link planMutatesBrowser}) so idle syncs + * don't spam backups. AWAITS completion before returning; the caller ABORTS the apply + * if this throws (a safety net that doesn't catch is worse than none). + * + * @param {object} plan - from {@link planSync}. + * @param {object} log - parent logger context. + * @returns {Promise} + * @throws if the backup itself fails (so the caller can abort before mutating). + */ +async function maybeBackupBeforeApply(plan, log) { + if (!planMutatesBrowser(plan)) { + return; // no-op sync ⇒ nothing to back up + } + + const {syncBackupBeforeApply} = await Storage.get('syncBackupBeforeApply'); + if (!syncBackupBeforeApply) { + return; // user opted out + } + + // round-robin slot so the on-disk set is bounded to PRE_APPLY_BACKUP_SLOTS files. + const prevSlot = Number(storage[PRE_APPLY_BACKUP_SLOT_KEY]); + const slot = Number.isInteger(prevSlot) && prevSlot >= 0 ? prevSlot % PRE_APPLY_BACKUP_SLOTS : 0; + + log.info('pre-apply safety backup', {slot}); + + // createBackup(includeTabFavIcons, includeTabThumbnails, isAutoBackup, filePathOverride). + // isAutoBackup=false ⇒ never skipped on empty groups and no autoBackup side effects; + // filePathOverride ⇒ non-interactive write to the rolling slot path. + // Throws on failure → propagates to the caller, which aborts the apply. + await backgroundSelf.createBackup(true, false, false, preApplyBackupFilePath(slot)); + + // advance the slot ONLY after a successful write, so a failed backup re-uses the + // same slot next time rather than silently skipping ahead. + storage[PRE_APPLY_BACKUP_SLOT_KEY] = (slot + 1) % PRE_APPLY_BACKUP_SLOTS; +} + +/** + * Gather THIS device's local sync inputs and run the BOOTSTRAP step — everything that is + * derivable WITHOUT pulling the cloud. Extracted so the conditional fast path (remote + * unchanged) can decide "do we have local pending to push?" without a download, while the + * full path reuses the exact same local state for the planner. + * + * Side effect: appends synthetic bootstrap events to the local delta log (idempotent; see + * {@link computeBootstrapEvents}). After it runs, `localPendingEvents` includes any such + * synthetic adds, so a never-synced local item is pushed even on the unchanged-remote path. + * + * @param {string} selfDeviceId + * @param {object} log - parent logger context. + * @returns {Promise<{localState: object, priorBaseline: object, localPendingEvents: object[], lastPushedSeq: number}>} + */ +async function gatherLocalPending(selfDeviceId, log) { + // this device's last-synced baseline (gates removals). Empty on first run. + const priorBaseline = loadBaseline(selfDeviceId); + + // live local state in snapshot shape, incl. this device's current values for the + // SYNCED option keys (per-device keys filtered out by syncedOptionKeys). + // includeFavIconUrl=true ⇒ each grouped tab carries its CURRENT favicon (live, or the + // session-stored one for a sleeping/discarded tab). This is what makes the sync CONVERGE + // the favicon: the planner diffs this live favicon against the resolved snapshot's folded + // value and emits an update when they differ — so the stored favicon always snaps to the + // tab's current value, even if the favicon-emit throttle dropped intermediate changes. + const {groups: loadedGroups} = await Groups.load(null, true, true); + const syncedKeys = syncedOptionKeys(Constants.ALL_OPTION_KEYS); + const allLocalOptions = await Storage.get(syncedKeys); + const localSyncedOptions = {}; + for (const key of syncedKeys) { + localSyncedOptions[key] = allLocalOptions[key]; + } + const livePinnedTabs = await getLivePinnedTabs(); + const localState = buildLocalState(loadedGroups, localSyncedOptions, livePinnedTabs); + + // BOOTSTRAP: emit synthetic add events for local groups/tabs this device never + // reconciled as synced AND that its local delta log doesn't already reference — so they + // get uploaded instead of lingering local-only. Idempotent (skipped if in baseline OR + // already in the log), so re-runs never double-add. + const localLogEvents = await DeltaLog.getEvents(); + const logUids = new Set(); + const logGroupIds = new Set(); + const logOptionKeys = new Set(); + for (const event of localLogEvents) { + if (event.groupId != null) { + logGroupIds.add(event.groupId); + } + if (event.group?.id != null) { + logGroupIds.add(event.group.id); + } + if (event.uid != null) { + logUids.add(event.uid); + } + if (event.tab?.uid != null) { + logUids.add(event.tab.uid); + } + if (event.op === DeltaLog.OPS.OPTION_SET && event.key != null) { + logOptionKeys.add(event.key); + } + } + + const bootstrapEvents = computeBootstrapEvents(localState, priorBaseline, logUids, logGroupIds, logOptionKeys); + for (const payload of bootstrapEvents) { + await DeltaLog.append(payload.op, payload); + } + if (bootstrapEvents.length) { + log.info('bootstrap-uploaded never-synced local items', {count: bootstrapEvents.length}); + } + + // RE-READ pending AFTER bootstrap appends so the synthetic adds (with their + // now-assigned seqs) are included in this round's push. + const lastPushedSeq = Number(storage[lastPushedSeqKey(selfDeviceId)]) || 0; + const localPendingEvents = await DeltaLog.getEventsSince(lastPushedSeq); + + return {localState, priorBaseline, localPendingEvents, lastPushedSeq}; +} + +/** + * CONDITIONAL FAST PATH: the remote gist was confirmed UNCHANGED since our last pull, so + * there is nothing new to download/replay/apply. We still must PUSH if THIS device has + * pending local events (a local change must propagate even when the remote is unchanged). + * + * Because the remote is unchanged, the cloud's copy of THIS device's delta file equals what + * we last pushed; the correct new self delta file is therefore simply our ENTIRE local log + * (= cloud-self + pending), container-mapped to portable — identical to what the full + * planner would emit for the self file (see plan-sync `buildFullLogs`/`deltaFileToWrite`). + * We write ONLY that delta file: the snapshot is already present + valid in the unchanged + * gist (it is re-consolidated by the next full-fetch cycle on this or another device). + * + * We DO NOT advance the watermark/baseline here — those record a fully-reconciled state, and + * this path deliberately skips plan/apply. Leaving them untouched (like the user-defer path) + * means the next full cycle reconciles them; correctness is never weakened, only deferred. + * + * @param {object} Cloud - provider instance. + * @param {string} selfDeviceId + * @param {object[]} localPendingEvents - this device's pending events (post-bootstrap). + * @param {number} lastPushedSeq + * @param {object} pulledContainers - currently null on this path; outbound mapping seeds an + * empty registry (the pending events carry their own local cookieStoreIds to translate). + * @param {object} log - parent logger context. + * @returns {Promise<{pushed: boolean}>} + */ +async function pushLocalPendingOnly(Cloud, selfDeviceId, localPendingEvents, lastPushedSeq, log) { + if (!localPendingEvents.length) { + return {pushed: false}; // remote unchanged AND nothing local to push ⇒ true no-op + } + + // The full self delta file = the entire local log (cloud-self == last-pushed under the + // unchanged precondition, so cloud-self + pending == all local events). Map outbound + // container ids to portable keys exactly as the full path does for the self file. + // DEEP-CLONE first: DeltaLog.getEvents() returns the in-memory event OBJECTS (only the + // array is copied), and mapEventContainers mutates cookieStoreId IN PLACE — writing the + // portable key back into the live local log would corrupt the next cycle's outbound + // mapping (and persist a portable id locally). Cloning isolates this round's push. + const {mapToPortable} = buildOutboundContainerMapping(null); + const allEvents = structuredClone(await DeltaLog.getEvents()); + for (const event of allEvents) { + mapEventContainers(event, mapToPortable); + } + + await Cloud.writeFiles({ + [deltaFileName(selfDeviceId)]: { + v: DeltaLog.SCHEMA_VERSION, + deviceId: selfDeviceId, + events: allEvents, + }, + }); + + // advance lastPushedSeq to the highest pending seq so we don't re-push next cycle. + const maxSelfSeq = localPendingEvents.reduce( + (max, e) => (e.seq > max ? e.seq : max), + lastPushedSeq, + ); + storage[lastPushedSeqKey(selfDeviceId)] = maxSelfSeq; + + // refresh the stored ETag from the post-push gist so the NEXT probe compares against + // the revision we just created (otherwise our own push reads back as "changed"). + await Cloud.refreshEtagFromWrite?.(); + + log.info('conditional fast path: pushed local pending without pull/apply', { + events: localPendingEvents.length, + }); + + return {pushed: true}; +} + +export async function deltaSynchronization() { + const syncResult = {ok: false}; + + if (inProgress) { + syncResult.inProgress = true; + return syncResult; + } + + const log = logger.start(deltaSynchronization); + let lastProgress = 0; + + const progress = percent => { + lastProgress = percent; + send('sync-progress', {progress: percent}); + }; + + // Advisory-lock state, hoisted so the `finally` can release whatever we acquired no + // matter which path (success / error / apply-watchdog) we exit through. `Cloud` is set + // inside the try (provider creation can throw), so the finally guards both. + let Cloud = null; + let lockAcquired = false; + + try { + inProgress = true; + send('sync-start'); + progress(1); + + const {syncOptionsLocation, syncProvider} = await Storage.get(['syncOptionsLocation', 'syncProvider']); + + // Read sync options from the same store as the legacy path: Firefox Sync + // storage when configured, otherwise local storage. Without this the token + // (saved in FF Sync by default on Firefox) reads back empty → githubInvalidToken. + if (syncOptionsLocation === Constants.SYNC_STORAGE_FSYNC && !SyncStorage.IS_AVAILABLE) { + const error = new CloudError('ffSyncNotSupported'); + storage.lastError = String(error); + log.throwError('sync not supported', error); + } + + const syncOptions = syncOptionsLocation === Constants.SYNC_STORAGE_FSYNC + ? await SyncStorage.get() + : await Storage.get(null, Constants.DEFAULT_SYNC_OPTIONS); + + try { + Cloud = createCloudProvider(syncProvider, syncOptions); + } catch (error) { + const cloudError = new CloudError(error.message, {cause: error}); + storage.lastError = String(cloudError); + log.throwError('create cloud provider instance', cloudError); + } + + const selfDeviceId = getDeviceId(); + + progress(10); + + // 0. LOCAL INPUTS + BOOTSTRAP (no network): build this device's local state and run + // the bootstrap step. Needed by BOTH the conditional fast path (to know if there is + // local pending to push) and the full path (planner inputs). See gatherLocalPending. + const {localState, priorBaseline, lastPushedSeq} = + await gatherLocalPending(selfDeviceId, log); + // re-read into a mutable binding: the E2 reset reconciliation below may rewrite the + // pending events' seqs after the pull, and container mapping mutates them in place. + let localPendingEvents = await DeltaLog.getEventsSince(lastPushedSeq); + + // E2: did a local resetSyncState arm the watermark-trap reconciliation? When set, the + // conditional fast path is BYPASSED (it can't read the cloud watermark) so we take the + // full pull and fast-forward this device's log above the stale cloud watermark[self]. + const resetPending = !!storage[resetPendingKey(selfDeviceId)]; + + // 0b. CONDITIONAL FAST PATH: ask the provider whether the remote changed since our + // last pull (HTTP ETag / If-None-Match). FAIL-SAFE: `isUnchangedSince` returns true + // ONLY on a positive "unchanged" confirmation (no stored marker / first sync / + // discovery / any transport error all return false ⇒ full fetch below). When it is + // unchanged we SKIP the download + replay/plan/apply entirely, but STILL push this + // device's pending local events so a local change propagates. The provider may not + // implement the probe at all (optional contract) ⇒ treated as "changed" (full fetch). + // SKIPPED while a reset is pending (resetPending), see E2 above. + const remoteUnchanged = !resetPending + && (Cloud.isUnchangedSince ? await Cloud.isUnchangedSince() : false); + if (remoteUnchanged) { + const {pushed} = await pushLocalPendingOnly( + Cloud, selfDeviceId, localPendingEvents, lastPushedSeq, log, + ); + + progress(100); + syncResult.ok = true; + syncResult.progress = 100; + syncResult.skippedPull = true; + // local = applied nothing this cycle (pull/apply skipped); cloud = pushed iff we + // had pending local events to propagate. + syncResult.changes = {local: false, cloud: pushed}; + + send('sync-end', syncResult); + log.stop('remote unchanged: skipped pull/apply', {pushedLocalPending: pushed}); + return syncResult; + } + + // 0c. ADVISORY LOCK. Serialize the snapshot-writing full cycle across devices so two + // don't write the snapshot concurrently and clobber it (GitHub has no atomic CAS). It + // is ADVISORY/best-effort — a crashed holder is reclaimed via the server-clock TTL, and + // deferred self-truncation (compaction.js) is the real data-safety backstop. Acquired + // BEFORE the pull so the whole pull→apply→push runs under it; ALWAYS released in the + // `finally`. If a peer holds it, skip THIS cycle cleanly and retry in ~30s (reusing the + // existing retry alarm — NOT an error). The provider may not implement the lock at all + // (optional contract) ⇒ we proceed unserialized, relying on the deferred-truncation + // backstop. The conditional fast path above is exempt: it writes only this device's + // per-device delta file (never the snapshot), which never clobbers a peer. + if (Cloud.acquireLock) { + lockAcquired = await Cloud.acquireLock(selfDeviceId); + if (!lockAcquired) { + log.info('advisory lock held by a peer; skipping this cycle, retry soon'); + await rescheduleSoonAfterLockContention(log); + + progress(100); + syncResult.ok = true; + syncResult.progress = 100; + syncResult.lockContended = true; + syncResult.changes = {local: false, cloud: false}; + + send('sync-end', syncResult); + log.stop('advisory lock contended: skipped cycle'); + return syncResult; + } + } + + // 1. pull base snapshot (+ legacy seed fallback) and all device delta logs + const {snapshot: pulledSnapshot, snapshotExists} = await resolveBaseSnapshot(Cloud); + progress(30); + const pulledDeltaLogs = await resolvePulledDeltaLogs(Cloud); + progress(45); + + // 1a. E2 RESET/WATERMARK-TRAP RECONCILIATION. If a local reset rewound our lastSeq + // (so our re-issued events now sit at seq 1..K) while the CLOUD still carries our prior + // history, replay would (a) dedup-skip every re-issued event with seq <= the cloud + // watermark[self]=N (replay.js rule 4) and (b) COLLIDE with the events still present in + // our cloud delta file (which plan-sync merges by seq). Now that we have both from the + // pull, fast-forward THIS device's log so every event sits strictly above the MAX of the + // stale cloud watermark AND the highest seq remaining in our cloud delta file — clearing + // both hazards — then RE-READ pending (their seqs may have shifted). No-op when the device + // already leads that floor (the normal, non-reset path is gated out entirely). Clear the + // flag AFTER the attempt so a crash before this point retries the reconciliation. + if (resetPending) { + const cloudSelfWatermark = Number(pulledSnapshot?.watermark?.[selfDeviceId]) || 0; + const pulledSelfLog = (pulledDeltaLogs || []).find(dl => dl.deviceId === selfDeviceId); + const highestCloudSelfSeq = (pulledSelfLog?.events || []).reduce( + (max, e) => (Number(e.seq) > max ? Number(e.seq) : max), 0, + ); + const floor = Math.max(cloudSelfWatermark, highestCloudSelfSeq); + const shifted = await DeltaLog.fastForwardSeqsAbove(floor); + if (shifted) { + localPendingEvents = await DeltaLog.getEventsSince(lastPushedSeq); + log.info('E2: fast-forwarded local log above stale cloud watermark/delta after reset', { + cloudSelfWatermark, + highestCloudSelfSeq, + floor, + pendingEvents: localPendingEvents.length, + }); + } + delete storage[resetPendingKey(selfDeviceId)]; + } + + // 1a-bis. DEFERRED SELF-TRUNCATION reconciliation (Part C — the data-loss backstop). + // A prior compaction cycle on this device recorded a pending marker = the self + // watermark seq it folded into the snapshot it wrote, but did NOT truncate (a peer + // could clobber that snapshot before its durability is confirmed; the snapshot is the + // ONLY home of folded history). Now that we have the PULLED snapshot, confirm: if its + // watermark[self] >= the pending seq, our snapshot survived (or a later one supersedes + // it) and the folded events are durably in the cloud base — it is safe to truncate up + // to that seq (clearUpTo locally below + drop seq <= it from the cloud self-delta in the + // push) and clear the marker. If the snapshot was CLOBBERED (watermark below the + // marker), the events still live in the cloud self-delta (we deferred), so they get + // re-folded — we keep the marker (do NOT truncate). See compaction.resolveDeferredTruncation. + const pendingTruncateSeq = Number(storage[pendingTruncateKey(selfDeviceId)]) || 0; + const {confirmed: deferredTruncateConfirmed, truncateSeq: confirmedTruncateSeq} = + resolveDeferredTruncation(pendingTruncateSeq, pulledSnapshot?.watermark, selfDeviceId); + + // 1b. COMPACTION decision (PURE): how many events are UNFOLDED — beyond the pulled + // snapshot's BASE watermark (the exact predicate replay uses to fold) — across ALL + // pulled device logs. Over the threshold ⇒ this cycle compacts: it persists the + // resolved snapshot as the new base (advancing each device's watermark to the highest + // folded seq) and truncates THIS device's own log. Non-blocking w.r.t. lagging/lost + // devices: we count + fold only this device's pulled view and never wait for any device + // to catch up; a behind device later pulls the new base (folded effect present) + its + // remaining unfolded deltas, losing nothing (see compaction.js). Evaluated on the + // PULLED (pre-replay) watermark so the count matches what replay will actually fold. + const {shouldCompact, unfoldedCount} = evaluateCompaction( + pulledDeltaLogs, pulledSnapshot?.watermark, + ); + + // 2. CONTAINER BOUNDARY (outbound): translate every LOCAL `cookieStoreId` in this + // device's inputs to a PORTABLE key (a container's name+color+icon identity, or a + // default/temporary marker), and collect a registry of {portableKey: {name,color,icon}} + // that travels in the snapshot so a receiving device can find-or-create the matching + // container. We translate (a) `localState` (its group/tab/pinned/defaultGroupProps + // fields) and (b) this device's `localPendingEvents` — both currently hold raw local + // cookieStoreIds (buildLocalState reads live tabs; capture logs the local id). The + // pulled snapshot + other devices' delta logs are ALREADY portable (prior delta + // syncs wrote them so). Pending events are mutated in place here so the SAME portable + // records flow into both `planSync` (replay) and `deltaFileToWrite` (the push). The + // registry is seeded from the pulled snapshot's registry so other devices' container + // defs survive into this device's next write. From here the pure engine (replay / + // plan-sync) sees only portable keys (see container-map.js). + const {registry: containerRegistry, mapToPortable} = buildOutboundContainerMapping(pulledSnapshot.containers); + mapStateContainers(localState, mapToPortable); + for (const event of localPendingEvents) { + mapEventContainers(event, mapToPortable); + } + + progress(50); + + // 6. PURE plan (priorBaseline gates removals) + const plan = planSync({ + pulledSnapshot, + pulledDeltaLogs, + localPendingEvents, + selfDeviceId, + localState, + priorBaseline, + }); + + // The resolved snapshot carries the merged container registry (pulled + this + // device's local defs) so it is written back as the portable definition source + // for every device. replay() carries the pulled registry through; we fold in any + // new local containers collected during the outbound translation above. + plan.resolvedSnapshot.containers = {...plan.resolvedSnapshot.containers, ...containerRegistry}; + + // SAFETY: never wipe everything from an empty cloud. If the resolved state has + // no groups but the user DOES have local groups (e.g. a brand-new install/empty + // gist, or a cloud that hasn't received this device's deltas yet), suppress the + // destructive removal ops for this round. The local groups/tabs are still + // captured locally and will be pushed; the next round reconciles non-destructively. + const resolvedEmpty = (plan.resolvedSnapshot.groups || []).length === 0 + && (plan.resolvedSnapshot.pinnedTabs || []).length === 0; + const localHasState = (localState.groups || []).length > 0 + || (localState.pinnedTabs || []).length > 0; + if (resolvedEmpty && localHasState) { + log.warn('resolved state empty but local has groups/pinned - suppressing removals this round'); + plan.browserOps.groupsToRemove = []; + plan.browserOps.tabsToRemove = []; + plan.browserOps.pinnedToRemove = []; + } + + log.info('plan', { + ops: { + groupsToCreate: plan.browserOps.groupsToCreate.length, + groupsToUpdate: plan.browserOps.groupsToUpdate.length, + groupsToRemove: plan.browserOps.groupsToRemove.length, + tabsToCreate: plan.browserOps.tabsToCreate.length, + tabsToMove: plan.browserOps.tabsToMove.length, + tabsToRemove: plan.browserOps.tabsToRemove.length, + pinnedToCreate: plan.browserOps.pinnedToCreate.length, + pinnedToMove: plan.browserOps.pinnedToMove.length, + pinnedToRemove: plan.browserOps.pinnedToRemove.length, + }, + willPush: !!plan.deltaFileToWrite, + }); + + progress(55); + + // 6b. CONTAINER BOUNDARY (inbound): translate every PORTABLE container key in the + // ops we are about to apply back to a real local `cookieStoreId`, find-or-creating + // the matching local container (name+color+icon) when absent. The resolved snapshot's + // `containers` registry is the portable→identity source. Markers map to local + // default/temporary. After this, browserOps/optionsToApply hold local ids the apply + // path can use directly (see translateInboundContainers). + await translateInboundContainers(plan.browserOps, plan.optionsToApply, plan.resolvedSnapshot.containers, log); + + // 6c. SAFETY NET: before mutating the live browser, take a full local STG backup + // (gated by the LOCAL-ONLY `syncBackupBeforeApply` option, default ON) so a bad + // sync is fully rollback-able. Fires only when the plan actually mutates the + // browser, AWAITS completion, and THROWS on failure — which jumps to catch and + // aborts the apply, so we never mutate without a safety net in place. + await maybeBackupBeforeApply(plan, log); + + // 7. APPLY to the live browser, under the USER-PRIORITY LOCK so it can't interleave + // with a concurrent user group/tab mutation (lost-update race: both do load→modify→ + // save against the blind `Groups.save`). The lock is held ONLY around this short + // local apply — NEVER around the network pull (steps 1-2) or push (step 8). + // + // YIELD POLICY: if the user is mutating right now (or in the short trailing window), + // `runSyncApply` DEFERS — it does NOT apply this cycle. We then skip the push too + // (nothing was applied; watermark/baseline are left untouched so no state is recorded) + // and reschedule a sync soon via the existing retry alarm. Deferring one periodic + // cycle is cheap; the user never waits on us. A safety timeout inside `runSyncApply` + // makes us defer rather than block forever if a user mutation is mid-flight. + // reset the phase marker so the watchdog (below) can only ever report a phase from + // THIS apply, never a stale one from a prior cycle. + currentApplyPhase = null; + const applyStartedAt = Date.now(); + const applyOutcome = await runSyncApply(async () => { + // 7a. live browser ops (groups save/reorder/order + tabs create/move/remove + + // pinned ops). Skip-flagged so it's not re-captured. Pass the resolved snapshot so + // the apply reconciles each group's live tab order to the authoritative order. + await applyBrowserOps(plan.browserOps, plan.resolvedSnapshot); + + // 7b. resolved global option changes via STG's real save path (so alarms/hotkeys/ + // container side effects run), suppressed from re-capture. + const endOptions = beginApplyPhase('apply-options', log); + await applyOptions(plan.optionsToApply); + endOptions(); + + // 7c. REBUILD per-group context menus. The apply step persists group create/remove/ + // reorder via `Groups.save`, which deliberately bypasses the `sendAdded/sendUpdated/ + // sendRemoved` helpers (so capture isn't re-triggered) — but those helpers are also + // what drive the `MenusMain.group*` calls that create/remove the per-group menu items + // (`tab-` etc. in menus-tab/link/bookmark). So a synced group exists in data + // but has NO menu item, and the next `Menus.update(groupId)` (e.g. from `Groups.apply` + // → `MenusMain.groupLoaded`) used to throw "doesn't exist" and abort the apply mid-way + // (dropping the group's tabs). We rebuild the full per-group menu set here (the same + // remove+recreate `MenusMain.groupsUpdated` does for `Groups.move`/`sort`), but ONLY + // when this round actually changed the local group set, so an idle sync is a no-op. + // This rebuild touches only the browser.menus API — it performs no tab/group mutation, + // so the delta-capture layer (which keys off real tab/group events) cannot re-record it. + const {groupsChanged} = summarizeOps(plan.browserOps, plan.optionsToApply); + if (groupsChanged) { + const endMenus = beginApplyPhase('menus-rebuild', log); + const {groups: rebuiltGroups} = await Groups.load(null, false); + await MenusMain.groupsUpdated(rebuiltGroups) + .catch(log.onCatch('cant rebuild group menus after delta sync', false)); + endMenus(); + } + currentApplyPhase = null; // apply finished cleanly; nothing is "in progress" + }, { + // COMPLETION WATCHDOG: if the apply holds the user-priority lock past this bound + // (a never-settling await would otherwise wedge EVERY future user action forever), + // log a WARN naming the last-started phase + elapsed ms and release the lock so the + // UI recovers. The in-flight apply keeps running detached. The phase name in this + // WARN is the diagnostic that pins where a stalled apply hung on the user's device. + watchdogMs: SYNC_APPLY_WATCHDOG_MS, + onWatchdog: ({elapsedMs}) => { + log.warn('SYNC APPLY WATCHDOG TRIPPED: apply exceeded the held-lock bound; releasing the user-priority lock so user actions recover. Apply continues detached.', { + stuckPhase: currentApplyPhase, + elapsedMs, + watchdogMs: SYNC_APPLY_WATCHDOG_MS, + sinceApplyStartMs: Date.now() - applyStartedAt, + }); + }, + }); + + if (applyOutcome.deferred) { + // user is mutating → we yielded this cycle WITHOUT applying or pushing. Reschedule + // soon and exit cleanly (a deferral is NOT an error; the next cycle re-pulls and + // applies once the user's burst settles). No watermark/baseline write happened. + log.info('apply DEFERRED: user is mutating groups/tabs; rescheduling sync soon', { + userActive: isUserActive(), + }); + await rescheduleSoonAfterDefer(log); + + syncResult.ok = true; + syncResult.deferred = true; + syncResult.progress = lastProgress; + syncResult.changes = {local: false, cloud: false}; + + send('sync-end', syncResult); + log.stop('deferred to user'); + return syncResult; + } + + progress(85); + + // 8. PUSH. The self delta file is written whenever this device has new events. + // The full snapshot (STG-snapshot.json) is the gist-discovery MARKER + the + // consolidated base; it is REWRITTEN ONLY when (a) this cycle COMPACTS, or (b) it + // does not yet exist (first create). Between compactions a normal sync pushes ONLY + // this device's delta file — no snapshot rewrite — so idle/steady cycles stop + // re-uploading the whole snapshot (the point-2 optimization, coupled to compaction). + // + // DEFERRED SELF-TRUNCATION (Part C): a COMPACTION cycle no longer truncates its own + // log in the SAME cycle it writes the snapshot. The snapshot is the ONLY home of folded + // history; a peer could clobber the just-written snapshot with an older one (rare under + // the advisory lock, but possible without conditional writes / after a crash), and the + // truncated events would then live ONLY in the clobbered snapshot ⇒ permanent loss. So: + // - the COMPACTION cycle writes the FULL self log to the cloud (no truncation) and + // records a pending marker = the self watermark seq it folded into the snapshot; + // - a SUBSEQUENT cycle, once the PULLED snapshot's watermark[self] proves >= that seq + // (durability confirmed — reconciled in step 1a-bis above), truncates up to it: both + // the LOCAL log (clearUpTo) and the cloud self-delta (drop seq <= it), then clears + // the marker. The invariant: an event always lives in (a) a cloud delta file OR (b) a + // cloud snapshot whose durability we CONFIRMED — never truncate the only copy first. + const writeSnapshot = shouldCompact || !snapshotExists; + + // The cloud self-delta is truncated ONLY by a CONFIRMED deferred truncation (step + // 1a-bis) — never in the compaction cycle that wrote the snapshot. CLAMPED to + // lastPushedSeq via selfFoldedSeq when the marker was first set, so it can never exceed + // an unpushed/unfolded tail. 0 ⇒ keep the full self log in the cloud (still deferring). + const cloudSelfTruncateSeq = deferredTruncateConfirmed ? confirmedTruncateSeq : 0; + + const filesToWrite = {}; + if (writeSnapshot) { + filesToWrite[SNAPSHOT_FILE_NAME] = plan.resolvedSnapshot; + } + if (plan.deltaFileToWrite) { + // Keep the full self log in the cloud UNLESS a deferred truncation is confirmed this + // cycle (then drop its confirmed-folded head). A compaction cycle writes FULL events + // (cloudSelfTruncateSeq === 0) so the not-yet-confirmed events stay recoverable. + const selfEvents = cloudSelfTruncateSeq > 0 + ? truncateSelfEvents(plan.deltaFileToWrite.events, cloudSelfTruncateSeq) + : plan.deltaFileToWrite.events; + filesToWrite[deltaFileName(selfDeviceId)] = { + v: DeltaLog.SCHEMA_VERSION, + deviceId: plan.deltaFileToWrite.deviceId, + events: selfEvents, + }; + } + + // A pure-compaction cycle (snapshot to write but no new self events to push) must + // still PATCH the snapshot. writeFiles with only the snapshot covers that; an empty + // object would be a no-op, so guard it. + if (Object.keys(filesToWrite).length) { + await Cloud.writeFiles(filesToWrite); + } + + // refresh the stored conditional-request ETag from the post-push gist so the NEXT + // cycle's isUnchangedSince probe compares against the revision we just wrote (else + // our own push always reads back as "changed"). Optional + best-effort (fail-safe). + await Cloud.refreshEtagFromWrite?.(); + + if (plan.deltaFileToWrite) { + // advance lastPushedSeq to the highest self seq written + const maxSelfSeq = plan.deltaFileToWrite.events.reduce( + (max, e) => (e.seq > max ? e.seq : max), + lastPushedSeq, + ); + storage[lastPushedSeqKey(selfDeviceId)] = maxSelfSeq; + } + + // 8b. CONFIRMED DEFERRED TRUNCATION (own log + clear marker). Runs BEFORE recording a new + // marker below so a cycle that BOTH confirms a prior deferral AND compacts does not delete + // the fresh marker (8c re-records it afterwards from the just-folded seq). When step + // 1a-bis confirmed the pulled snapshot durably carries the folded events, NOW drop the + // folded head of THIS device's LOCAL delta log (the cloud self-delta head was dropped in + // the write above) and clear the marker. clearUpTo keeps lastSeq monotonic so future + // appends never collide. Safe by the invariant: the cloud snapshot's watermark[self] >= + // the seq we trim, so any device pulling sees the folded effect in the base and replay + // skips the trimmed events (seq <= watermark) — no double-apply, no loss. + if (deferredTruncateConfirmed && confirmedTruncateSeq > 0) { + await DeltaLog.clearUpTo(confirmedTruncateSeq); + delete storage[pendingTruncateKey(selfDeviceId)]; + log.info('deferred truncation CONFIRMED: cloud snapshot durably carries folded events; truncated own log', { + truncatedUpToSeq: confirmedTruncateSeq, + cloudSelfWatermark: Number(pulledSnapshot?.watermark?.[selfDeviceId]) || 0, + }); + } + + // 8c. RECORD the deferred-truncation marker for THIS compaction (no local truncation + // here). Runs AFTER the confirmed-truncation step above so a same-cycle confirm can't + // delete this fresh marker. The seq we'd fold = the new base's advanced self watermark, + // CLAMPED to lastPushedSeq (never mark an unpushed/unfolded tail). The snapshot carrying + // that fold has just been written, but its durability is not yet CONFIRMED (a peer could + // clobber it), so we only RECORD; a later cycle truncates once the pulled watermark proves + // it survived. If a marker is already pending (an earlier compaction not yet confirmed), + // keep the LARGER seq — the latest snapshot folds at least as much. No truncation until + // confirmed ⇒ the only copy of folded events is never removed prematurely. + if (shouldCompact) { + const foldedSelfSeq = selfFoldedSeq(plan.newWatermark, selfDeviceId, lastPushedSeq); + if (foldedSelfSeq > 0) { + const newPending = Math.max(pendingTruncateSeq, foldedSelfSeq); + storage[pendingTruncateKey(selfDeviceId)] = newPending; + log.info('compaction: wrote snapshot base + recorded DEFERRED self-truncation marker', { + unfoldedCount, + pendingTruncateSeq: newPending, + newWatermark: plan.newWatermark, + }); + } else { + log.info('compaction: rewrote snapshot base (nothing foldable to defer-truncate)', { + unfoldedCount, + newWatermark: plan.newWatermark, + }); + } + } + + // 8d. NO ORPHAN DELTA-FILE GC. We deliberately do NOT delete other devices' folded + // delta files. GitHub has no conditional write (If-Match → bare 400), so a peer can + // clobber the snapshot we just wrote with an older one; a delta file deleted here + // would then be unrecoverable (the snapshot is the ONLY home of folded history — delta + // files are truncated tails, not a full replay source). Instead we rely on the + // permanently-kept watermark: a returning device re-pushing its full log is skipped by + // replay for every event with seq <= watermark[D] (replay.js: `event.seq <= folded` ⇒ + // skip), so a never-deleted peer file never double-applies or resurrects. The only cost + // is that a dead device's file lingers — harmless over-keeping. Each device still trims + // ONLY its OWN delta tail (deferred, on a later confirmed cycle above), which is safe + // because that tail's effect is durably in the snapshot the pulled watermark confirmed. + + progress(90); + + // 9. persist the resolved watermark as the base for the next round. This advances + // every device's high-water mark to the highest folded seq, exactly as written into + // the snapshot when this cycle compacted. (Snapshot rewrites are gated above; on a + // non-compacting cycle the cloud snapshot keeps its prior watermark, but the resolved + // watermark is a superset that is re-persisted to the snapshot on the next compaction.) + storage[watermarkKey(selfDeviceId)] = JSON.stringify(plan.newWatermark || {}); + + // 10. persist the NEW baseline = the ids/uids of the resolved snapshot. This + // runs ONLY on the success path (after apply + push succeeded); if any earlier + // step threw, control jumps to catch and the baseline is left untouched, so we + // never record a state we didn't actually reconcile. + saveBaseline(selfDeviceId, baselineFromSnapshot(plan.resolvedSnapshot)); + + progress(100); + + syncResult.ok = true; + syncResult.progress = 100; + + // Shape-compatible with the legacy synchronization() result so UI listeners + // (tab-groups.mixin `sync-end` → request.changes.local) don't crash. + // local = applied any change to this browser; cloud = pushed a delta file. + // Routed through summarizeOps (same source of truth as the pre-apply backup + // gate) so a content-only apply (tabsToUpdate / pinnedToUpdate) flips local=true + // and the UI refreshes (B2). + syncResult.changes = { + local: summarizeOps(plan.browserOps, plan.optionsToApply).mutatesBrowser, + cloud: !!plan.deltaFileToWrite, + }; + + send('sync-end', syncResult); + log.stop(); + } catch (e) { + syncResult.langId = e.langId; + syncResult.progress = lastProgress; + Object.assign(syncResult, {message: String(e), stack: e.stack}); + + send('sync-error', syncResult); + log.logError('cant delta sync', e); + log.stopError(); + } finally { + // Release the advisory lock no matter how we exit (success, error, or the apply + // watchdog firing). Best-effort + idempotent; a lock we fail to delete is reclaimed by + // the TTL. Only release one we actually acquired (a contended-skip never held it). + if (lockAcquired && Cloud?.releaseLock) { + await Cloud.releaseLock().catch(e => + log.warn('cant release advisory lock; TTL will reclaim it', String(e))); + } + inProgress = false; + send('sync-finish', syncResult); + } + + return syncResult; +} diff --git a/addon/src/js/sync/delta/device-id.js b/addon/src/js/sync/delta/device-id.js new file mode 100644 index 00000000..8b2696d2 --- /dev/null +++ b/addon/src/js/sync/delta/device-id.js @@ -0,0 +1,68 @@ + +/** + * Per-install device identity for delta sync (Phase P1). + * + * Each install owns a persistent `deviceId` (UUID) plus a human-readable `label`. + * The id names this device's append-only delta file in the cloud layout + * (`STG-delta-.json`) and keys per-device watermarks; the label is only + * surfaced to the user. See `.project/DESIGN_DELTA_SYNC.md` "Device identity". + * + * Generated once on first read and reused thereafter. + * + * Backing store: the synchronous prefixed `localStorage` under the CLOUD module, + * matching how other durable cloud config is kept (`githubGistFileName`, + * `lastError` in cloud.js). localStorage persists across browser restarts for the + * background page, and the identity is tiny + read frequently, so a synchronous + * store avoids an await on every delta append. The CLOUD root is never cleared on + * start (only its `sync` sub-storage is), so the identity survives restarts. + * + * P1 is inert: nothing consumes the id yet. + */ + +import '/js/prefixed-storage.js'; +import * as Constants from '/js/constants.js'; + +const storage = localStorage.create(Constants.MODULES.CLOUD); + +const DEVICE_ID_KEY = 'deviceId'; +const DEVICE_LABEL_KEY = 'deviceLabel'; + +/** + * Returns this install's stable device id (UUID), generating + persisting it once. + * @returns {string} + */ +export function getDeviceId() { + let deviceId = storage[DEVICE_ID_KEY]; + + if (!deviceId) { + deviceId = self.crypto.randomUUID(); + storage[DEVICE_ID_KEY] = deviceId; + } + + return deviceId; +} + +/** + * Returns the human-readable device label. Defaults to the browser full name + * (e.g. "Firefox Mozilla") the first time it's read; persisted thereafter so a + * future user-set label is honoured. + * @returns {string} + */ +export function getDeviceLabel() { + let label = storage[DEVICE_LABEL_KEY]; + + if (!label) { + label = Constants.BROWSER_FULL_NAME; + storage[DEVICE_LABEL_KEY] = label; + } + + return label; +} + +/** + * Overrides the human-readable device label (e.g. from a future settings UI). + * @param {string} label + */ +export function setDeviceLabel(label) { + storage[DEVICE_LABEL_KEY] = label; +} diff --git a/addon/src/js/sync/delta/group-relative-index.js b/addon/src/js/sync/delta/group-relative-index.js new file mode 100644 index 00000000..e5cc730d --- /dev/null +++ b/addon/src/js/sync/delta/group-relative-index.js @@ -0,0 +1,42 @@ +/** + * Pure helper for the capture layer's GROUP-RELATIVE index computation. + * + * Import-free (no extension-host dependencies) so it can be unit-tested with `node` + * directly, like {@link module:tab-move-split}. The live wrapper in + * {@link module:sync/delta/delta-capture} feeds it `browser.tabs.query` results plus a + * `Cache.getTabGroup` resolver. + * + * @module sync/delta/group-relative-index + */ + +/** + * Compute a tab's 0-based position among the tabs of the SAME group within the same + * window, ordered by browser index. + * + * The delta/replay model treats a tab's `index` as the position WITHIN its group's + * ordered tab list (0..n-1), NOT the browser-window-absolute `tab.index` (which is + * shifted by pinned tabs and other groups' tabs sharing the window, and differs per + * machine). Replaying by the absolute index shuffles the group's order on every other + * device, so capture derives the within-group position instead. + * + * Returns null (⇒ caller omits `index` ⇒ replay appends at end) when the position can't + * be determined — better an append than a wrong slot. + * + * @param {Array<{id:number, index:number}>} windowTabs - the window's live tabs. + * @param {(tabId:number)=>(string|undefined)} getTabGroupFn - resolves a tab's groupId. + * @param {number} tabId - the tab whose position we want. + * @param {string} groupId - the group to scope the position to. + * @returns {number|null} + */ +export function computeGroupRelativeIndex(windowTabs, getTabGroupFn, tabId, groupId) { + if (!Array.isArray(windowTabs) || typeof getTabGroupFn !== 'function' || !groupId) { + return null; + } + + const groupTabs = windowTabs + .filter(t => getTabGroupFn(t.id) === groupId) + .sort((a, b) => a.index - b.index); + + const position = groupTabs.findIndex(t => t.id === tabId); + return position === -1 ? null : position; +} diff --git a/addon/src/js/sync/delta/layout.js b/addon/src/js/sync/delta/layout.js new file mode 100644 index 00000000..9a6e4b09 --- /dev/null +++ b/addon/src/js/sync/delta/layout.js @@ -0,0 +1,64 @@ + +/** + * Cloud storage layout naming for hybrid snapshot + delta sync (Phase P3a). + * + * Single source of truth for the file names the gist holds (see + * `.project/DESIGN_DELTA_SYNC.md` "Raskladka"/storage layout): + * - `STG-snapshot.json` — the compacted base snapshot. + * - `STG-delta-.json` — one append-only delta log per device. + * - `STG-backup.json` — the OLD single-file full-state backup that + * existing (pre-delta) users already have; never deleted, used to seed the + * first snapshot (see {@link module:sync/delta/seed}). + * + * ## Purity + * This module is PURE (literals only, no `browser.*`, no `constants.js` import) so + * it can be shared by the pure planner/seed AND by the impure gist client. The + * names intentionally mirror `Constants.GIT_GIST_FILE_NAME_PARTS` (`STG-`/`.json`) + * but are duplicated as literals here to keep this module import-free. + * + * @module sync/delta/layout + */ + +/** The compacted base snapshot file. */ +export const SNAPSHOT_FILE_NAME = 'STG-snapshot.json'; + +/** Filename prefix for per-device delta logs (`STG-delta-.json`). */ +export const DELTA_FILE_PREFIX = 'STG-delta-'; + +/** The legacy single-file full-state backup that pre-delta users still have. */ +export const LEGACY_BACKUP_FILE_NAME = 'STG-backup.json'; + +/** + * Advisory distributed-lock file. Holds `{deviceId, expiresAt}` (`expiresAt` is an + * ABSOLUTE time on the SERVER clock, ms) and serializes sync cycles across devices so two + * devices don't write the snapshot concurrently. ADVISORY/best-effort only: GitHub has no + * conditional write (an `If-Match` PATCH returns a bare 400), so the lock is acquired by + * write-then-read-back-to-confirm, not compare-and-set. Deferred self-truncation + * (see {@link module:sync/delta/compaction}) is the data-safety backstop. See + * {@link module:sync/delta/lock}. + */ +export const LOCK_FILE_NAME = 'STG-lock.json'; + +/** Common suffix of every layout file. */ +const FILE_SUFFIX = '.json'; + +/** + * Build the delta filename for a device id (`STG-delta-.json`). + * @param {string} deviceId + * @returns {string} + */ +export function deltaFileName(deviceId) { + return `${DELTA_FILE_PREFIX}${deviceId}${FILE_SUFFIX}`; +} + +/** + * Extract the device id from a delta filename, or null if it is not one. + * @param {string} fileName + * @returns {string|null} + */ +export function deviceIdFromDeltaFileName(fileName) { + if (typeof fileName !== 'string' || !fileName.startsWith(DELTA_FILE_PREFIX) || !fileName.endsWith(FILE_SUFFIX)) { + return null; + } + return fileName.slice(DELTA_FILE_PREFIX.length, -FILE_SUFFIX.length) || null; +} diff --git a/addon/src/js/sync/delta/lock.js b/addon/src/js/sync/delta/lock.js new file mode 100644 index 00000000..dd187959 --- /dev/null +++ b/addon/src/js/sync/delta/lock.js @@ -0,0 +1,101 @@ + +/** + * Pure decision helpers for the advisory distributed sync lock (Part A). + * + * The lock is a single gist file ({@link module:sync/delta/layout.LOCK_FILE_NAME}) holding + * `{deviceId, expiresAt}`, where `expiresAt` is an ABSOLUTE time on the SERVER clock (ms). + * Its purpose is to serialize sync cycles across devices so two devices don't write the + * snapshot concurrently and clobber it — GitHub has NO conditional write (an `If-Match` + * PATCH returns a bare 400), so we cannot do an atomic compare-and-set. The lock is + * therefore ADVISORY/best-effort: acquire = write-our-stamp THEN re-read to confirm we won + * any race; if a peer's stamp came back instead, we lose and back off. A crashed holder's + * stamp is reclaimed once it goes stale (server time past `expiresAt`). Deferred + * self-truncation (see {@link module:sync/delta/compaction}) is the real data-safety + * backstop; this lock only reduces how often two devices write the snapshot at once. + * + * ## Purity + * No `browser.*`, no network, no `constants.js`. Reads inputs, returns plain data — so the + * race/staleness decisions can be unit-tested without a live gist (the impure read/write/ + * confirm-delay lives in `../cloud/githubgist.js`, calling these to decide). + * + * @module sync/delta/lock + */ + +/** + * Lock time-to-live (ms). A holder's stamp is considered STALE — and so freely reclaimable + * by any device — once the server clock passes `acquiredAt + LOCK_TTL_MS`. A full sync cycle + * finishes well under this (there is a 60s apply watchdog), so the TTL only ever reclaims a + * lock left behind by a crashed / killed device. 2 minutes per the design. + * @type {number} + */ +export const LOCK_TTL_MS = 120000; + +/** + * Confirm re-read delay (ms) between writing our stamp and re-reading it to resolve a race. + * Short — just long enough for a concurrent peer's write to become observable so exactly one + * winner is read back. Kept here (not in the impure layer) so the value is documented next to + * the protocol it serves. + * @type {number} + */ +export const LOCK_CONFIRM_DELAY_MS = 1500; + +/** + * Is a parsed lock value STALE (its holder presumed gone) at the given server time? A null / + * malformed lock (no numeric `expiresAt`) counts as stale — there is nothing valid to honor, + * so it is freely reclaimable. Stale iff `serverNow >= expiresAt`. + * + * @param {?object} lock - the parsed lock content (`{deviceId, expiresAt}`) or null/absent. + * @param {number} serverNow - the current SERVER time (ms). + * @returns {boolean} true iff the lock is absent, malformed, or expired. + */ +export function isLockStale(lock, serverNow) { + const expiresAt = Number(lock?.expiresAt); + if (!Number.isFinite(expiresAt)) { + return true; // absent / malformed ⇒ nothing valid to honor ⇒ reclaimable + } + return serverNow >= expiresAt; +} + +/** + * Decide, from the FIRST read of the lock file, whether this device may WRITE its stamp. + * We may write when the lock is absent/stale, OR when it is already ours (re-entrant / + * renewal). We may NOT write when it is held, fresh, and owned by another device. + * + * @param {?object} lock - the parsed lock content, or null if the file is absent. + * @param {string} selfDeviceId + * @param {number} serverNow - current SERVER time (ms). + * @returns {boolean} true iff this device should write its stamp and proceed to confirm. + */ +export function canWriteLock(lock, selfDeviceId, serverNow) { + if (isLockStale(lock, serverNow)) { + return true; + } + return lock?.deviceId === selfDeviceId; +} + +/** + * Resolve the CONFIRM read: after writing our stamp and waiting, did WE win the lock? The + * lock is ours iff the re-read stamp's `deviceId` is ours. (A peer that wrote last between + * our write and our re-read wins instead — exactly one stamp survives the last write, and + * the read-back reveals who.) + * + * @param {?object} confirmedLock - the parsed lock re-read after the confirm delay. + * @param {string} selfDeviceId + * @returns {boolean} true iff this device holds the lock (acquired). + */ +export function didWinLock(confirmedLock, selfDeviceId) { + return confirmedLock?.deviceId === selfDeviceId; +} + +/** + * Build the lock stamp this device writes: `{deviceId, expiresAt}` with `expiresAt` an + * absolute SERVER time `serverNow + ttlMs`. + * + * @param {string} selfDeviceId + * @param {number} serverNow - current SERVER time (ms). + * @param {number} [ttlMs] - lock lifetime; defaults to {@link LOCK_TTL_MS}. + * @returns {{deviceId: string, expiresAt: number}} + */ +export function makeLockStamp(selfDeviceId, serverNow, ttlMs = LOCK_TTL_MS) { + return {deviceId: selfDeviceId, expiresAt: serverNow + ttlMs}; +} diff --git a/addon/src/js/sync/delta/option-keys.js b/addon/src/js/sync/delta/option-keys.js new file mode 100644 index 00000000..ed3e15f8 --- /dev/null +++ b/addon/src/js/sync/delta/option-keys.js @@ -0,0 +1,75 @@ + +/** + * Single source of truth for WHICH global option keys roam through delta sync. + * + * STG persists a flat bag of option values (see `constants.js` `ALL_OPTION_KEYS` / + * `DEFAULT_OPTIONS`). Most are user preferences that should follow the user between + * machines; a few are inherently per-device / local and must stay put. This module + * holds the ONE predicate that decides "does this option key sync?", so the capture + * layer (which appends `option.set` deltas) and the transport (which bootstraps + + * applies them) agree exactly on the synced subset. + * + * ## What stays local (mirrors the OLD `cloud.js` `syncOptions()` exclusions) + * Keys whose name starts with any of {@link LOCAL_ONLY_OPTION_KEY_PREFIXES} are NOT + * synced: + * - `sync*` — sync provider/token/interval/location/lastUpdate are per-device + * (the token in particular must never roam to another machine); + * - `autoBackup*` — local backup schedule + paths are per-device. + * + * ## What DOES sync (intentional parity change vs. the old code) + * Everything else in `ALL_OPTION_KEYS`, INCLUDING `defaultGroupProps` and `hotkeys` + * (the old `syncOptions()` excluded `defaultGroupProps` from its loop and merged it + * separately; here both roam as plain `option.set` values for full parity per the + * client request). `groups` / `version` are NOT option keys (`NON_OPTION_KEYS`) and + * never reach here — groups are synced by the group/tab delta ops. + * + * ## Purity + * This module is PURE (no `browser.*`, no `constants.js` import): {@link isSyncedOptionKey} + * is a string predicate, so the pure replay/plan-sync engines can use it directly. The + * IMPURE callers (capture / transport) pass it `Constants.ALL_OPTION_KEYS` to derive the + * concrete synced key list via {@link syncedOptionKeys}. + * + * @module sync/delta/option-keys + */ + +/** + * Option-key name prefixes that are per-device / local and must NOT sync. Mirrors the + * OLD `cloud.js` `EXCLUDE_OPTION_KEY_STARTS_WITH` minus `defaultGroupProps` (which now + * syncs). Frozen so callers can reference it without accidental mutation. + * @readonly + */ +export const LOCAL_ONLY_OPTION_KEY_PREFIXES = Object.freeze(['sync', 'autoBackup']); + +/** + * Exact option keys that are per-device and must NOT sync, even though they don't match + * a local-only prefix: + * - `temporaryContainerTitle` — its default is a locale-derived i18n string, so syncing + * it would converge two differently-localed machines onto one locale's title (a + * cosmetic wart, not a feature). Per-device. + * @readonly + */ +export const LOCAL_ONLY_OPTION_KEYS = Object.freeze(['temporaryContainerTitle']); + +/** + * Does this option key roam through delta sync? True unless its name starts with one + * of {@link LOCAL_ONLY_OPTION_KEY_PREFIXES} or is listed in {@link LOCAL_ONLY_OPTION_KEYS}. + * Pure string predicate. + * @param {string} key + * @returns {boolean} + */ +export function isSyncedOptionKey(key) { + if (LOCAL_ONLY_OPTION_KEYS.includes(key)) { + return false; + } + return !LOCAL_ONLY_OPTION_KEY_PREFIXES.some(prefix => key.startsWith(prefix)); +} + +/** + * Filter a list of option keys down to the synced subset. The impure callers pass + * `Constants.ALL_OPTION_KEYS` here so the concrete list always tracks the constants. + * @param {string[]} allOptionKeys + * @returns {string[]} + */ +export function syncedOptionKeys(allOptionKeys) { + return (allOptionKeys || []).filter(isSyncedOptionKey); +} diff --git a/addon/src/js/sync/delta/plan-sync.js b/addon/src/js/sync/delta/plan-sync.js new file mode 100644 index 00000000..f70227cd --- /dev/null +++ b/addon/src/js/sync/delta/plan-sync.js @@ -0,0 +1,682 @@ + +/** + * Pure sync planner for hybrid snapshot + delta sync (Phase P3a). + * + * Given what was pulled from the cloud (base snapshot + every device's delta log) + * plus this device's not-yet-pushed local events and its current live state, this + * module computes — WITHOUT touching the browser or the network — three things: + * 1. the resolved effective state (via the pure {@link module:sync/delta/replay}); + * 2. the delta file this device should PATCH back (its own events incl. pending); + * 3. a declarative `browserOps` diff describing how to bring the live browser into + * line with the resolved state. + * + * The transport (P3b) does the I/O around this: pull → **planSync** → execute + * `browserOps` via STG's existing apply logic → write `deltaFileToWrite` → persist + * `newWatermark`. Keeping the decision logic pure means the riskiest part — the + * conflict resolution AND the live-diff — runs and is unit-tested under plain + * `node`. See `.project/DESIGN_DELTA_SYNC.md` "Sync flow". + * + * ## Purity (hard requirement) + * No `browser.*`, no network, no `constants.js` import. Only depends on the pure + * `replay.js`. Inputs are read, never mutated. + * + * ## Shapes + * pulledSnapshot : { groups: [...], watermark?: { [deviceId]: seq } } + * pulledDeltaLogs : [ { deviceId, events: [ {seq, ts, op, ...} ] } ] + * localPendingEvents : [ {seq, ts, op, ...} ] // self events not yet pushed + * selfDeviceId : string + * localState : { groups: [ {id, ...props, tabs: [ {uid, ...} ]} ], pinnedTabs?: [ {uid, ...} ], options?: {key: value} } + * priorBaseline : { tabUids: Set|array, groupIds: Set|array, pinnedUids: Set|array } // gates removals + * + * returns { + * resolvedSnapshot : { groups, pinnedTabs, options, watermark }, // == replay() result + * newWatermark : { [deviceId]: seq }, // == resolved.watermark + * deltaFileToWrite : { deviceId, events } | null, + * browserOps : { groupsToCreate, groupsToRemove, groupsToUpdate, + * tabsToCreate, tabsToRemove, tabsToMove, tabsToUpdate, + * pinnedToCreate, pinnedToRemove, pinnedToMove, pinnedToUpdate }, + * optionsToApply : { [key]: value }, // resolved options differing from local + * } + * + * @module sync/delta/plan-sync + */ + +import {replay} from './replay.js'; + +/** + * Deep clone of a plain JSON-ish value (mirrors replay.js to stay obviously pure). + * @template T + * @param {T} value + * @returns {T} + */ +function deepClone(value) { + if (value === null || typeof value !== 'object') { + return value; + } + if (Array.isArray(value)) { + return value.map(deepClone); + } + const out = {}; + for (const key of Object.keys(value)) { + out[key] = deepClone(value[key]); + } + return out; +} + +/** + * Merge this device's not-yet-pushed `localPendingEvents` onto its entry in the + * pulled delta logs, returning the FULL set of logs to replay. + * + * The pulled logs already contain whatever this device previously pushed; the + * pending events are the tail it has captured locally but not yet uploaded. We + * append only pending events with a `seq` strictly greater than the highest seq + * already present for this device (dedup: a pending event may overlap the pulled + * log if a prior push partially landed). The result for self is exactly what + * `deltaFileToWrite` will carry. + * + * @param {Array<{deviceId: string, events: object[]}>} pulledDeltaLogs + * @param {object[]} localPendingEvents + * @param {string} selfDeviceId + * @returns {{fullLogs: Array<{deviceId: string, events: object[]}>, selfEvents: object[]}} + */ +function buildFullLogs(pulledDeltaLogs, localPendingEvents, selfDeviceId) { + const logs = deepClone(pulledDeltaLogs || []); + + let selfLog = logs.find(log => log.deviceId === selfDeviceId); + if (!selfLog) { + selfLog = {deviceId: selfDeviceId, events: []}; + logs.push(selfLog); + } + if (!Array.isArray(selfLog.events)) { + selfLog.events = []; + } + + const highestPulledSeq = selfLog.events.reduce((max, e) => (e.seq > max ? e.seq : max), 0); + + for (const event of localPendingEvents || []) { + if (event.seq == null || event.seq > highestPulledSeq) { + selfLog.events.push(deepClone(event)); + } + } + + return {fullLogs: logs, selfEvents: selfLog.events}; +} + +/** + * Index every tab by uid across a snapshot's groups. + * @param {object} snapshot - { groups: [ {id, tabs:[{uid,...}]} ] } + * @returns {Map} + */ +function indexTabs(snapshot) { + const byUid = new Map(); + for (const group of snapshot.groups || []) { + const tabs = Array.isArray(group.tabs) ? group.tabs : []; + tabs.forEach((tab, index) => { + if (tab.uid != null) { + byUid.set(tab.uid, {groupId: group.id, index, tab}); + } + }); + } + return byUid; +} + +/** + * Shallow group props (everything except `tabs`), used for groupsToUpdate change + * detection. Tab membership is expressed via the tab ops, never group props. + * @param {object} group + * @returns {object} + */ +function groupProps(group) { + const {tabs, ...props} = group; + void tabs; + return props; +} + +/** + * Stable stringify of group props for equality (keys sorted so key order can't + * cause a spurious "update"). + * @param {object} props + * @returns {string} + */ +function stableStringify(props) { + const keys = Object.keys(props).sort(); + return JSON.stringify(keys.map(k => [k, props[k]])); +} + +/** + * Coerce a `priorBaseline` (which may arrive as arrays — that is how it is + * persisted in JSON — or as Sets) into a normalized `{tabUids: Set, groupIds: Set}`. + * A missing/partial baseline degrades to empty sets (first run, or a baseline that + * was cleared) which keeps the removal gate conservative: nothing is removed. + * + * @param {{tabUids?: (Set<*>|Array<*>), groupIds?: (Set<*>|Array<*>), optionKeys?: (Set<*>|Array<*>), pinnedUids?: (Set<*>|Array<*>)}} [priorBaseline] + * @returns {{tabUids: Set<*>, groupIds: Set<*>, optionKeys: Set<*>, pinnedUids: Set<*>}} + */ +function normalizeBaseline(priorBaseline) { + const src = priorBaseline || {}; + return { + tabUids: new Set(src.tabUids || []), + groupIds: new Set(src.groupIds || []), + optionKeys: new Set(src.optionKeys || []), + pinnedUids: new Set(src.pinnedUids || []), + }; +} + +/** + * The per-tab CONTENT attributes reconciled on a uid-matched EXISTING tab (group tabs + * and global pinned tabs alike). Identity (`uid`), placement (`groupId`/`index`) and + * `lastModified` are intentionally EXCLUDED: placement is reconciled via the move ops + * and identity/lastModified are bookkeeping, not user-visible content. + * + * `pinned` is the group-scoped pin flag (only meaningful for group tabs). It is an additive + * boolean that may be ABSENT on the resolved record (replay/capture only carries it when + * relevant) — `resolveTabContentChanges` treats a genuine on⟷off transition as a change so a + * flip propagates, but a record that simply never carried the flag does not manufacture a + * spurious update. + * + * `loaded` is deliberately NOT here. It is a CREATE-TIME hint only: a tab.add record carries + * `loaded:true` so a receiving device with `syncActivatePreviouslyActiveTabs` can create the + * tab loaded instead of asleep. But the apply NEVER force-loads an already-existing tab (lazy + * sleep-by-default UX — see `applyTabContentUpdate`, which intentionally ignores `loaded`). So + * if `loaded` were diffed here it would produce a `tabsToUpdate{loaded:true}` the apply can + * never satisfy: the source device had the tab loaded, but THIS device created it discarded, + * so `buildLocalState` keeps omitting `loaded` and the SAME update re-emits EVERY cycle — a + * non-converging diff (wasted work, log/op churn, part of the reported post-sync sluggishness). + * Excluding it makes "resolved wants loaded but the tab was created asleep" a reconciled no-op, + * restoring the core invariant that repeated syncs with no real change converge to ZERO ops. + * `loaded` still propagates at CREATE time via the tab.add record; it just never drives an update. + * + * `favIconUrl` is deliberately NOT here either: favicons have no dedicated delta events, and + * `buildLocalState` reads the LIVE favicon while the cloud snapshot holds a possibly-folded + * value — so including it would emit a favicon-only `tabsToUpdate` EVERY sync cycle (log + * churn / wasted work) without any user-meaningful change. The favicon STILL rides inside + * records written for other reasons (the tab.add on create, the compaction snapshot via + * buildTabRecord/buildLocalState), so it propagates at those points; it just never triggers + * an update on its own. + */ +const TAB_CONTENT_FIELDS = ['url', 'title', 'cookieStoreId', 'pinned']; + +/** + * Compute the subset of {@link TAB_CONTENT_FIELDS} whose RESOLVED value differs from the + * matched LOCAL tab — the payload of a `tabsToUpdate` / `pinnedToUpdate` entry's `target`. + * + * Normalizes any additive boolean field (currently `pinned`; `loaded` is also recognised + * defensively though it is no longer a {@link TAB_CONTENT_FIELDS}) so `undefined`/`false`/ + * absent all compare equal (a tab that was never pinned and one explicitly `pinned:false` + * are the same state) — this stops a record gaining/losing the field as mere metadata from + * looking like a content change, while a true⟷false(/absent) transition is still detected. + * Other fields compare with `null`/`undefined` coalesced so a value simply missing on one + * side and empty on the other does not churn. + * + * @param {object} resolved - the resolved (authoritative) tab record. + * @param {object} local - the matched live local tab record. + * @returns {object} `{field: resolvedValue}` for each changed field (empty ⇒ no change). + */ +function resolveTabContentChanges(resolved, local) { + const changed = {}; + for (const field of TAB_CONTENT_FIELDS) { + if (field === 'pinned' || field === 'loaded') { + // additive boolean: absent / false / undefined are all "off". + if ((resolved[field] === true) !== (local[field] === true)) { + changed[field] = resolved[field] === true; + } + } else if ((resolved[field] ?? null) !== (local[field] ?? null)) { + changed[field] = deepClone(resolved[field]); + } + } + return changed; +} + +/** + * Diff the resolved snapshot against the live local state into a declarative op set. + * + * Reconciliation is by stable identity: groups by `id`, tabs by `uid`. The ops are + * DATA ONLY — no execution — so P3b can route them through STG's existing apply + * logic (and through its skip machinery, so applied changes are not re-captured as + * new deltas). Decisions: + * - group in resolved but not local ⇒ `groupsToCreate` (full resolved group, sans + * tabs — its tabs arrive via `tabsToCreate` so creation order is independent); + * - group in local but not resolved ⇒ `groupsToRemove` (id only); + * - group in both with differing props ⇒ `groupsToUpdate` (full resolved props); + * - tab uid in resolved but not local ⇒ `tabsToCreate` with `target {groupId, index}`; + * - tab uid in local but not resolved ⇒ `tabsToRemove` (uid + its local groupId); + * - tab uid in both but group or index differs ⇒ `tabsToMove` with + * `target {groupId, index}`. (A tab resurrected by replay rule 1 that is absent + * locally surfaces as `tabsToCreate`; if it still exists locally it surfaces as a + * move/no-op — either way no work is lost.) + * - tab uid in both with differing CONTENT (any of url/title/cookieStoreId/ + * pinned[group-pin]; favIconUrl and loaded are excluded — see TAB_CONTENT_FIELDS) ⇒ + * `tabsToUpdate` with `target {…changed fields…}`. This is + * independent of the move op — a tab can be in BOTH `tabsToMove` and `tabsToUpdate`. It + * is the per-tab analogue of `groupsToUpdate`; without it a content change to a tab that + * already exists on the peer was never reconciled. + * + * ## Group ORDER reconciliation + * The resolved snapshot's `groups` ARRAY ORDER is the authoritative group order (it is + * what STG persists + shows). When the local group order differs from the resolved + * order — among the groups they share — we emit `groupsOrder`: the resolved group-id + * order. The transport reorders the saved groups array to match it before `Groups.save` + * (groups present only locally and not in the resolved order are kept, appended after + * the ordered ones — never dropped). `groupsOrder` is null when the shared groups are + * already in the same relative order, so an identity-only diff never triggers a reorder. + * + * ## Removal gate — per-device baseline (replaces the cloud-known stopgap) + * A local group/tab is REMOVED only if it is in this device's `priorBaseline` (the + * set of ids/uids it last reconciled as synced) AND absent from the resolved state. + * `in baseline ⇒ was synced before ⇒ its absence now is a delete elsewhere ⇒ remove`. + * `not in baseline ⇒ new local, never synced ⇒ keep` (it is bootstrap-uploaded, not + * removed). Unlike the old "is it in the current cloud snapshot/deltas" gate, the + * baseline is local + durable and survives cloud COMPACTION (which prunes the very + * delete events that gate relied on) — so a delete still propagates to a device that + * was offline across a compaction window. A local pending `modify` resurrects the + * item into the resolved state (modification beats deletion), so it never reaches + * the remove branch. + * + * ## Pinned tabs (global, no group) + * Diffed in parallel as a flat list keyed by `uid` (they belong to no group; target is + * just an `{index}` in the global pinned strip): + * - uid in resolved but not local ⇒ `pinnedToCreate` with `target {index}`; + * - uid in local but not resolved AND in `priorBaseline.pinnedUids` ⇒ `pinnedToRemove` + * (uid only). Same baseline gate as group tabs: a local-only pinned tab the device + * never reconciled as synced is KEPT (bootstrap-uploaded), never removed; + * - uid in both at a different index ⇒ `pinnedToMove` with `target {index}`; + * - uid in both with differing CONTENT ⇒ `pinnedToUpdate` with `target {…changed fields…}` + * (url/title/cookieStoreId; favIconUrl is excluded — see TAB_CONTENT_FIELDS; the group-pin/`loaded` flags do not apply to a + * global pinned tab and are filtered out). The pinned analogue of `tabsToUpdate`. + * + * @param {object} resolvedSnapshot - { groups: [...], pinnedTabs?: [...] } + * @param {object} localState - { groups: [...], pinnedTabs?: [...] } + * @param {{tabUids: Set<*>, groupIds: Set<*>, pinnedUids: Set<*>}} priorBaseline - normalized baseline. + * @returns {{groupsToCreate: object[], groupsToRemove: object[], groupsToUpdate: object[], + * tabsToCreate: object[], tabsToRemove: object[], tabsToMove: object[], tabsToUpdate: object[], + * pinnedToCreate: object[], pinnedToRemove: object[], pinnedToMove: object[], pinnedToUpdate: object[]}} + */ +function diffToBrowserOps(resolvedSnapshot, localState, priorBaseline = {tabUids: new Set(), groupIds: new Set(), pinnedUids: new Set()}) { + const resolvedGroups = resolvedSnapshot.groups || []; + const localGroups = (localState && localState.groups) || []; + + const localGroupById = new Map(localGroups.map(g => [g.id, g])); + const resolvedGroupById = new Map(resolvedGroups.map(g => [g.id, g])); + + const groupsToCreate = []; + const groupsToRemove = []; + const groupsToUpdate = []; + + for (const group of resolvedGroups) { + const local = localGroupById.get(group.id); + if (!local) { + const {tabs, ...props} = group; + void tabs; + groupsToCreate.push(deepClone(props)); + } else if (stableStringify(groupProps(group)) !== stableStringify(groupProps(local))) { + groupsToUpdate.push(deepClone(groupProps(group))); + } + } + + for (const group of localGroups) { + // Only remove a local group this device previously reconciled as synced + // (in baseline) and that the resolved state has now dropped — a delete + // elsewhere. A group not in the baseline is new-local, never synced — keep + // it (it is bootstrap-uploaded), never delete. + if (!resolvedGroupById.has(group.id) && priorBaseline.groupIds.has(group.id)) { + groupsToRemove.push({id: group.id}); + } + } + + const resolvedTabs = indexTabs(resolvedSnapshot); + const localTabs = indexTabs({groups: localGroups}); + + const tabsToCreate = []; + const tabsToRemove = []; + const tabsToMove = []; + const tabsToUpdate = []; + + for (const [uid, {groupId, index, tab}] of resolvedTabs) { + const local = localTabs.get(uid); + if (!local) { + tabsToCreate.push({ + ...deepClone(tab), + target: {groupId, index}, + }); + } else { + // placement (group/index) reconciles via tabsToMove… + if (local.groupId !== groupId || local.index !== index) { + tabsToMove.push({ + uid, + target: {groupId, index}, + }); + } + // …and CONTENT (url/title/container/group-pin) via tabsToUpdate. + // A tab can legitimately be in BOTH (it moved AND its content changed). Without + // this branch a content change to an already-existing peer tab was never + // reconciled (the "half-wired" sync bug). + const changed = resolveTabContentChanges(tab, local.tab); + if (Object.keys(changed).length) { + tabsToUpdate.push({uid, target: changed}); + } + } + } + + for (const [uid, {groupId}] of localTabs) { + // Only remove a local tab this device previously reconciled as synced (in + // baseline) and that the resolved state has now dropped — a delete elsewhere. + // A uid not in the baseline is new-local, never synced (e.g. a tab that + // pre-dates delta tracking) — keep it (it is bootstrap-uploaded), never + // delete. This was the data-loss bug. + if (!resolvedTabs.has(uid) && priorBaseline.tabUids.has(uid)) { + tabsToRemove.push({uid, groupId}); + } + } + + // --- pinned tabs (flat global list, keyed by uid; target is just {index}) --- + const indexPinned = list => { + const byUid = new Map(); + (Array.isArray(list) ? list : []).forEach((tab, index) => { + if (tab && tab.uid != null) { + byUid.set(tab.uid, {index, tab}); + } + }); + return byUid; + }; + + const resolvedPinned = indexPinned(resolvedSnapshot.pinnedTabs); + const localPinned = indexPinned(localState && localState.pinnedTabs); + + const pinnedToCreate = []; + const pinnedToRemove = []; + const pinnedToMove = []; + const pinnedToUpdate = []; + + for (const [uid, {index, tab}] of resolvedPinned) { + const local = localPinned.get(uid); + if (!local) { + pinnedToCreate.push({ + ...deepClone(tab), + target: {index}, + }); + } else { + if (local.index !== index) { + pinnedToMove.push({uid, target: {index}}); + } + // CONTENT (url/title/cookieStoreId) change on an existing GLOBAL pinned tab. + // The group-pin flag is meaningless for a global pinned tab, so it is filtered + // out below. (`loaded` is no longer a TAB_CONTENT_FIELD — it never reaches here + // — so it needs no explicit delete; a discarded global pinned tab is never + // force-loaded by the apply, the reason `loaded` was excluded from the diff.) + const changed = resolveTabContentChanges(tab, local.tab); + delete changed.pinned; + if (Object.keys(changed).length) { + pinnedToUpdate.push({uid, target: changed}); + } + } + } + + for (const [uid] of localPinned) { + // Same baseline gate as group tabs: only remove a local pinned tab this device + // previously reconciled as synced and that the resolved state has now dropped. + // A pinned uid not in the baseline is new-local, never synced — keep it (it is + // bootstrap-uploaded), never delete. + if (!resolvedPinned.has(uid) && priorBaseline.pinnedUids.has(uid)) { + pinnedToRemove.push({uid}); + } + } + + const groupsOrder = computeGroupsOrder(resolvedGroups, localGroups); + + return { + groupsToCreate, groupsToRemove, groupsToUpdate, + tabsToCreate, tabsToRemove, tabsToMove, tabsToUpdate, groupsOrder, + pinnedToCreate, pinnedToRemove, pinnedToMove, pinnedToUpdate, + }; +} + +/** + * Compute the resolved group-id order to impose locally, or null if the groups the + * resolved and local states SHARE are already in the same relative order. + * + * The resolved `groups` array order is authoritative. We compare the resolved order + * restricted to ids that also exist locally against the local order restricted to ids + * that also exist in the resolved state — i.e. only the shared groups, in each side's + * order. If those sequences match, no reorder is needed (groups created/removed this + * round are handled by groupsToCreate/groupsToRemove and don't by themselves count as + * a reorder). Otherwise we return the full resolved group-id order; the transport maps + * it onto the saved array (keeping any local-only group, appended at the end). + * + * @param {object[]} resolvedGroups + * @param {object[]} localGroups + * @returns {Array<*>|null} resolved group-id order, or null when already in order. + */ +function computeGroupsOrder(resolvedGroups, localGroups) { + const localIds = new Set(localGroups.map(g => g.id)); + const resolvedIds = new Set(resolvedGroups.map(g => g.id)); + + const resolvedShared = resolvedGroups.map(g => g.id).filter(id => localIds.has(id)); + const localShared = localGroups.map(g => g.id).filter(id => resolvedIds.has(id)); + + const sameOrder = resolvedShared.length === localShared.length + && resolvedShared.every((id, i) => id === localShared[i]); + + if (sameOrder) { + return null; + } + + return resolvedGroups.map(g => g.id); +} + +/** + * Diff the resolved synced options against the local option values into the subset to + * APPLY locally — `{key: value}` for every resolved key whose value differs from the + * local one. Equality is by stable JSON so object/array values (e.g. `defaultGroupProps`, + * `hotkeys`) compare by content, not reference. Keys absent from `resolvedOptions` are + * never touched (an option is only ever set, never deleted, by sync); a key whose + * resolved value equals local is omitted (no spurious write / side-effect). + * + * The synced-key filtering happens UPSTREAM (capture only logs synced keys → replay only + * resolves those), so `resolvedOptions` already excludes `sync*`/`autoBackup*`. + * + * @param {object} resolvedOptions - resolved synced settings from replay. + * @param {object} localOptions - this device's current values for the same keys. + * @returns {object} `{key: value}` to apply locally (empty when nothing differs). + */ +function diffOptionsToApply(resolvedOptions, localOptions) { + const resolved = resolvedOptions || {}; + const local = localOptions || {}; + const toApply = {}; + + for (const key of Object.keys(resolved)) { + if (JSON.stringify(resolved[key]) !== JSON.stringify(local[key])) { + toApply[key] = deepClone(resolved[key]); + } + } + + return toApply; +} + +/** + * Plan a sync round — PURE. See module docs for the full contract. + * + * @param {object} args + * @param {object} args.pulledSnapshot - base snapshot pulled from the cloud. + * @param {Array<{deviceId: string, events: object[]}>} args.pulledDeltaLogs + * @param {object[]} args.localPendingEvents - self events not yet pushed. + * @param {string} args.selfDeviceId + * @param {object} args.localState - { groups: [...] } live browser state. + * @param {{tabUids?: (Set<*>|Array<*>), groupIds?: (Set<*>|Array<*>), pinnedUids?: (Set<*>|Array<*>)}} [args.priorBaseline] + * this device's last-synced baseline (ids/uids it reconciled as synced at the + * end of its previous successful sync). Gates removals. Default empty. + * @returns {{resolvedSnapshot: object, newWatermark: object, deltaFileToWrite: ?object, browserOps: object}} + */ +export function planSync({pulledSnapshot, pulledDeltaLogs, localPendingEvents, selfDeviceId, localState, priorBaseline}) { + const {fullLogs, selfEvents} = buildFullLogs(pulledDeltaLogs, localPendingEvents, selfDeviceId); + + const {snapshot: resolvedSnapshot, watermark: newWatermark} = replay(pulledSnapshot || {groups: []}, fullLogs); + + // We "changed" the self log iff there were pending events to push. Compare + // against what the pulled self log already held. + const pulledSelfLog = (pulledDeltaLogs || []).find(log => log.deviceId === selfDeviceId); + const pulledSelfCount = pulledSelfLog && Array.isArray(pulledSelfLog.events) ? pulledSelfLog.events.length : 0; + + const deltaFileToWrite = selfEvents.length > pulledSelfCount + ? {deviceId: selfDeviceId, events: selfEvents} + : null; + + // This device's last-synced baseline — gates removals so we never delete + // local groups/tabs this device never reconciled as synced (new-local items), + // while still propagating deletes for items it DID sync (survives compaction). + const baseline = normalizeBaseline(priorBaseline); + + const browserOps = diffToBrowserOps(resolvedSnapshot, localState || {groups: []}, baseline); + + const optionsToApply = diffOptionsToApply(resolvedSnapshot.options, (localState || {}).options); + + return { + resolvedSnapshot, + newWatermark, + deltaFileToWrite, + browserOps, + optionsToApply, + }; +} + +/** + * Compute the synthetic "bootstrap" add events this device must emit so its + * never-synced local groups/tabs get uploaded to the cloud (instead of silently + * lingering local-only forever). PURE — returns plain payloads; the transport + * (`delta-sync.js`) assigns seqs by appending them to the local delta log. + * + * A local item needs bootstrapping iff it is BOTH: + * - NOT in `priorBaseline` (it was never reconciled as synced), AND + * - NOT already referenced by this device's local delta log (no add/event for it + * yet) — `knownLocalLogUids` / `knownLocalLogGroupIds`. + * Both checks make this idempotent: re-running a sync re-derives the same set and + * skips anything already in the baseline or already logged, so it never double-adds. + * + * Group adds are ordered BEFORE their tabs so a replay/apply that consumes them in + * order always has the target group available first. Option bootstraps come last (they + * are independent of groups/tabs). + * + * ## Options bootstrap + * Each synced local option value in `localState.options` becomes an `option.set` unless + * it is already in the baseline's `optionKeys` (reconciled as synced before) or already + * logged (`knownLocalLogOptionKeys`). Same idempotency contract as groups/tabs. The + * caller supplies `localState.options` already filtered to the synced subset, so this + * never uploads a per-device key. + * + * ## Pinned bootstrap + * Each local global-pinned tab in `localState.pinnedTabs` becomes a `pinned.add` unless + * it is already in the baseline's `pinnedUids` or already logged (`knownLocalLogUids` — + * pinned ops are keyed by tab uid, the same namespace as group tab uids). Emitted after + * groups/tabs (they are independent of any group). Same idempotency contract. + * + * @param {object} localState - { groups: [{id, ...props, tabs:[{uid,...}]}], pinnedTabs?: [{uid,...}], options?: {key: value} }. + * @param {{tabUids?: (Set<*>|Array<*>), groupIds?: (Set<*>|Array<*>), optionKeys?: (Set<*>|Array<*>), pinnedUids?: (Set<*>|Array<*>)}} [priorBaseline] + * @param {Set<*>|Array<*>} [knownLocalLogUids] - tab uids already in this device's log (group + pinned share the uid namespace). + * @param {Set<*>|Array<*>} [knownLocalLogGroupIds] - group ids already in this device's log. + * @param {Set<*>|Array<*>} [knownLocalLogOptionKeys] - option keys already in this device's log. + * @returns {Array<{op: string, group?: object, groupId?: *, tab?: object, key?: string, value?: *}>} + */ +export function computeBootstrapEvents(localState, priorBaseline, knownLocalLogUids, knownLocalLogGroupIds, knownLocalLogOptionKeys) { + const baseline = normalizeBaseline(priorBaseline); + const logUids = new Set(knownLocalLogUids || []); + const logGroupIds = new Set(knownLocalLogGroupIds || []); + const logOptionKeys = new Set(knownLocalLogOptionKeys || []); + + const events = []; + const groups = (localState && localState.groups) || []; + + // Track every uid that belongs to a group tab in THIS bootstrap so we never also emit a + // global `pinned.add` for the same uid (a uid is EITHER a group tab OR a global pin, never + // both). delta-sync.getLivePinnedTabs already excludes group tabs from the global pinned + // set, so localState.pinnedTabs should not overlap localState.groups[].tabs — this is a + // belt-and-suspenders guard so the double-identity (group tab.add + pinned.add for one uid + // → a DUPLICATE pinned copy on the peer) can never occur even if a leaked browser-pinned + // group tab ever slips into pinnedTabs. See delta-sync.js getLivePinnedTabs. + const groupTabUids = new Set(); + + for (const group of groups) { + if (group.id == null) { + continue; + } + + // group.add first, so its tabs always have a target on replay/apply + if (!baseline.groupIds.has(group.id) && !logGroupIds.has(group.id)) { + const {tabs, ...props} = group; + void tabs; + events.push({op: 'group.add', group: deepClone(props)}); + } + + for (const tab of Array.isArray(group.tabs) ? group.tabs : []) { + if (tab.uid == null) { + continue; + } + groupTabUids.add(tab.uid); + if (!baseline.tabUids.has(tab.uid) && !logUids.has(tab.uid)) { + events.push({op: 'tab.add', groupId: group.id, tab: deepClone(tab)}); + } + } + } + + const localOptions = (localState && localState.options) || {}; + for (const key of Object.keys(localOptions)) { + if (!baseline.optionKeys.has(key) && !logOptionKeys.has(key)) { + events.push({op: 'option.set', key, value: deepClone(localOptions[key])}); + } + } + + // global pinned tabs (no group): bootstrap each never-synced one as a pinned.add. + // Keyed by tab uid (same namespace as group tabs), gated by baseline.pinnedUids and + // the log uids so re-runs never double-add. + const localPinnedTabs = (localState && localState.pinnedTabs) || []; + for (const tab of Array.isArray(localPinnedTabs) ? localPinnedTabs : []) { + if (tab.uid == null) { + continue; + } + // DOUBLE-IDENTITY GUARD: a uid that belongs to a group tab is NOT a global pin. + // Never emit pinned.add for it, even if it leaked into pinnedTabs — that would + // create a duplicate pinned copy of a tab that's also a group tab on the peer. + if (groupTabUids.has(tab.uid)) { + continue; + } + if (!baseline.pinnedUids.has(tab.uid) && !logUids.has(tab.uid)) { + events.push({op: 'pinned.add', tab: deepClone(tab)}); + } + } + + return events; +} + +/** + * Derive a fresh baseline from a resolved snapshot: every group id and every tab + * uid present in it. PURE. The transport persists this after a SUCCESSFUL sync so + * that the next round can tell "synced before, now gone" (remove) from "never + * synced" (keep / bootstrap). See {@link diffToBrowserOps} removal gate. + * + * @param {object} snapshot - { groups: [{id, tabs:[{uid,...}]}], pinnedTabs?: [{uid,...}], options?: {key: value} }. + * @returns {{tabUids: Array<*>, groupIds: Array<*>, optionKeys: Array<*>, pinnedUids: Array<*>}} + */ +export function baselineFromSnapshot(snapshot) { + const tabUids = []; + const groupIds = []; + + for (const group of (snapshot && snapshot.groups) || []) { + if (group.id != null) { + groupIds.push(group.id); + } + for (const tab of Array.isArray(group.tabs) ? group.tabs : []) { + if (tab.uid != null) { + tabUids.push(tab.uid); + } + } + } + + const optionKeys = Object.keys((snapshot && snapshot.options) || {}); + + const pinnedUids = []; + for (const tab of Array.isArray(snapshot && snapshot.pinnedTabs) ? snapshot.pinnedTabs : []) { + if (tab.uid != null) { + pinnedUids.push(tab.uid); + } + } + + return {tabUids, groupIds, optionKeys, pinnedUids}; +} diff --git a/addon/src/js/sync/delta/replay.js b/addon/src/js/sync/delta/replay.js new file mode 100644 index 00000000..42bbee95 --- /dev/null +++ b/addon/src/js/sync/delta/replay.js @@ -0,0 +1,692 @@ + +/** + * Pure replay / merge engine for hybrid snapshot + delta sync (Phase P2). + * + * Given a compacted base snapshot and a set of per-device delta logs, this module + * computes the effective state by replaying every event in a deterministic global + * order and resolving conflicts per the rules in `.project/DESIGN_DELTA_SYNC.md` + * ("Replay / conflict rules"). It is the heart of the reliability story: no event is + * ever silently dropped, and concurrent edits degrade to a duplicate rather than to + * lost work. + * + * ## Purity (hard requirement) + * This module is PURE: it touches no `browser.*` API, no DOM, and imports nothing + * that performs side effects at evaluation time (in particular NOT `constants.js`, + * which fetches at module load). It clones its inputs and never mutates them. This + * lets the engine — and its conflict rules, the riskiest part of sync — run and be + * tested under plain `node` with no extension host. Any constant it needs (e.g. the + * default `cookieStoreId`) is a local literal or a parameter. + * + * Later phases consume this: P3 (transport) pulls the snapshot + delta files, calls + * {@link replay}, diffs the resolved snapshot against the live browser and applies it + * via existing STG apply logic; P4 (compaction) folds events into a new base using + * THIS SAME function, guaranteeing the compacted base equals the replayed state. + * + * ## Snapshot shape + * { + * groups: [ { id, title, ...props, tabs: [ { uid, url, title, cookieStoreId?, index?, ... } ] } ], + * pinnedTabs?: [ { uid, url, title, cookieStoreId?, index?, ... } ], // global pinned, missing ⇒ [] + * options?: { [key]: value } // resolved synced global settings; missing ⇒ {} + * containers?: { [portableKey]: {name, color, icon} } // portable container registry; missing ⇒ {} + * watermark?: { [deviceId]: lastFoldedSeq } // missing ⇒ {} + * } + * + * All `cookieStoreId` values in tab/group/pinned/`defaultGroupProps` records are PORTABLE + * KEYS (a container's `name+color+icon` identity, or a reserved default/temporary marker), + * never a raw per-install `cookieStoreId`. The `containers` registry maps each portable key + * to the `{name, color, icon}` the receiving device needs to find-or-create the matching + * local container. Translation local⟷portable happens ONLY at the impure boundary in + * `delta-sync.js` (see `container-map.js`); replay is portable-key-agnostic and just carries + * the registry through untouched (it is the same on every device for a given identity). + * + * `pinnedTabs` is a single ordered array of the window-global pinned tabs (they belong + * to no STG group; this mirrors the legacy backup field `data.pinnedTabs`). Pinned + * events (`pinned.*`) fold into it under the SAME conflict rules as group tabs: + * modify-beats-delete resurrects, watermark dedups, identity is `uid`. + * + * ## Delta log shape (per P1 `delta-log.js`) + * { deviceId, events: [ { seq, ts, op, ...payload } ] } + * Payloads by op: + * tab.add { groupId, tab: } + * tab.modify { groupId, tab: } + * tab.move { groupId, uid, toIndex } + * tab.remove { groupId, uid } + * group.add { group: } + * group.modify { group: } + * group.remove { groupId } + * option.set { key, value } + * pinned.add { tab: } + * pinned.modify { tab: } + * pinned.move { uid, toIndex } + * pinned.remove { uid } + * + * ## Global options resolution (per-key last-writer-wins) + * `option.set` events fold into `snapshot.options` — a flat `{key: value}` bag of the + * synced global STG settings. Resolution is per-key LAST-WRITER-WINS by the winning + * event's `ts`: as events are applied in global order (ts asc, tie-break deviceId then + * seq), a later-or-equal `ts` for a key overrides the value held for it. Because the + * global order is already total and deterministic, "process in order, later overrides" + * is exactly LWW with the (deviceId, seq) tie-break — we don't track a separate per-key + * ts since the apply order encodes it. Which keys are eligible is decided UPSTREAM (the + * capture layer only emits `option.set` for synced keys); replay folds whatever arrives. + * + * @module sync/delta/replay + */ + +/** + * Default container used when a recreated tab carries no `cookieStoreId`. Mirrors + * `Constants.DEFAULT_COOKIE_STORE_ID_FIREFOX` but is a local literal so this module + * stays import-free / pure (see module docs). It is only a cosmetic default for a + * resurrected tab record; the merge logic itself never depends on it. + */ +const DEFAULT_COOKIE_STORE_ID = 'firefox-default'; + +/** + * Matches a canonical UUID (e.g. the `randomUUID()` group ids STG mints). Used to detect + * a group whose only available "title" is in fact its raw id, so we never persist a UUID + * as a human-readable group name (see {@link defaultGroupTitle} / {@link ensureGroup}). + */ +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * STG default group-title template. Mirrors `Lang('newGroupTitle', uid)` ("Group $id$", + * see `_locales/.../messages.json`) with `uid = groups.extractUId(id)` (the id's last 4 + * chars). Duplicated as a literal here so the replay engine stays import-free / pure + * (it cannot pull `groups.js`, which needs `Lang`/`Constants`/`Utils`). The visible + * result for a UUID-titled group therefore matches what STG itself would name a new group. + * + * @param {string} groupId + * @returns {string} + */ +function defaultGroupTitle(groupId) { + const uid = (groupId == null ? '' : String(groupId)).slice(-4) || '{uid}'; + return `Group ${uid}`; +} + +/** + * Normalise a group title so a raw id / UUID is never written as a display title. + * + * A `tab.add`/`tab.modify` can reference a group whose `group.add` the receiver never saw + * (missed or compacted away on this peer) — {@link ensureGroup} then fabricates the group + * and, lacking any real props, used to fall back to the id itself, surfacing a group named + * with a raw UUID in the UI with no later event to repair it (the sender won't re-emit + * `group.add` for an id already in its baseline). When the only title we have IS the id or + * looks like a UUID, substitute STG's own default name instead. + * + * @param {*} title - the candidate title (may be undefined/null/the id). + * @param {string} groupId + * @returns {string} a human-readable title (STG default when the candidate is a UUID/id). + */ +function sanitizeGroupTitle(title, groupId) { + const str = title == null ? '' : String(title); + if (!str || str === String(groupId) || UUID_RE.test(str)) { + return defaultGroupTitle(groupId); + } + return str; +} + +/** + * Additive boolean tab flags whose ABSENCE on an incoming record must NOT be treated as + * "turn off". `pinned` (group-scoped pin) and `loaded` (source loaded-state) are written + * by capture only when relevant, so a `tab.modify` that carries neither could historically + * WIPE a previously-synced `pinned:true`/`loaded:true` (the upserts wholesale-replace the + * record). See {@link preserveAdditiveFlags}. + */ +const ADDITIVE_TAB_FLAGS = ['pinned', 'loaded']; + +/** + * Clobber-safety for the wholesale-replace upserts: when an incoming add/modify record + * OMITS an additive flag (`pinned`/`loaded`) that the record it is replacing carried, + * carry the prior value forward onto the incoming record — preserve, don't wipe. + * + * Rationale (the chosen fix; see DESIGN_DELTA_SYNC clobber note): capture now ALWAYS emits + * these flags explicitly (true OR false), so a record produced by the current code never + * omits them and a genuine un-pin / un-load carries `false` and DOES clear the flag. The + * preserve here is the safety net for LEGACY records already in the cloud (captured before + * the always-emit change) which carry the flag only when true: such a record omitting the + * flag is ambiguous, and the safe reading — given events replay in global ts order so the + * prior value reflects every earlier event — is "no information ⇒ keep what we had". A + * present `false` is unambiguous and still clears it. `incoming` is mutated in place. + * + * @param {object} incoming - the record about to be inserted (mutated). + * @param {object|undefined} prior - the same-uid record being replaced, if any. + */ +function preserveAdditiveFlags(incoming, prior) { + if (!prior) { + return; + } + for (const flag of ADDITIVE_TAB_FLAGS) { + if (!Object.hasOwn(incoming, flag) && Object.hasOwn(prior, flag)) { + incoming[flag] = prior[flag]; + } + } +} + +/** Op constants — kept in sync with `delta-log.js` OPS, duplicated to stay import-free. */ +const OPS = { + TAB_ADD: 'tab.add', + TAB_MODIFY: 'tab.modify', + TAB_MOVE: 'tab.move', + TAB_REMOVE: 'tab.remove', + GROUP_ADD: 'group.add', + GROUP_MODIFY: 'group.modify', + GROUP_MOVE: 'group.move', + GROUP_REMOVE: 'group.remove', + OPTION_SET: 'option.set', + PINNED_ADD: 'pinned.add', + PINNED_MODIFY: 'pinned.modify', + PINNED_MOVE: 'pinned.move', + PINNED_REMOVE: 'pinned.remove', +}; + +/** + * Structured deep clone of a plain JSON-ish value (the shapes here are JSON: arrays, + * objects, primitives). Avoids `structuredClone` to keep behaviour identical under + * any node/browser version and to make the purity obvious. + * @template T + * @param {T} value + * @returns {T} + */ +function deepClone(value) { + if (value === null || typeof value !== 'object') { + return value; + } + if (Array.isArray(value)) { + return value.map(deepClone); + } + const out = {}; + for (const key of Object.keys(value)) { + out[key] = deepClone(value[key]); + } + return out; +} + +/** + * Build the deterministic global replay order from all device logs. + * + * Per the design: order by `ts` ascending, tie-break by `deviceId` then `seq`. The + * `(deviceId, seq)` tie-break makes the order total and reproducible regardless of + * the order the logs were supplied in; within one device `seq` is authoritative even + * if two events happen to share a `ts`. + * + * @param {Array<{deviceId: string, events: object[]}>} deltaLogs + * @returns {Array<{deviceId: string, event: object}>} flattened, sorted entries + */ +function buildOrderedEvents(deltaLogs) { + const entries = []; + + for (const log of deltaLogs || []) { + const deviceId = log?.deviceId; + for (const event of log?.events || []) { + entries.push({deviceId, event}); + } + } + + entries.sort((a, b) => { + const tsA = a.event.ts ?? 0; + const tsB = b.event.ts ?? 0; + if (tsA !== tsB) { + return tsA - tsB; + } + if (a.deviceId !== b.deviceId) { + return a.deviceId < b.deviceId ? -1 : 1; + } + return (a.event.seq ?? 0) - (b.event.seq ?? 0); + }); + + return entries; +} + +/** + * Insert `tab` into `group.tabs` at the GROUP-RELATIVE position `index` (its 0-based + * slot within this group's ordered tab list), keeping BOTH on a hard index conflict. + * + * `index` is strictly an array position within `group.tabs` — NOT a browser-window + * index. Capture stores it group-relative (see delta-capture.js); replaying it as an + * array slot is what keeps tab order identical across machines. + * + * Implements design rule 2 ("Duplicate on hard index conflict"): the incoming tab + * takes the requested slot and whatever already sat there (and everything after it) + * shifts down — no overwrite, no drop. A missing or out-of-range index CLAMPS to + * append-at-end. The caller is responsible for having removed any same-uid tab first + * when that is the intended semantics (add/move do; this helper is uid-agnostic). + * + * @param {{tabs: object[]}} group + * @param {object} tab + * @param {number} [index] + */ +function insertTabAt(group, tab, index) { + const len = group.tabs.length; + let at = Number.isInteger(index) ? index : len; + if (at < 0) { + at = 0; + } + if (at > len) { + at = len; + } + group.tabs.splice(at, 0, tab); +} + +/** + * Find `[group, tabIndex]` for a tab `uid` anywhere in the snapshot, or nulls. + * @param {object[]} groups + * @param {string} uid + * @returns {{group: object|null, tabIndex: number}} + */ +function findTab(groups, uid) { + for (const group of groups) { + const tabIndex = group.tabs.findIndex(t => t.uid === uid); + if (tabIndex !== -1) { + return {group, tabIndex}; + } + } + return {group: null, tabIndex: -1}; +} + +/** + * Resolve (and if necessary recreate) the target group for a tab event. + * + * Implements design rule 3 ("Group fallback"): a tab event that references a group + * which is gone recreates a MINIMAL group with that id so the tab still lands + * somewhere recoverable rather than being dropped. We intentionally recreate by id + * (not a single shared fallback bucket) so a later `group.add`/`group.modify` for the + * same id naturally reconciles onto it (add-of-existing ⇒ modify) and the user keeps + * the original grouping. Title defaults to STG's own "Group " name (never the raw + * id/UUID — see {@link sanitizeGroupTitle}); the real title arrives via group.modify. + * + * @param {object[]} groups + * @param {string} groupId + * @returns {object} the existing or newly-created group + */ +function ensureGroup(groups, groupId) { + let group = groups.find(g => g.id === groupId); + if (!group) { + group = {id: groupId, title: defaultGroupTitle(groupId), tabs: []}; + groups.push(group); + } + if (!Array.isArray(group.tabs)) { + group.tabs = []; + } + return group; +} + +/** + * Apply a single tab.add / tab.modify event (they share resurrection semantics). + * + * - add of an existing uid ⇒ behaves as modify (design rule 5); + * - modify of an absent uid ⇒ re-create it (design rule 1, modify-beats-delete); + * - the tab always ends up in `groupId` (recreated if gone — rule 3); + * - placement honours the event's `index` with the duplicate-on-conflict rule. + * + * @param {object[]} groups + * @param {object} event - { groupId, tab } + */ +function applyTabUpsert(groups, event) { + const incoming = deepClone(event.tab); + if (!incoming || incoming.uid == null) { + return; // malformed; nothing to key on + } + + if (incoming.cookieStoreId == null) { + incoming.cookieStoreId = DEFAULT_COOKIE_STORE_ID; + } + + // Remove any existing copy of this uid first (modify replaces the full record, + // add-of-existing folds to modify). This also handles a uid that had drifted into + // a different group: it is re-homed to the event's groupId. + const existing = findTab(groups, incoming.uid); + if (existing.group) { + // clobber-safety: an incoming record omitting pinned/loaded must not wipe a + // previously-synced true value (legacy records carry these only when true). + preserveAdditiveFlags(incoming, existing.group.tabs[existing.tabIndex]); + existing.group.tabs.splice(existing.tabIndex, 1); + } + + const target = ensureGroup(groups, event.groupId); + insertTabAt(target, incoming, incoming.index); +} + +/** + * Apply a tab.move event: move `uid` to `toIndex` within `groupId`. + * + * - unknown uid ⇒ ignore (design: a later modify resurrects if needed); + * - non-existent/out-of-range index ⇒ append at end (rule 3); + * - duplicate-on-conflict applies at the destination slot (rule 2). + * + * @param {object[]} groups + * @param {object} event - { groupId, uid, toIndex } + */ +function applyTabMove(groups, event) { + const found = findTab(groups, event.uid); + if (!found.group) { + return; // unknown uid — ignore + } + + const [tab] = found.group.tabs.splice(found.tabIndex, 1); + + const target = ensureGroup(groups, event.groupId); + insertTabAt(target, tab, event.toIndex); +} + +/** + * Apply a tab.remove event: drop `uid` wherever it lives. Absent ⇒ no-op (rule 5). + * @param {object[]} groups + * @param {object} event - { uid } + */ +function applyTabRemove(groups, event) { + const found = findTab(groups, event.uid); + if (found.group) { + found.group.tabs.splice(found.tabIndex, 1); + } +} + +/** + * Insert `tab` into the flat `list` at `index`, keeping BOTH on a hard index conflict. + * The pinned-tab analogue of {@link insertTabAt} (same rule 2 semantics) on a plain + * array rather than a group's `tabs`. Missing / out-of-range index clamps to append. + * @param {object[]} list + * @param {object} tab + * @param {number} [index] + */ +function insertInListAt(list, tab, index) { + const len = list.length; + let at = Number.isInteger(index) ? index : len; + if (at < 0) { + at = 0; + } + if (at > len) { + at = len; + } + list.splice(at, 0, tab); +} + +/** + * Apply a pinned.add / pinned.modify event onto the flat `pinnedTabs` array. + * + * Same resurrection semantics as {@link applyTabUpsert} but on the single global + * pinned list (no group): add-of-existing folds to modify, modify-of-absent + * re-creates (rule 1, modify-beats-delete), placement honours `index` with the + * duplicate-on-conflict rule. + * + * @param {object[]} pinnedTabs + * @param {object} event - { tab } + */ +function applyPinnedUpsert(pinnedTabs, event) { + const incoming = deepClone(event.tab); + if (!incoming || incoming.uid == null) { + return; // malformed; nothing to key on + } + + if (incoming.cookieStoreId == null) { + incoming.cookieStoreId = DEFAULT_COOKIE_STORE_ID; + } + + const existingIdx = pinnedTabs.findIndex(t => t.uid === incoming.uid); + if (existingIdx !== -1) { + // clobber-safety: see applyTabUpsert. `loaded` is the only additive flag that + // applies to a global pinned tab, but preserve both for uniformity. + preserveAdditiveFlags(incoming, pinnedTabs[existingIdx]); + pinnedTabs.splice(existingIdx, 1); + } + + insertInListAt(pinnedTabs, incoming, incoming.index); +} + +/** + * Apply a pinned.move event: move `uid` to `toIndex` within the global pinned list. + * Unknown uid ⇒ ignore (a later modify resurrects); out-of-range index ⇒ append. + * @param {object[]} pinnedTabs + * @param {object} event - { uid, toIndex } + */ +function applyPinnedMove(pinnedTabs, event) { + const idx = pinnedTabs.findIndex(t => t.uid === event.uid); + if (idx === -1) { + return; // unknown uid — ignore + } + const [tab] = pinnedTabs.splice(idx, 1); + insertInListAt(pinnedTabs, tab, event.toIndex); +} + +/** + * Apply a pinned.remove event: drop `uid` from the global pinned list. Absent ⇒ no-op. + * @param {object[]} pinnedTabs + * @param {object} event - { uid } + */ +function applyPinnedRemove(pinnedTabs, event) { + const idx = pinnedTabs.findIndex(t => t.uid === event.uid); + if (idx !== -1) { + pinnedTabs.splice(idx, 1); + } +} + +/** + * Apply a group.add / group.modify event. + * + * - add of an existing id ⇒ behaves as modify (rule 5); + * - last-write-wins on group props; + * - **tab membership is driven by tab.* events, never by group events.** Decision + * (per design note): we merge the event's group props onto the group but KEEP the + * replayed `tabs` array. A group.modify carrying a stale `tabs` snapshot must not + * clobber tabs that tab.add/move/remove events have already resolved. A brand-new + * group created here adopts the event's `tabs` only if it carried any (an add can + * legitimately seed initial tabs), else an empty array. + * + * @param {object[]} groups + * @param {object} event - { group } + */ +function applyGroupUpsert(groups, event) { + const incoming = deepClone(event.group); + if (!incoming || incoming.id == null) { + return; + } + + const existing = groups.find(g => g.id === incoming.id); + const {tabs: incomingTabs, ...props} = incoming; + + // Never let a real group event seed/overwrite the title with the raw id/UUID. A + // genuine `group.add` carries the human title; a fabricated-elsewhere fallback (or a + // sender that lost the title) must not surface a UUID as a group name. Only coerce + // when the incoming event actually carries a `title` (a partial group.modify that + // omits it must leave the existing title untouched). + if (Object.hasOwn(props, 'title')) { + props.title = sanitizeGroupTitle(props.title, incoming.id); + } + + if (existing) { + // merge props; keep the replayed tabs (membership owned by tab.* events) + Object.assign(existing, props); + if (!Array.isArray(existing.tabs)) { + existing.tabs = []; + } + } else { + groups.push({ + ...props, + tabs: Array.isArray(incomingTabs) ? deepClone(incomingTabs) : [], + }); + } +} + +/** + * Apply a group.move event: reorder `groupId` to 0-based array position `toIndex`. + * + * Group order IS array position in the snapshot (see replay header + plan-sync), so a + * reorder is a splice-out / insert-at. Last-writer-wins falls out of the global event + * order: the latest group.move (by ts, tie-break deviceId/seq) is applied last and wins. + * An unknown groupId is a no-op (rule 5). A missing/out-of-range `toIndex` clamps to the + * end (append), matching the tab.move append-on-unknown-index convention. + * + * @param {object[]} groups + * @param {object} event - { groupId, toIndex } + */ +function applyGroupMove(groups, event) { + const from = groups.findIndex(g => g.id === event.groupId); + if (from === -1) { + return; // unknown group — ignore + } + + const [group] = groups.splice(from, 1); + + let to = Number.isInteger(event.toIndex) ? event.toIndex : groups.length; + if (to < 0) { + to = 0; + } + if (to > groups.length) { + to = groups.length; + } + + groups.splice(to, 0, group); +} + +/** + * Apply a group.remove event: drop the group and its tabs. + * + * Decision (per design): removing a group discards its current tabs. They are NOT + * preserved, because a tab that genuinely survives the group's deletion will be + * resurrected by its own later tab.modify into a (re-created) group via rule 1/3. + * Removing an absent group is a no-op (rule 5). + * + * @param {object[]} groups + * @param {object} event - { groupId } + */ +function applyGroupRemove(groups, event) { + const idx = groups.findIndex(g => g.id === event.groupId); + if (idx !== -1) { + groups.splice(idx, 1); + } +} + +/** + * Replay delta logs on top of a base snapshot and return the resolved state. + * + * Pure and deterministic: inputs are deep-cloned, never mutated. Events are merged + * across all logs into one global order (ts asc, tie-break deviceId then seq), then + * applied with the conflict rules documented above. Events already folded into the + * base — `seq <= watermark[deviceId]` — are skipped (design rule 4, watermark dedup). + * + * @param {object} baseSnapshot - { groups: [...], watermark?: { [deviceId]: seq } } + * @param {Array<{deviceId: string, events: object[]}>} deltaLogs + * @param {object} [options] - reserved for future tuning; currently unused. + * @returns {{snapshot: {groups: object[], pinnedTabs: object[], options: object, containers: object, watermark: object}, watermark: object}} + * The resolved snapshot (groups + global pinnedTabs + resolved options + carried-through + * container registry + updated watermark) and, for convenience, the same watermark object. + * `watermark[deviceId]` is the max applied seq + * per device — the max of the base watermark and the highest seq that survived dedup + * for that device. `snapshot.options` is the per-key LWW-resolved synced settings. + */ +export function replay(baseSnapshot, deltaLogs = [], options = {}) { + void options; + + const groups = deepClone(baseSnapshot?.groups || []).map(group => ({ + ...group, + tabs: Array.isArray(group.tabs) ? group.tabs : [], + })); + + // Global pinned tabs fold into this flat ordered array (mirrors legacy data.pinnedTabs). + const pinnedTabs = Array.isArray(baseSnapshot?.pinnedTabs) ? deepClone(baseSnapshot.pinnedTabs) : []; + + // Resolved synced global settings. Seeded from the base snapshot's options (folded + // by a prior compaction) and overridden per-key as `option.set` events are applied + // in global order — last-writer-wins by ts (the apply order already encodes it). + const resolvedOptions = deepClone(baseSnapshot?.options || {}); + + // Portable container registry ({portableKey: {name, color, icon}}). Carried through + // unchanged: events reference containers only by their portable key, and the registry + // is identity-derived (the same key always maps to the same {name,color,icon}), so it + // needs no merge logic in the pure engine. The transport (delta-sync.js) folds THIS + // device's local container defs into the result before writing/applying. + const containers = deepClone(baseSnapshot?.containers || {}); + + // The FROZEN base watermark — what is already baked into the base snapshot. Dedup + // compares against this only; it must not drift as we fold, otherwise a device whose + // ts order disagrees with its seq order (a later-seq event with an earlier ts) would + // wrongly skip a still-pending lower-seq event. `seq` is per-device authoritative for + // identity/dedup; `ts` only chooses the global apply order. + const baseWatermark = baseSnapshot?.watermark || {}; + + // The RETURNED watermark — max seq actually applied per device. Seeded from the base + // so a device that contributed nothing this round keeps its folded high-water mark. + const watermark = {...baseWatermark}; + + const ordered = buildOrderedEvents(deltaLogs); + + for (const {deviceId, event} of ordered) { + const folded = baseWatermark[deviceId] ?? 0; + + // watermark dedup: skip events already baked into the base (rule 4) + if (event.seq != null && event.seq <= folded) { + continue; + } + + switch (event.op) { + case OPS.TAB_ADD: + case OPS.TAB_MODIFY: + applyTabUpsert(groups, event); + break; + case OPS.TAB_MOVE: + applyTabMove(groups, event); + break; + case OPS.TAB_REMOVE: + applyTabRemove(groups, event); + break; + case OPS.GROUP_ADD: + case OPS.GROUP_MODIFY: + applyGroupUpsert(groups, event); + break; + case OPS.GROUP_MOVE: + applyGroupMove(groups, event); + break; + case OPS.GROUP_REMOVE: + applyGroupRemove(groups, event); + break; + case OPS.PINNED_ADD: + case OPS.PINNED_MODIFY: + applyPinnedUpsert(pinnedTabs, event); + break; + case OPS.PINNED_MOVE: + applyPinnedMove(pinnedTabs, event); + break; + case OPS.PINNED_REMOVE: + applyPinnedRemove(pinnedTabs, event); + break; + case OPS.OPTION_SET: + // per-key last-writer-wins: events arrive in global ts order, so the + // last write for a key (by ts, tie-break deviceId/seq) overrides. + if (event.key != null) { + resolvedOptions[event.key] = deepClone(event.value); + } + break; + default: + // unknown op: skip but still advance the watermark below so a future + // schema addition replayed by an old engine never re-folds it. + break; + } + + // advance watermark to the max applied seq for this device + if (event.seq != null && (watermark[deviceId] == null || event.seq > watermark[deviceId])) { + watermark[deviceId] = event.seq; + } + } + + // The resolved `group.tabs` ARRAY ORDER is the single authoritative tab order. Stamp + // each tab's `index` to its array position so the persisted field can never diverge + // from the array (a stale browser-absolute or group-relative `index` carried in from + // a base snapshot / event record would otherwise be misread downstream). Consumers + // that need an order use the array position; this just keeps the field honest. + for (const group of groups) { + group.tabs.forEach((tab, position) => { + tab.index = position; + }); + } + + // Same contract for the global pinned strip: the resolved `pinnedTabs` ARRAY ORDER is + // authoritative, so stamp each pinned tab's `index` to its array position (pinned tabs + // are first in the window, so the pinned-relative position is also their browser index). + pinnedTabs.forEach((tab, position) => { + tab.index = position; + }); + + return { + snapshot: {groups, pinnedTabs, options: resolvedOptions, containers, watermark}, + watermark, + }; +} diff --git a/addon/src/js/sync/delta/seed.js b/addon/src/js/sync/delta/seed.js new file mode 100644 index 00000000..24682e13 --- /dev/null +++ b/addon/src/js/sync/delta/seed.js @@ -0,0 +1,70 @@ + +/** + * Backward-compat seed for hybrid snapshot + delta sync (Phase P3a). + * + * Existing users sync via the OLD single-file `STG-backup.json`, a full-state + * backup ({ groups, ...options, containers }). The delta era needs an initial + * `STG-snapshot.json` of the shape the replay engine consumes + * (`{ groups, watermark }`). This helper produces that initial snapshot from an + * old backup WITHOUT discarding it: the groups/tabs already exist, so we just wrap + * them and add an empty watermark (nothing has been folded yet). + * + * ## How P3b will use it (documented, not wired here) + * When the transport pulls the gist: if `STG-snapshot.json` is ABSENT but + * `STG-backup.json` is present (a pre-delta user's first delta sync), read the old + * backup, run {@link seedSnapshotFromLegacyBackup} on it, and treat the result as + * `pulledSnapshot`. The old file is left in place (the classic `cloud.js` sync flow + * still reads/writes it until fully migrated). The first delta-era write then + * PATCHes `STG-snapshot.json`; replay on top of an empty watermark replays every + * pulled delta event, so no work is lost. + * + * ## Purity + * PURE: literals only, deep-clones its input, never mutates it, no `browser.*` and + * no `constants.js` import — so it runs/tests under plain `node`. + * + * @module sync/delta/seed + */ + +/** + * Deep clone of a plain JSON-ish value (mirrors replay.js' deepClone to stay + * import-free / obviously pure). + * @template T + * @param {T} value + * @returns {T} + */ +function deepClone(value) { + if (value === null || typeof value !== 'object') { + return value; + } + if (Array.isArray(value)) { + return value.map(deepClone); + } + const out = {}; + for (const key of Object.keys(value)) { + out[key] = deepClone(value[key]); + } + return out; +} + +/** + * Wrap an old single-file backup into an initial delta-era snapshot. + * + * `groups` (with their tabs) AND the global `pinnedTabs` carry over into the + * replay-engine snapshot shape; the legacy backup already stores pinned tabs in + * `data.pinnedTabs` (see `background.js` createBackup), so they map straight across. + * The watermark starts empty (`{}`) so every subsequent delta event is replayed + * (rule 4 dedup has nothing folded yet). The input is never mutated. + * + * @param {object} legacyBackup - parsed `STG-backup.json` ({ groups, pinnedTabs?, ...rest }). + * @returns {{groups: object[], pinnedTabs: object[], watermark: object}} the seed snapshot. + */ +export function seedSnapshotFromLegacyBackup(legacyBackup) { + const groups = Array.isArray(legacyBackup?.groups) ? deepClone(legacyBackup.groups) : []; + const pinnedTabs = Array.isArray(legacyBackup?.pinnedTabs) ? deepClone(legacyBackup.pinnedTabs) : []; + + return { + groups, + pinnedTabs, + watermark: {}, + }; +} diff --git a/addon/src/js/sync/delta/tab-sleep.js b/addon/src/js/sync/delta/tab-sleep.js new file mode 100644 index 00000000..d5d7ce26 --- /dev/null +++ b/addon/src/js/sync/delta/tab-sleep.js @@ -0,0 +1,75 @@ + +/** + * PURE decision for whether a sync-created tab should be created ASLEEP (discarded) or + * LOADED, given the user's LOCAL-ONLY sync options. + * + * Two independent axes, because Firefox treats pinned tabs differently: + * + * GROUP (non-pinned) tabs — "sleep by default, optionally wake some": + * - `syncSleepNewTabs` (default true): when FALSE nothing sleeps (legacy: every + * group tab is created loaded). When TRUE group tabs default to asleep. + * - `syncActivatePreviouslyActiveTabs` (default false): wake a tab that was loaded + * (not discarded) on the SOURCE machine — read from the record's additive `loaded` + * field (absent ⇒ treated as asleep, the safe default for old records). + * + * PINNED tabs — "load by default, optionally sleep": + * - `syncSleepPinnedTabs` (default false): pinned tabs LOAD by default, because + * Firefox forbids creating a discarded pinned tab outright (`tabs.create` rejects + * `{discarded:true, pinned:true}` with "Pinned tabs cannot be created and + * discarded."). STG works around it via create-then-discard in `Tabs.create`, but + * that path is opt-in: pinned tabs only sleep when this flag is ON. + * + * This is the ONLY place the option semantics live, so the grouped-create and pinned-create + * apply paths agree exactly. It is PURE (no `browser.*`, no constants import) so the node + * tests can import it directly. + * + * NOTE: this decides the `discarded` HINT passed to `Tabs.create`. STG's own create path + * still refuses to discard a tab that has no restorable URL or that it is making active + * (about:/long-url/foreground tabs always load) — see `tabs.js`. So returning + * `discarded: true` here is "sleep if possible", never a guarantee. + * + * @module sync/delta/tab-sleep + */ + +/** + * Should a sync-created tab be created discarded (asleep)? + * + * @param {object} tabRecord - the synced tab record being created. Only `record.loaded` + * (additive, true when the tab was loaded on the source) is consulted (group tabs only). + * @param {boolean} isPinned - true for the global-pinned create path, false for group tabs. + * @param {object} options - the resolved option bag (LOCAL values). Reads + * `syncSleepNewTabs`, `syncSleepPinnedTabs`, `syncActivatePreviouslyActiveTabs`. + * @returns {boolean} true ⇒ create asleep (discarded); false ⇒ create loaded. + */ +export function shouldSleepSyncedTab(tabRecord, isPinned, options = {}) { + if (isPinned) { + // Pinned tabs LOAD by default (Firefox can't create them discarded; STG only + // sleeps them via create-then-discard when the user explicitly opts in). + return options.syncSleepPinnedTabs === true; + } + + // group/non-pinned tabs: sleep-by-default OFF ⇒ legacy behavior, nothing sleeps. + if (!options.syncSleepNewTabs) { + return false; + } + + // exception: user wants tabs that were active/loaded on the source machine + // loaded here too. `loaded` is additive + true-only; absent ⇒ asleep. + if (options.syncActivatePreviouslyActiveTabs && tabRecord && tabRecord.loaded === true) { + return false; + } + + // default: asleep. + return true; +} + +/** + * Option keys this decision reads. Exported so the apply path can fetch exactly these + * from storage in one call without drifting from the decision logic. + * @readonly + */ +export const SLEEP_OPTION_KEYS = Object.freeze([ + 'syncSleepNewTabs', + 'syncSleepPinnedTabs', + 'syncActivatePreviouslyActiveTabs', +]); diff --git a/addon/src/js/sync/delta/url-sync.js b/addon/src/js/sync/delta/url-sync.js new file mode 100644 index 00000000..6677812e --- /dev/null +++ b/addon/src/js/sync/delta/url-sync.js @@ -0,0 +1,199 @@ + +/** + * Pure URL classifiers shared by the delta capture + transport layers so they all + * agree on WHICH tab URLs roam through delta sync, and how STG's "unsupported URL" + * stub page maps back to the original URL it embeds. + * + * ## Why a separate predicate from `Utils.isUrlAllowToCreate` + * `isUrlAllowToCreate` governs REAL tab creation: it deliberately rejects privileged + * `about:` URLs (about:debugging, about:config, …) so that `Tabs.create` substitutes + * the moz-extension "unsupported URL" stub page instead of failing. We must NOT broaden + * it, or the stub substitution stops triggering. + * + * Sync, however, SHOULD carry those privileged `about:` URLs: the receiving machine + * renders them as the stub (showing the original URL text) rather than silently dropping + * the tab. So {@link isUrlSyncable} is a WIDER allow-list than `isUrlAllowToCreate`: it + * admits non-trivial `about:` URLs on top of everything `isUrlAllowToCreate` admits, but + * still rejects the trivial / default new-tab states (about:blank, about:newtab, + * about:home, about:privatebrowsing) which are pure noise. + * + * ## Feedback-loop guard ({@link unwrapStubUrl}) + * After a receiving machine renders a synced `about:debugging` tab as the stub, that + * tab's LIVE url becomes `moz-extension:///help/stg-unsupported-url.html?url=about:debugging` + * — which IS allowed by `isUrlAllowToCreate`, so capture would otherwise record it as a + * competing tab record that diverges from the original `about:` identity. {@link unwrapStubUrl} + * decodes the stub back to the embedded original `about:` URL so the captured record keeps + * the original identity (no moz-extension divergence loop). It is the inverse of + * `tabs.js` `createUnsupportedUrlPage` (which stores the original in the `?url=` param). + * + * ## Purity + * No `browser.*` and no `constants.js` import (which is browser-dependent), so the pure + * unit tests can import this module directly under node. The stub page is matched by its + * stable path suffix (`/help/stg-unsupported-url.html`) rather than the per-install + * moz-extension UUID, which keeps the match pure and install-independent. + * + * @module sync/delta/url-sync + */ + +/** + * Trivial / empty `about:` URLs that represent a default new-tab / blank state. These are + * noise (every fresh tab is one of these) and must NOT sync, even though they are `about:` + * URLs. `about:blank` additionally passes `Utils.isUrlAllowToCreate`, but it is still not + * worth roaming. + * @readonly + */ +export const NON_SYNCABLE_ABOUT_URLS = Object.freeze(new Set([ + 'about:blank', + 'about:newtab', + 'about:home', + 'about:privatebrowsing', +])); + +/** Path suffix of STG's "unsupported URL" stub page (see `tabs.js` createUnsupportedUrlPage). */ +const STUB_PAGE_PATH_SUFFIX = '/help/stg-unsupported-url.html'; + +/** + * Does this tab URL roam through delta sync? A WIDER allow-list than + * `Utils.isUrlAllowToCreate`: everything that admits, PLUS non-trivial `about:` URLs + * (about:debugging, about:config, about:addons, about:preferences, …) which the receiving + * machine renders as the stub page. Excludes the trivial new-tab/blank `about:` states + * ({@link NON_SYNCABLE_ABOUT_URLS}). + * + * Pure string predicate (no `browser.*`). + * + * @param {string} url + * @returns {boolean} + */ +export function isUrlSyncable(url) { + if (typeof url !== 'string' || !url) { + return false; + } + + if (NON_SYNCABLE_ABOUT_URLS.has(url)) { + return false; + } + + // non-trivial about: URL (about:debugging, about:config, …): syncable. The receiving + // machine can't create it directly (isUrlAllowToCreate rejects it) so apply renders + // the stub page — but the synced RECORD carries the real about: url for identity. + if (url.startsWith('about:')) { + return true; + } + + // everything the real-creation allow-list admits (http, moz-extension, view-source, + // about:blank). about:blank is already excluded above, so it never reaches here. + return /^((http|moz-extension|view-source)|about:blank)/.test(url); +} + +/** + * If `url` is STG's moz-extension "unsupported URL" stub page, return the ORIGINAL url it + * embeds (the `?url=` param); otherwise return `url` unchanged. Inverse of `tabs.js` + * `createUnsupportedUrlPage`. Used by the capture layer so a synced `about:` tab whose + * LIVE url is the stub is recorded under its original `about:` identity rather than the + * moz-extension url (which would diverge from the originating machine). + * + * Pure (matches the stub by its stable path suffix, not the per-install UUID). Never + * throws — a malformed url falls through to the original. + * + * @param {string} url + * @returns {string} + */ +export function unwrapStubUrl(url) { + if (typeof url !== 'string' || !url.startsWith('moz-extension://')) { + return url; + } + + try { + const parsed = new URL(url); + if (parsed.pathname.endsWith(STUB_PAGE_PATH_SUFFIX)) { + const original = parsed.searchParams.get('url'); + if (original) { + return original; + } + } + } catch { + // malformed url — fall through to original + } + + return url; +} + +/** + * STUB-AWARE url match used by the sync-apply uid STAMPING step ({@link module:sync/delta/delta-sync}): + * does a freshly-created live tab's url correspond to a create source's url? + * + * A privileged `about:` source (about:debugging, about:memory, …) is created as STG's + * moz-extension "unsupported URL" stub, so the live tab's url is the stub while the source url is + * the original `about:…`. Without decoding, the stamp's url match FAILS for such tabs, leaving them + * UNSTAMPED → re-created every cycle (the about:/stub re-flood). Comparing the live url BOTH raw and + * stub-decoded fixes that. Pure (delegates to {@link unwrapStubUrl}). + * + * @param {string} liveUrl - the created tab's live url (may be the stub). + * @param {string} sourceUrl - the sync record's url. + * @returns {boolean} + */ +export function liveUrlMatchesSource(liveUrl, sourceUrl) { + return liveUrl === sourceUrl || unwrapStubUrl(liveUrl) === sourceUrl; +} + +/** + * NO-OP CONVERGENCE GUARD for the content-update apply ({@link module:sync/delta/delta-sync} + * `applyTabContentUpdate`): should the transport actually navigate a LOADED tab to `targetUrl`, + * or is it already there (so navigating would be a wasteful re-load — the "loads infinitely" + * spinner when the planner re-emits the same url every cycle)? + * + * Navigate ONLY when the tab's CURRENT url (stub-decoded so a stub-rendered about: tab compares by + * its embedded identity, which is what the target carries) differs from the target. Equal ⇒ no-op. + * Pure (delegates to {@link unwrapStubUrl}). + * + * @param {string} liveUrl - the live tab's current url. + * @param {string} targetUrl - the url the update wants the tab to be at. + * @returns {boolean} true ⇒ navigate; false ⇒ already converged, do nothing. + */ +export function shouldNavigateLiveTabUrl(liveUrl, targetUrl) { + return unwrapStubUrl(liveUrl) !== targetUrl; +} + +/** + * Maximum length of a `favIconUrl` we are willing to carry in a synced record. We KEEP + * favicons — including inline `data:` URIs — so every synced/sleeping tab shows an icon; + * a normal 16–32px PNG favicon is ~1–4 KB. The cap exists only to drop a PATHOLOGICALLY + * large favicon (a stray multi-hundred-KB data: blob) that would bloat the snapshot/log + * out of proportion. ~50 000 chars ≈ 50 KB comfortably clears real favicons. + * @readonly + */ +export const MAX_SYNCABLE_FAVICON_LENGTH = 50000; + +/** + * Sanitize a `favIconUrl` for storage in a delta event / snapshot record. + * + * Favicons are KEPT — including `data:` URIs — so every synced tab (and every sleeping + * tab restored on another device) shows its icon. The cost is BOUNDED structurally: a + * favicon is never its own event — it only ever rides along as one field inside a record + * written for a real reason (tab.add / tab.modify on a url/title change, or the compaction + * snapshot), so the resolved snapshot holds exactly one (latest) favicon per tab and a + * single favicon can never be duplicated across hundreds of thousands of events again + * (the original 5 GB bloat was a favicon-only event firing on every favicon tick). + * + * The ONLY thing dropped here (returns undefined) is a single PATHOLOGICALLY large favicon + * — over {@link MAX_SYNCABLE_FAVICON_LENGTH} (~50 KB). That is far above a real 16–32px + * favicon, so normal data: PNGs pass through unchanged. Favicons are cosmetic (they + * re-fetch from the live page on load), so dropping an oversized one never affects tab + * identity (url/title/group/pinned). + * + * Pure string function (no `browser.*`). Returns `undefined` when the favicon must be + * dropped, so callers can simply assign the result (an `undefined` field is omitted by + * structured clone / JSON). + * + * @param {string|undefined} favIconUrl + * @returns {string|undefined} the favicon url to store, or undefined to drop it. + */ +export function sanitizeFavIconUrl(favIconUrl) { + if (typeof favIconUrl !== 'string' || !favIconUrl) { + return undefined; + } + if (favIconUrl.length > MAX_SYNCABLE_FAVICON_LENGTH) { + return undefined; + } + return favIconUrl; +} diff --git a/addon/src/js/sync/delta/user-priority-lock.js b/addon/src/js/sync/delta/user-priority-lock.js new file mode 100644 index 00000000..7b7c0acd --- /dev/null +++ b/addon/src/js/sync/delta/user-priority-lock.js @@ -0,0 +1,329 @@ +/** + * USER-priority concurrency lock for the group/tab store (Phase: sync-user-priority). + * + * ## The race this fixes + * Delta-sync's local APPLY (`delta-sync.js` `applyBrowserOps`/`applyPinnedOps`) does a + * `Groups.load(null,false)` → modify → `Groups.save(nextGroups)`. USER-initiated group + * mutations (`Groups.remove`/`add`/`move`/`sort`/...) do their OWN `load → splice → save`. + * `Groups.save` is a blind `Storage.set({groups})` with no merge, so a user deletion that + * interleaves with a concurrent sync apply is a lost update: the manual delete's `save` + * is overwritten by the sync's `save` (or vice-versa), the group survives in the cloud and + * resurrects, and the user's action silently fails. The old `inProgress` guard in + * delta-sync serializes sync-vs-sync ONLY, never sync-vs-user. + * + * ## Policy (exact) + * 1. USER actions have PRIORITY. While the user is mutating, sync must WAIT and must NOT + * interleave. + * 2. If the user STARTS an action while sync is about to apply, sync YIELDS: it DEFERS + * applying this cycle and reschedules soon (sync is periodic — deferring one cycle is + * cheap). The user NEVER waits on a long sync. + * 3. Sync stays FAST: the lock is held ONLY around the short local apply critical section, + * NEVER around network pull/push. + * 4. Apply and user mutations are mutually exclusive (no interleaved load/modify/save). + * + * ## Mechanism — why a promise-chain mutex (not `navigator.locks`) + * A tiny single-tail promise-chain mutex: every critical section is appended to a shared + * `tail` Promise and runs only after the previous one settles. This is chosen over the + * WebExtensions `navigator.locks` API because: + * - the DEFER-on-user-active semantics (sync must give up rather than queue behind the + * user) need a synchronously-readable "user active" signal + a non-blocking + * try-acquire, which `navigator.locks` only models via `ifAvailable` + an extra + * promise dance — strictly more moving parts for no gain here; + * - it is PURE (no `browser.*`, no runtime feature-detection in the MV2 background page) + * so it unit-tests directly as a `.mjs` with the rest of the delta pipeline; + * - the critical sections are all in ONE persistent background page (single JS realm), + * so the cross-context guarantees `navigator.locks` adds are unnecessary. + * + * The mutex NEVER lets one failed critical section wedge the lock forever — it always + * releases (in `finally`) and swallows the prior section's rejection when waiting its turn, + * so callers still get their own result/throw via the returned promise. + * + * @module sync/delta/user-priority-lock + */ + +// Single shared promise tail. Each runExclusive() appends to it; the next waiter starts +// only after the previous critical section settles. Resolved-to-start. +let tail = Promise.resolve(); + +// "User is mutating" signal. Set for the duration of a user mutation's critical section +// PLUS a short trailing window (covers a burst of rapid edits — a user dragging several +// groups in a row shouldn't let a sync slip in between). Synchronously readable so the +// sync side can decide to DEFER without awaiting. +let userActiveCount = 0; // # of in-flight user critical sections (re-entrant safe) +let trailingTimer = null; // trailing-window timer handle +let trailingUntil = 0; // epoch ms until which we still count as user-active + +// Default trailing window after a user mutation's critical section ends. A few hundred ms +// is enough to bridge a rapid burst of edits without meaningfully starving sync (which is +// periodic). Exposed/overridable for tests. +export const DEFAULT_TRAILING_MS = 400; + +// Default safety timeout for the SYNC side acquiring the lock. Sync should never wait +// forever on a user critical section (which itself is short and always releases): if it +// can't get in within this window it gives up and DEFERS this cycle. Tunable per call. +export const DEFAULT_SYNC_ACQUIRE_TIMEOUT_MS = 5_000; + +// Default COMPLETION watchdog for the sync apply critical section. The acquisition +// timeout above governs only getting INTO the lock; once the apply `fn` starts it runs +// to completion holding the lock. If any await inside the apply never settles, the mutex +// would never release and EVERY future user mutation (which all queue on the same tail) +// would hang forever — the reported post-sync UI freeze. This watchdog bounds how long +// the apply may HOLD the lock: when it trips we stop waiting on the apply, let the +// critical section settle (so the normal `finally` release hands the lock to the next +// waiter), and let the in-flight apply keep running DETACHED in the background. Generous +// — well above any legitimate apply (a real apply of dozens of tabs is a few seconds per +// the debug logs); it exists only to self-recover from a never-settling await. +export const DEFAULT_SYNC_APPLY_WATCHDOG_MS = 60_000; + +/** + * Is the user currently mutating (in a critical section OR inside the trailing window)? + * Synchronous so the sync side can branch on it without awaiting. + * @returns {boolean} + */ +export function isUserActive() { + return userActiveCount > 0 || Date.now() < trailingUntil; +} + +/** + * Run `fn` as an exclusive critical section: it starts only after every previously-queued + * section settles, and the next waiter starts only after `fn` settles. The lock is held + * for EXACTLY the duration of `fn` — keep `fn` short and never await network/user-prompts + * inside it. Always releases (even if `fn` throws); the caller still sees `fn`'s + * result/rejection. + * @template T + * @param {() => (T | Promise)} fn + * @returns {Promise} + */ +export function runExclusive(fn) { + // chain a barrier the previous tail must clear; `release` lets the NEXT waiter proceed. + const {promise: gate, resolve: release} = withResolvers(); + const previous = tail; + tail = gate; // everyone after us waits on our gate + + return (async () => { + try { + await previous; // wait our turn + } catch { + // a prior section's failure must not block us; its own caller already saw it. + } + try { + return await fn(); + } finally { + release(); // hand the lock to the next waiter no matter what + } + })(); +} + +/** + * Mark the start of a USER mutation. Increments the active count and (re)arms the trailing + * window. MUST be paired with {@link endUserMutation} in a `finally`. Prefer + * {@link runUserMutation}, which pairs them for you. + * @param {number} [trailingMs=DEFAULT_TRAILING_MS] + */ +export function beginUserMutation(trailingMs = DEFAULT_TRAILING_MS) { + userActiveCount++; + // while a section is in-flight the count alone keeps us active; pre-extend the trailing + // mark so an immediately-following endUserMutation still leaves a window even if the + // timer below hasn't been (re)armed yet. + trailingUntil = Math.max(trailingUntil, Date.now() + trailingMs); +} + +/** + * Mark the end of a USER mutation. Decrements the active count and arms the trailing + * window so a rapid follow-up edit still counts as user-active (sync keeps deferring). + * @param {number} [trailingMs=DEFAULT_TRAILING_MS] + */ +export function endUserMutation(trailingMs = DEFAULT_TRAILING_MS) { + if (userActiveCount > 0) { + userActiveCount--; + } + if (userActiveCount === 0) { + trailingUntil = Date.now() + trailingMs; + if (trailingTimer) { + clearTimeout(trailingTimer); + } + // a no-op timer that just lets the window lapse; isUserActive() reads trailingUntil + // directly, so this only deterministically clears the handle (and gives tests a hook). + // Guard for environments without setTimeout (none in the bg page). + if (typeof setTimeout === 'function') { + trailingTimer = setTimeout(() => { + trailingTimer = null; + }, trailingMs); + } + } +} + +/** + * Run a USER mutation's critical section with PRIORITY: marks the user active for the + * duration + trailing window AND takes the mutex so it can't interleave with a sync apply. + * The user never DEFERS — they always get the lock (sync's hold is short and bounded). + * @template T + * @param {() => (T | Promise)} fn + * @param {number} [trailingMs=DEFAULT_TRAILING_MS] + * @returns {Promise} + */ +export function runUserMutation(fn, trailingMs = DEFAULT_TRAILING_MS) { + beginUserMutation(trailingMs); + return runExclusive(fn).finally(() => endUserMutation(trailingMs)); +} + +/** + * Run the SYNC apply critical section, YIELDING to the user. + * + * Behavior: + * - If the user is active RIGHT NOW (in a critical section or trailing window), DEFER + * immediately: do NOT run `fn`, return `{deferred: true}` so the caller reschedules. + * - Otherwise take the mutex with a safety timeout. If acquisition can't complete within + * `timeoutMs` (e.g. a user mutation slipped in and is holding it), DEFER too — sync + * never waits forever. + * - Once `fn` starts (we hold the lock) it is the short, bounded apply — but a COMPLETION + * WATCHDOG (`watchdogMs`) bounds how long it may HOLD the lock. If a never-settling await + * inside the apply keeps `fn` from resolving, the watchdog fires: we STOP waiting on the + * apply, the critical section settles (so the mutex's normal `finally` release frees the + * lock for the next waiter, e.g. a queued user mutation), and the in-flight apply is left + * to keep running DETACHED. We do NOT abort/rollback the in-flight browser ops; we only + * stop them from holding the lock hostage so user input can never wedge permanently. + * `onWatchdog()` (if given) is invoked so the caller can log WARN diagnostics. + * + * RELEASE-RACE / IDEMPOTENCY: the lock release is owned SOLELY by the mutex's `finally` in + * {@link runExclusive}, which `release()`s exactly once when the critical-section function + * settles. We never force-release here. When the watchdog fires near the deadline AND the + * apply finishes ~simultaneously, only ONE thing settles the critical section: a `settled` + * guard flag ensures the section's function returns exactly once (either the apply's value or + * the watchdog's marker), so `release()` fires exactly once for THIS section. The detached + * apply continuing afterward touches no lock state — it has no `release` handle — so it can + * neither double-release nor release a newer section. + * + * @template T + * @param {() => (T | Promise)} fn the apply critical section (short; no network inside). + * @param {object} [opts] + * @param {number} [opts.timeoutMs=DEFAULT_SYNC_ACQUIRE_TIMEOUT_MS] acquisition safety timeout. + * @param {number} [opts.watchdogMs=DEFAULT_SYNC_APPLY_WATCHDOG_MS] completion watchdog (held-lock bound). + * @param {(info: {elapsedMs: number}) => void} [opts.onWatchdog] called once if the watchdog trips. + * @returns {Promise<{deferred: boolean, result?: T, watchdog?: boolean}>} + */ +export async function runSyncApply(fn, { + timeoutMs = DEFAULT_SYNC_ACQUIRE_TIMEOUT_MS, + watchdogMs = DEFAULT_SYNC_APPLY_WATCHDOG_MS, + onWatchdog, +} = {}) { + // 1. fast pre-check: user active ⇒ defer without touching the lock at all. + if (isUserActive()) { + return {deferred: true}; + } + + // 2. race lock acquisition against a safety timeout. If the timeout wins we DEFER; the + // queued critical section then no-ops (it re-checks `timedOut`) so it never runs the + // apply after we've already given up, and the chain still advances cleanly. + let acquired = false; + let timedOut = false; + + const {promise: timeoutPromise, resolve: fireTimeout} = withResolvers(); + const timer = (typeof setTimeout === 'function') + ? setTimeout(() => { + timedOut = true; + fireTimeout({deferred: true}); + }, timeoutMs) + : null; + + const exclusive = runExclusive(async () => { + if (timedOut) { + return {deferred: true}; // timeout already won; do nothing under lock. + } + // re-check the user signal now that we actually hold the lock: a user mutation could + // have started+finished while we waited in the chain. Belt-and-suspenders (the user + // path takes the lock too, so it can't be mid-mutation here), but cheap. + if (isUserActive()) { + return {deferred: true}; + } + acquired = true; + + // COMPLETION WATCHDOG: race the apply against a generous timeout. Whichever settles + // FIRST decides this critical section's return value (and thus when the mutex releases + // the lock for the next waiter). A `settled` guard makes the section resolve exactly + // once even if the apply finishes the same tick the watchdog fires — so `release()` + // (owned by runExclusive's finally) is never raced into a double-release. + const startedAt = Date.now(); + let settled = false; + const {promise: watchdogPromise, resolve: fireWatchdog} = withResolvers(); + const watchdogTimer = (typeof setTimeout === 'function') + ? setTimeout(() => { + if (settled) { + return; // apply already won the race; leave the lock to the normal path. + } + const elapsedMs = Date.now() - startedAt; + if (typeof onWatchdog === 'function') { + try { + onWatchdog({elapsedMs}); + } catch { + // diagnostics must never break the safety release. + } + } + // settling THIS section frees the lock via runExclusive's finally; the + // in-flight apply keeps running detached (we intentionally do not await it). + fireWatchdog({deferred: false, watchdog: true}); + }, watchdogMs) + : null; + + // Run the apply and mark `settled` the instant it settles (resolve OR reject) so a + // watchdog firing on the same tick becomes a no-op (it checks `settled`), and clear + // the watchdog timer. A rejection still settles the section (and releases the lock) + // with the error — the watchdog only matters when the apply NEVER settles. + const guardedApply = Promise.resolve().then(fn) + .then(result => ({deferred: false, result})) + .finally(() => { + settled = true; + if (watchdogTimer) { + clearTimeout(watchdogTimer); + } + }); + + // If the watchdog wins the race below, this section settles via `watchdogPromise` and + // `guardedApply` is left running DETACHED. Swallow its eventual rejection so a later + // failure of the abandoned apply can't surface as an unhandledRejection (the original + // caller has already moved on; a detached apply is best-effort). This catch does NOT + // affect the race: `Promise.race` reads `guardedApply`'s own settlement, not this fork. + guardedApply.catch(() => {}); + + return Promise.race([guardedApply, watchdogPromise]); + }); + + try { + const winner = await Promise.race([exclusive, timeoutPromise]); + // If our section actually ran the apply, return its outcome; otherwise DEFER. + return acquired ? winner : {deferred: true}; + } finally { + if (timer) { + clearTimeout(timer); + } + } +} + +/** + * Cross-realm shim for Promise.withResolvers (older Firefox in the test runner may lack it). + * Tiny and local so the module has no external dependency. + */ +function withResolvers() { + if (typeof Promise.withResolvers === 'function') { + return Promise.withResolvers(); + } + let resolve, reject; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return {promise, resolve, reject}; +} + +/** + * TEST-ONLY: reset all module state to pristine. Not used by production code. + */ +export function __resetForTests() { + tail = Promise.resolve(); + userActiveCount = 0; + trailingUntil = 0; + if (trailingTimer) { + clearTimeout(trailingTimer); + trailingTimer = null; + } +} diff --git a/addon/src/js/utils.js b/addon/src/js/utils.js index 59be66e3..248e2a6a 100644 --- a/addon/src/js/utils.js +++ b/addon/src/js/utils.js @@ -10,6 +10,10 @@ export function unixNow() { return Math.floor(Date.now() / 1000); } +export function unixNowMs() { + return Date.now(); +} + export function encodeToBytes(str) { return encoder.encode(str); } diff --git a/addon/src/options/Options.vue b/addon/src/options/Options.vue index 459596ca..b638445a 100644 --- a/addon/src/options/Options.vue +++ b/addon/src/options/Options.vue @@ -1315,6 +1315,7 @@ export default {
diff --git a/addon/src/options/github-gist.vue b/addon/src/options/github-gist.vue index 07fce127..805fe86c 100644 --- a/addon/src/options/github-gist.vue +++ b/addon/src/options/github-gist.vue @@ -29,6 +29,19 @@ export default { return { confirmRestoreBackupItem: null, + confirmResetSyncState: false, + + // LOCAL-ONLY safety toggle (default ON): take a backup before applying sync. + syncBackupBeforeApply: Constants.DEFAULT_OPTIONS.syncBackupBeforeApply, + + // LOCAL-ONLY sleep toggles for sync-created tabs (see tab-sleep.js). + // syncSleepNewTabs (default ON) creates synced GROUP tabs asleep, with + // syncActivatePreviouslyActiveTabs (default OFF) as its sub-exception. + // syncSleepPinnedTabs (default OFF) is a separate axis: pinned tabs load by + // default and only sleep when it is on. + syncSleepNewTabs: Constants.DEFAULT_OPTIONS.syncSleepNewTabs, + syncSleepPinnedTabs: Constants.DEFAULT_OPTIONS.syncSleepPinnedTabs, + syncActivatePreviouslyActiveTabs: Constants.DEFAULT_OPTIONS.syncActivatePreviouslyActiveTabs, sync: { title: 'syncOptionLocatedFFSync', @@ -107,6 +120,27 @@ export default { }); }); + // load + persist the LOCAL-ONLY pre-apply backup toggle (a plain global STG + // option, stored via Storage like any other option). + Storage.get('syncBackupBeforeApply').then(({syncBackupBeforeApply}) => { + this.syncBackupBeforeApply = syncBackupBeforeApply; + this.$watch('syncBackupBeforeApply', value => { + Storage.set({syncBackupBeforeApply: value}); + }); + }); + + // load + persist the LOCAL-ONLY sleep/activate toggles for sync-created tabs. + // Each is a plain global STG option, stored via Storage like any other. + Storage.get(['syncSleepNewTabs', 'syncSleepPinnedTabs', 'syncActivatePreviouslyActiveTabs']) + .then(values => { + ['syncSleepNewTabs', 'syncSleepPinnedTabs', 'syncActivatePreviouslyActiveTabs'].forEach(key => { + this[key] = values[key]; + this.$watch(key, value => { + Storage.set({[key]: value}); + }); + }); + }); + this.$on('sync-finish', () => this.area.load(false)); }, methods: { @@ -236,6 +270,14 @@ export default { async restoreBackup({version}) { await this.syncCloud(Cloud.TRUST_CLOUD, version); }, + + async resetSyncState() { + // LOCAL-only recovery: clears this device's delta-sync state in the + // background; the cloud gist is left untouched. Reload the area afterwards + // so the displayed gist/last-update info reflects the reset state. + await this.sendMessage('reset-cloud-sync-state'); + await this.area.load(false); + }, }, }; @@ -305,6 +347,34 @@ export default {
+
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
+
+ +
@@ -493,6 +572,25 @@ export default { + +
+ +
From 25afb80be4ec89d9c0b0f3303209c25466845f21 Mon Sep 17 00:00:00 2001 From: Danyil Date: Sun, 28 Jun 2026 00:52:12 +0300 Subject: [PATCH 002/108] feat(groups): group-scoped pinned tabs Pin a tab within an active group (browser-pinned only while its group is shown). Adds the groupPinned session flag and ordering, native tab-menu and context-menu toggles, group-pin-aware moves (tab-move-split), and supporting cache/menu/migration changes. Includes the thumbtack asset and locale strings. --- addon/src/_locales/en/messages.json | 32 ++ addon/src/components/context-menu-tab.vue | 8 + addon/src/icons/thumbtack.svg | 3 + addon/src/js/cache.js | 154 ++++++++- addon/src/js/groups.js | 387 +++++++++++++++++++++- addon/src/js/menus-bookmark.js | 16 + addon/src/js/menus-link.js | 16 + addon/src/js/menus-tab.js | 88 ++++- addon/src/js/menus.js | 16 +- addon/src/js/migration.js | 32 ++ addon/src/js/mixins/tab-groups.mixin.js | 103 ++++-- addon/src/js/tab-move-split.js | 46 +++ addon/src/js/tabs.js | 307 ++++++++++++++++- 13 files changed, 1146 insertions(+), 62 deletions(-) create mode 100644 addon/src/icons/thumbtack.svg create mode 100644 addon/src/js/tab-move-split.js diff --git a/addon/src/_locales/en/messages.json b/addon/src/_locales/en/messages.json index c0f8baf2..8cc564d4 100644 --- a/addon/src/_locales/en/messages.json +++ b/addon/src/_locales/en/messages.json @@ -759,6 +759,14 @@ "message": "Discard tab", "description": "Discard tab" }, + "pinTabInGroupTitle": { + "message": "Pin to group", + "description": "Pin this tab within its group (pinned only while the group is active)" + }, + "unpinTabInGroupTitle": { + "message": "Unpin from group", + "description": "Remove the group-scoped pin from this tab" + }, "folderNameTitle": { "message": "Folder name", "description": "Folder name title" @@ -1393,10 +1401,34 @@ "message": "The data in the cloud can be very different from the current state.", "description": "The data in the cloud can be very different from the current state." }, + "resetSyncStateButton": { + "message": "Reset sync state", + "description": "Reset this device's local sync state (recovery action)" + }, + "resetSyncStateConfirm": { + "message": "This clears the local synchronization state for this device only (baseline and change log). The data in the cloud is NOT touched. The next synchronization will behave like a first sync: nothing is removed and your current tabs are re-uploaded.", + "description": "Confirmation text for the reset sync state action" + }, "syncEnableTitle": { "message": "Enable auto backup to the cloud", "description": "Enable auto backup to the cloud" }, + "syncBackupBeforeApply": { + "message": "Create a backup before applying sync changes (recommended)", + "description": "Toggle: take a local STG backup before delta-sync applies any change, so a bad sync can be rolled back" + }, + "syncSleepNewTabs": { + "message": "Create new synced tabs asleep (recommended)", + "description": "Toggle: tabs created by sync show title+favicon and only load when activated" + }, + "syncSleepPinnedTabs": { + "message": "Sleep synced pinned tabs (Firefox loads pinned tabs by default)", + "description": "Toggle: also create synced pinned tabs asleep via create-then-discard; off by default since pinned tabs normally load" + }, + "syncActivatePreviouslyActiveTabs": { + "message": "Activate tabs that were active on the other machine", + "description": "Sub-toggle: load a synced tab if it was active (loaded) on the source machine" + }, "syncSettingsLocation": { "message": "Sync settings location", "description": "Enable auto backup to the cloud" diff --git a/addon/src/components/context-menu-tab.vue b/addon/src/components/context-menu-tab.vue index 8b9fb5f2..a1a98072 100644 --- a/addon/src/components/context-menu-tab.vue +++ b/addon/src/components/context-menu-tab.vue @@ -93,6 +93,14 @@ export default { +
  • +
    + +
    + +
    • +
    + + +
    +
    + +
    + + + + +
    +
    + +
    +
    diff --git a/addon/src/options/backup-location-downloads.vue b/addon/src/options/backup-location-downloads.vue index 1f86a4a3..f2a2c453 100644 --- a/addon/src/options/backup-location-downloads.vue +++ b/addon/src/options/backup-location-downloads.vue @@ -13,6 +13,12 @@ import optionsMixin from '/js/mixins/options.mixin.js'; export default { mixins: [optionsMixin], + props: { + pathKey: { + type: String, + default: 'autoBackupFilePath', + }, + }, data() { this.FILE_PATH_VARIABLES = Utils.getFilePathVariables(); @@ -27,7 +33,7 @@ export default { methods: { lang: Lang, addCustomWatchers() { - this.optionsWatch('autoBackupFilePath', async (value, oldValue) => { + this.optionsWatch(this.pathKey, async (value, oldValue) => { try { value = value.trim(); @@ -44,9 +50,9 @@ export default { }); }, insertVariableToFilePath(variable) { - this.options.autoBackupFilePath = Utils.insertVariable( - this.$refs.autoBackupFilePath, - this.options.autoBackupFilePath, + this.options[this.pathKey] = Utils.insertVariable( + this.$refs.filePathInput, + this.options[this.pathKey], variable ); }, @@ -72,8 +78,8 @@ export default {
    { + this.optionsWatch(this.pathKey, async (value, oldValue) => { try { value = value.trim(); @@ -125,9 +131,9 @@ export default { } }, insertVariableToFilePath(variable) { - this.options.autoBackupFilePath = Utils.insertVariable( - this.$refs.autoBackupFilePath, - this.options.autoBackupFilePath, + this.options[this.pathKey] = Utils.insertVariable( + this.$refs.filePathInput, + this.options[this.pathKey], variable ); }, @@ -211,7 +217,7 @@ export default {
    - +
    .json diff --git a/addon/src/options/github-gist-access.vue b/addon/src/options/github-gist-access.vue new file mode 100644 index 00000000..4ee5bbab --- /dev/null +++ b/addon/src/options/github-gist-access.vue @@ -0,0 +1,192 @@ + + + diff --git a/addon/src/options/github-gist-backup.vue b/addon/src/options/github-gist-backup.vue new file mode 100644 index 00000000..59483f1f --- /dev/null +++ b/addon/src/options/github-gist-backup.vue @@ -0,0 +1,215 @@ + + + + + + diff --git a/addon/src/options/github-gist-sync.vue b/addon/src/options/github-gist-sync.vue new file mode 100644 index 00000000..a7189a0e --- /dev/null +++ b/addon/src/options/github-gist-sync.vue @@ -0,0 +1,189 @@ + + + + + + diff --git a/addon/src/options/github-gist.vue b/addon/src/options/github-gist.vue deleted file mode 100644 index 805fe86c..00000000 --- a/addon/src/options/github-gist.vue +++ /dev/null @@ -1,627 +0,0 @@ - - - - - - From f9b75520ffd254a8e4d02eb2c03e47a2e724eb3c Mon Sep 17 00:00:00 2001 From: Danyil Date: Mon, 29 Jun 2026 01:30:55 +0300 Subject: [PATCH 012/108] i18n: messages for the 3 cloud blocks Add keys for the GitHub Gist access / cloud sync / cloud backup block titles, the storage-location selector label, the Create backup button, and the pre-apply sync backup settings (en authoritative; ru/uk translated). Remove the now-unused trust-dropdown keys (syncStartTrustLocal, syncStartTrustCloud, syncDataInCloudCanBeDifferent). --- addon/src/_locales/en/messages.json | 30 ++++++++++++++++++++--------- addon/src/_locales/ru/messages.json | 30 ++++++++++++++++++++--------- addon/src/_locales/uk/messages.json | 30 ++++++++++++++++++++--------- 3 files changed, 63 insertions(+), 27 deletions(-) diff --git a/addon/src/_locales/en/messages.json b/addon/src/_locales/en/messages.json index cec7dafb..904ceff1 100644 --- a/addon/src/_locales/en/messages.json +++ b/addon/src/_locales/en/messages.json @@ -1389,17 +1389,29 @@ "message": "Start synchronization", "description": "Start synchronization" }, - "syncStartTrustLocal": { - "message": "Start synchronization and trust local data", - "description": "Start synchronization and trust local data" + "githubGistAccessTitle": { + "message": "GitHub Gist access", + "description": "Title of the GitHub Gist access settings block (token, file, storage location)" }, - "syncStartTrustCloud": { - "message": "Start synchronization and trust cloud data", - "description": "Start synchronization and trust cloud data" + "githubGistAccessLocation": { + "message": "Store cloud settings in", + "description": "Label for the selector choosing where the roaming sync option bag is stored (Firefox Sync or local profile)" }, - "syncDataInCloudCanBeDifferent": { - "message": "The data in the cloud can be very different from the current state.", - "description": "The data in the cloud can be very different from the current state." + "githubGistSyncTitle": { + "message": "Cloud sync", + "description": "Title of the cloud sync settings block" + }, + "githubGistBackupTitle": { + "message": "Cloud backup", + "description": "Title of the cloud backup settings block" + }, + "createBackupButton": { + "message": "Create backup", + "description": "Button that pushes the current full state to the cloud as a new gist revision" + }, + "syncBackupSettingsTitle": { + "message": "Pre-apply backup before sync", + "description": "Label for the pre-apply sync backup location and file path settings" }, "resetSyncStateButton": { "message": "Reset sync state", diff --git a/addon/src/_locales/ru/messages.json b/addon/src/_locales/ru/messages.json index dd0d9459..379b1948 100644 --- a/addon/src/_locales/ru/messages.json +++ b/addon/src/_locales/ru/messages.json @@ -1369,17 +1369,29 @@ "message": "Синхронизировать", "description": "Start synchronization" }, - "syncStartTrustLocal": { - "message": "Синхронизировать и доверять локальным данным", - "description": "Start synchronization and trust local data" + "githubGistAccessTitle": { + "message": "Доступ к GitHub Gist", + "description": "Title of the GitHub Gist access settings block (token, file, storage location)" }, - "syncStartTrustCloud": { - "message": "Синхронизировать и доверьтесь облачным данным", - "description": "Start synchronization and trust cloud data" + "githubGistAccessLocation": { + "message": "Хранить настройки облака в", + "description": "Label for the selector choosing where the roaming sync option bag is stored (Firefox Sync or local profile)" }, - "syncDataInCloudCanBeDifferent": { - "message": "Данные в облаке могут сильно отличаться от текущего состояния.", - "description": "The data in the cloud can be very different from the current state." + "githubGistSyncTitle": { + "message": "Облачная синхронизация", + "description": "Title of the cloud sync settings block" + }, + "githubGistBackupTitle": { + "message": "Облачная резервная копия", + "description": "Title of the cloud backup settings block" + }, + "createBackupButton": { + "message": "Создать резервную копию", + "description": "Button that pushes the current full state to the cloud as a new gist revision" + }, + "syncBackupSettingsTitle": { + "message": "Резервная копия перед применением синхронизации", + "description": "Label for the pre-apply sync backup location and file path settings" }, "syncEnableTitle": { "message": "Включить синхронизацию с облаком", diff --git a/addon/src/_locales/uk/messages.json b/addon/src/_locales/uk/messages.json index b1ab98ab..73ddd032 100644 --- a/addon/src/_locales/uk/messages.json +++ b/addon/src/_locales/uk/messages.json @@ -1369,17 +1369,29 @@ "message": "Синхронізувати", "description": "Start synchronization" }, - "syncStartTrustLocal": { - "message": "Синхронізувати та довіряти локальним даним", - "description": "Start synchronization and trust local data" + "githubGistAccessTitle": { + "message": "Доступ до GitHub Gist", + "description": "Title of the GitHub Gist access settings block (token, file, storage location)" }, - "syncStartTrustCloud": { - "message": "Синхронізувати та довіряти хмарним даним", - "description": "Start synchronization and trust cloud data" + "githubGistAccessLocation": { + "message": "Зберігати налаштування хмари в", + "description": "Label for the selector choosing where the roaming sync option bag is stored (Firefox Sync or local profile)" }, - "syncDataInCloudCanBeDifferent": { - "message": "Дані в хмарі можуть сильно відрізнятися від поточного стану.", - "description": "The data in the cloud can be very different from the current state." + "githubGistSyncTitle": { + "message": "Хмарна синхронізація", + "description": "Title of the cloud sync settings block" + }, + "githubGistBackupTitle": { + "message": "Хмарна резервна копія", + "description": "Title of the cloud backup settings block" + }, + "createBackupButton": { + "message": "Створити резервну копію", + "description": "Button that pushes the current full state to the cloud as a new gist revision" + }, + "syncBackupSettingsTitle": { + "message": "Резервна копія перед застосуванням синхронізації", + "description": "Label for the pre-apply sync backup location and file path settings" }, "syncEnableTitle": { "message": "Увімкнути синхронізацію з хмарою", From 41483e9cbb020c05f55a1d75974a023b8b58be85 Mon Sep 17 00:00:00 2001 From: Danyil Date: Mon, 29 Jun 2026 02:01:45 +0300 Subject: [PATCH 013/108] feat(options): group cloud settings under one section; move pre-sync backup into Cloud sync --- addon/src/_locales/en/messages.json | 8 +- addon/src/_locales/ru/messages.json | 8 +- addon/src/_locales/uk/messages.json | 8 +- addon/src/options/Options.vue | 165 +++++++++++++------------ addon/src/options/github-gist-sync.vue | 13 +- 5 files changed, 99 insertions(+), 103 deletions(-) diff --git a/addon/src/_locales/en/messages.json b/addon/src/_locales/en/messages.json index 904ceff1..a6ff2c69 100644 --- a/addon/src/_locales/en/messages.json +++ b/addon/src/_locales/en/messages.json @@ -1389,6 +1389,10 @@ "message": "Start synchronization", "description": "Start synchronization" }, + "cloudSettingsTitle": { + "message": "Cloud", + "description": "Title of the section grouping the cloud access, sync and backup settings blocks" + }, "githubGistAccessTitle": { "message": "GitHub Gist access", "description": "Title of the GitHub Gist access settings block (token, file, storage location)" @@ -1409,10 +1413,6 @@ "message": "Create backup", "description": "Button that pushes the current full state to the cloud as a new gist revision" }, - "syncBackupSettingsTitle": { - "message": "Pre-apply backup before sync", - "description": "Label for the pre-apply sync backup location and file path settings" - }, "resetSyncStateButton": { "message": "Reset sync state", "description": "Reset this device's local sync state (recovery action)" diff --git a/addon/src/_locales/ru/messages.json b/addon/src/_locales/ru/messages.json index 379b1948..53afe5a3 100644 --- a/addon/src/_locales/ru/messages.json +++ b/addon/src/_locales/ru/messages.json @@ -1369,6 +1369,10 @@ "message": "Синхронизировать", "description": "Start synchronization" }, + "cloudSettingsTitle": { + "message": "Облако", + "description": "Title of the section grouping the cloud access, sync and backup settings blocks" + }, "githubGistAccessTitle": { "message": "Доступ к GitHub Gist", "description": "Title of the GitHub Gist access settings block (token, file, storage location)" @@ -1389,10 +1393,6 @@ "message": "Создать резервную копию", "description": "Button that pushes the current full state to the cloud as a new gist revision" }, - "syncBackupSettingsTitle": { - "message": "Резервная копия перед применением синхронизации", - "description": "Label for the pre-apply sync backup location and file path settings" - }, "syncEnableTitle": { "message": "Включить синхронизацию с облаком", "description": "Enable sync to the cloud" diff --git a/addon/src/_locales/uk/messages.json b/addon/src/_locales/uk/messages.json index 73ddd032..c4858967 100644 --- a/addon/src/_locales/uk/messages.json +++ b/addon/src/_locales/uk/messages.json @@ -1369,6 +1369,10 @@ "message": "Синхронізувати", "description": "Start synchronization" }, + "cloudSettingsTitle": { + "message": "Хмара", + "description": "Title of the section grouping the cloud access, sync and backup settings blocks" + }, "githubGistAccessTitle": { "message": "Доступ до GitHub Gist", "description": "Title of the GitHub Gist access settings block (token, file, storage location)" @@ -1389,10 +1393,6 @@ "message": "Створити резервну копію", "description": "Button that pushes the current full state to the cloud as a new gist revision" }, - "syncBackupSettingsTitle": { - "message": "Резервна копія перед застосуванням синхронізації", - "description": "Label for the pre-apply sync backup location and file path settings" - }, "syncEnableTitle": { "message": "Увімкнути синхронізацію з хмарою", "description": "Enable sync to the cloud" diff --git a/addon/src/options/Options.vue b/addon/src/options/Options.vue index 518cb3c7..d503c45d 100644 --- a/addon/src/options/Options.vue +++ b/addon/src/options/Options.vue @@ -1291,107 +1291,112 @@ export default {
    -
    - -
    - -
    +
    + -
    -
    - +
    +
    -