From df332f14072a2a8839ec99e30f43ad5150cedc17 Mon Sep 17 00:00:00 2001 From: Aaron Rono Date: Wed, 4 Nov 2026 17:52:55 +0300 Subject: [PATCH 1/2] feat: sync layouts, drawings, watchlists and settings across computers Phase 3 of sync: every renderer store now syncs through the encrypted engine. Layouts and their charts, chart, layout and symbol drawings, profiles, chart and drawing templates, drawing looks, imported themes, watchlists, favourite indicators, timeframes and drawing tools, custom colours, price alerts, Pine scripts, and the theme, replay, Trade Assistant, Risk Ticket and risk settings. Panels, docks, the layout, chart and watchlist in front, and one-click consent stay per computer. - Renderer adapters (src/lib/sync-sources.ts) over a pure workspace model (src/lib/sync-model.ts), served over IPC to main-side proxies (electron/sync-renderer.ts). Validation runs in the renderer; source validation may be async and gets the record id. - New kinds layoutCharts and layoutDrawings; ordered maps carry an order item so drawing z-order and watchlist order survive merges. - A computer joining existing synced content chooses Merge or Use synced data instead of pushing its default layout everywhere. - Closing the window pushes unsynced edits first, while the stores can still be read. --- CHANGELOG.md | 11 + electron/main.ts | 35 ++- electron/preload.ts | 13 +- electron/sync-engine-fixtures.ts | 2 +- electron/sync-engine.test.ts | 14 + electron/sync-engine.ts | 39 ++- electron/sync-renderer.test.ts | 64 ++++ electron/sync-renderer.ts | 91 ++++++ electron/sync-service.test.ts | 45 ++- electron/sync-service.ts | 31 +- packages/meridian-charts/src/index.ts | 1 + .../meridian-charts/src/ui/color-picker.ts | 16 + shared/sync.ts | 92 ++++-- src/components/chart/ChartWindow.tsx | 7 +- src/components/dialogs/RiskTicket.tsx | 2 +- src/components/sync/SyncDialog.tsx | 3 +- src/lib/pine/pine-scripts.ts | 14 + src/lib/risk-preferences.ts | 4 + src/lib/sync-model.test.ts | 146 +++++++++ src/lib/sync-model.ts | 295 ++++++++++++++++++ src/lib/sync-sources.ts | 290 +++++++++++++++++ src/lib/sync.ts | 35 ++- src/lib/watchlists.ts | 17 + src/lib/workspace.ts | 10 + 24 files changed, 1210 insertions(+), 67 deletions(-) create mode 100644 electron/sync-renderer.test.ts create mode 100644 electron/sync-renderer.ts create mode 100644 src/lib/sync-model.test.ts create mode 100644 src/lib/sync-model.ts create mode 100644 src/lib/sync-sources.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c314e8..b507657 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Sync Across Computers now syncs your content**: layouts and their charts (symbol, timeframe, + indicators, chart settings and themes), every drawing (on a chart, shared across a layout, or on a + symbol for all layouts), chart profiles and templates, drawing templates and last-used looks, + imported themes, watchlists, favourite indicators, timeframes and drawing tools, custom colours, + price alerts, Pine scripts, and the theme, replay, Trade Assistant, Risk Ticket and risk settings. + Panels, docks, which layout and chart are in front, and one-click trading consent stay with each + computer. A computer joining data that already syncs asks whether to merge its own layouts in or + use the synced ones. + ## [1.0.0] - 2026-11-04 ### Added diff --git a/electron/main.ts b/electron/main.ts index dcc0336..4402a9d 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -33,7 +33,8 @@ import { accountKey } from './journaling-store'; import { autoUpdater, DebUpdater } from 'electron-updater'; import { UpdateService } from './updater'; import type { UpdateMethod, UpdateState } from '../shared/updates'; -import type { SyncMethod, SyncView } from '../shared/sync'; +import type { SyncMethod, SyncSourceReply, SyncView } from '../shared/sync'; +import { RendererSources } from './sync-renderer'; import { SyncService } from './sync-service'; import { SYNC_FILE_EXTENSION } from './file-remote'; @@ -54,6 +55,14 @@ let automation: AutomationHost | undefined; let journaling: JournalingService | undefined; let updates: UpdateService | undefined; let sync: SyncService | undefined; +const syncSources = new RendererSources({ + send: (id, request) => { + if (!window || window.isDestroyed()) return false; + window.webContents.send('sync:source', id, request); + return true; + }, +}); +let isSyncFlushed = false; let isChanging = false; let isWindowLoaded = false; let uiAccount: Account | null = null; @@ -170,6 +179,7 @@ function closePopouts(): void { } function createWindow(): void { + isSyncFlushed = false; window = new BrowserWindow({ width: 1440, height: 900, @@ -212,9 +222,20 @@ function createWindow(): void { : [{ role: 'copy' }]; Menu.buildFromTemplate(template).popup({ window }); }); - // A reload or a closed main window takes the pop-outs' page with it. - window.webContents.on('did-start-loading', closePopouts); + // A reload or a closed main window takes the pop-outs' page with it, and the stores sync reads. + window.webContents.on('did-start-loading', () => { + closePopouts(); + syncSources.setReady(false); + }); + // Edits not yet synced are pushed while the window can still be asked for them. + window.on('close', (event) => { + if (isSyncFlushed || !sync?.hasUnpushedChanges) return; + event.preventDefault(); + isSyncFlushed = true; + void Promise.race([sync.flush(), new Promise((resolve) => setTimeout(resolve, 5000))]).finally(() => window?.close()); + }); window.on('closed', () => { + syncSources.setReady(false); closePopouts(); window = undefined; if (process.platform !== 'darwin') app.quit(); @@ -495,6 +516,11 @@ ipcMain.handle('sync:request', async (_event, method: SyncMethod, params: unknow return { error: { code: 0, message: (error as Error).message } }; } }); +// Only the main window serves the stores; pop-outs share its page but have no preload of their own. +const isMainWindow = (sender: Electron.WebContents) => !!window && !window.isDestroyed() && sender === window.webContents; +ipcMain.on('sync:source-ready', (event) => { if (isMainWindow(event.sender)) syncSources.setReady(true); }); +ipcMain.on('sync:source-reply', (event, id: number, reply: SyncSourceReply) => { if (isMainWindow(event.sender)) syncSources.receive(id, reply); }); +ipcMain.on('sync:changed', (event) => { if (isMainWindow(event.sender)) sync?.noteLocalChange(); }); ipcMain.handle('updates:request', async (_event, method: UpdateMethod, params: unknown) => { try { if (!updates) throw new Error('Updates are still starting'); @@ -621,6 +647,7 @@ if (!app.requestSingleInstanceLock()) { journal: (message) => journal(message, 'Sync'), openExternal: (url) => shell.openExternal(url), revealFile: (file) => shell.showItemInFolder(file), + sources: () => syncSources.sources(), chooseFile: async (mode) => { const filters = [{ name: 'OpenTerm5 sync file', extensions: [SYNC_FILE_EXTENSION] }]; if (mode === 'open') { @@ -647,8 +674,6 @@ app.on('window-all-closed', () => { } }); -let isSyncFlushed = false; - // Quitting waits up to 5 s for edits not yet synced, so they are not left on one computer. app.on('before-quit', (event) => { if (!isSyncFlushed && sync?.hasUnpushedChanges) { diff --git a/electron/preload.ts b/electron/preload.ts index 8ec224b..ce7a527 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -3,7 +3,7 @@ import type { TerminalEvent, TerminalMethod, ConnectionSettings, TerminalApi } f import type { AutomationApi, AutomationEvent, AutomationMethod } from '../shared/automation'; import type { JournalingApi, JournalingEvent, JournalingMethod } from '../shared/journaling'; import type { UpdateApi, UpdateEvent, UpdateMethod } from '../shared/updates'; -import type { SyncApi, SyncEvent, SyncMethod } from '../shared/sync'; +import type { SyncApi, SyncEvent, SyncMethod, SyncSourceRequest } from '../shared/sync'; import type { ZoomStep } from '../shared/zoom'; class RequestError extends Error { @@ -67,6 +67,17 @@ const updates: UpdateApi = { const sync: SyncApi = { request: (method: SyncMethod, params: unknown) => invoke('sync:request', method, params), on: (event: SyncEvent, listener: (data: never) => void) => onSync(event, listener), + serve: (handler: (request: SyncSourceRequest) => Promise) => { + ipcRenderer.removeAllListeners('sync:source'); + ipcRenderer.on('sync:source', (_: IpcRendererEvent, id: number, request: SyncSourceRequest) => { + handler(request).then( + (result) => ipcRenderer.send('sync:source-reply', id, { result }), + (error: unknown) => ipcRenderer.send('sync:source-reply', id, { error: error instanceof Error ? error.message : String(error) }), + ); + }); + ipcRenderer.send('sync:source-ready'); + }, + changed: () => ipcRenderer.send('sync:changed'), } as SyncApi; contextBridge.exposeInMainWorld('terminal', api); diff --git a/electron/sync-engine-fixtures.ts b/electron/sync-engine-fixtures.ts index 9e9f91a..e0befa9 100644 --- a/electron/sync-engine-fixtures.ts +++ b/electron/sync-engine-fixtures.ts @@ -68,7 +68,7 @@ export class MemorySource implements SyncSource { async snapshot(): Promise>> { return structuredClone(this.data); } - validate(value: Json): void { + validate(value: Json): void | Promise { if (!this.isValid(value)) throw new Error('invalid'); } diff --git a/electron/sync-engine.test.ts b/electron/sync-engine.test.ts index 6c14284..169b58b 100644 --- a/electron/sync-engine.test.ts +++ b/electron/sync-engine.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { randomBytes } from 'node:crypto'; import { test } from 'node:test'; import { deriveKeys, newRootKey } from './sync-crypto'; +import { SyncSourceUnavailableError } from './sync-engine'; import { device, FakeRemote, REPO_ID } from './sync-engine-fixtures'; import { MANIFEST_PATH, openManifest, sealManifest, SyncIntegrityError } from './sync-manifest'; @@ -312,3 +313,16 @@ test('two computers syncing many random edits end up identical', async () => { for (const computer of [a, b, a]) await computer.engine.sync({ isMassChangeAllowed: true }); assert.deepEqual(a.source('chartDrawings').data, b.source('chartDrawings').data); }); + +test('a source that cannot answer stops the sync instead of reading its values as invalid', async () => { + const { a, b } = setup(); + a.source('layout').data = { L1: { name: 'Layout' } }; + await a.engine.sync(); + const source = b.source('layout'); + source.validate = async () => { throw new SyncSourceUnavailableError('still loading'); }; + await assert.rejects(b.engine.sync(), SyncSourceUnavailableError); + assert.deepEqual(source.data, {}); + source.validate = async () => undefined; + await b.engine.sync(); + assert.deepEqual(source.data, { L1: { name: 'Layout' } }); +}); diff --git a/electron/sync-engine.ts b/electron/sync-engine.ts index b1a5fde..aa5532f 100644 --- a/electron/sync-engine.ts +++ b/electron/sync-engine.ts @@ -1,4 +1,4 @@ -import { isSameJson, isSyncKind, recordKey, splitRecordKey, syncKindInfo, type Json, type SyncKind } from '../shared/sync'; +import { isSameJson, isSyncKind, recordKey, splitRecordKey, syncKindInfo, type Json, type SyncKind, type SyncSourceChange } from '../shared/sync'; import { hlcAt, HybridClock } from './sync-clock'; import { decodePayload, encodePayload, envelopeHash, MAX_ENVELOPE_BYTES, objectPath, open, recordCipherKey, seal, type SyncKeys } from './sync-crypto'; import { emptyManifest, MANIFEST_PATH, openManifest, sealManifest, SyncIntegrityError, type Manifest, type ManifestDevice, type ManifestEntry } from './sync-manifest'; @@ -21,16 +21,17 @@ export interface SyncRemote { commit(parent: string | null, writes: ReadonlyMap): Promise; } -export interface SourceChange { - readonly upserts: Readonly>; - readonly deletes: readonly string[]; -} +export type SourceChange = SyncSourceChange; + +// A source that cannot answer at all (its window is still loading) stops the sync, rather than +// having its values read as invalid or missing. +export class SyncSourceUnavailableError extends Error {} export interface SyncSource { readonly kind: SyncKind; snapshot(): Promise>>; - // Throws when a value would be unsafe to apply; it never changes the value. - validate(value: Json): void; + // Throws (or rejects) when a value would be unsafe to apply under that id; it never changes the value. + validate(value: Json, id: string): void | Promise; apply(change: SourceChange): Promise; } @@ -137,7 +138,7 @@ export class SyncEngine { for (const id of Object.keys(snapshot)) keys.add(recordKey(kind, id)); for (const key of keys) { const current = snapshot[splitRecordKey(key).id]; - if (current !== undefined && !this.isValidPlain(source, current)) continue; + if (current !== undefined && !await this.isValidPlain(source, current, splitRecordKey(key).id)) continue; const baseline = state.pending[key] ?? state.bases[key]; const stamped = stampRecord(shape, baseline, current, () => this.clock.tick()); if (stamped === baseline || !stamped) continue; @@ -213,7 +214,7 @@ export class SyncEngine { catch { throw new SyncIntegrityError('tampered', `A synced record (${splitRecordKey(key).kind}) failed authentication.`); } // An authentic record that fails validation came from a buggy build, not an attacker: it is // left alone on this computer rather than blocking every other record. - if (!this.isValid(key, record)) invalid.add(key); + if (!await this.isValid(key, record)) invalid.add(key); else records[key] = record; } for (const record of Object.values(records)) { @@ -254,7 +255,7 @@ export class SyncEngine { if (view.invalid.has(key) || (entry && !this.isReadable(key, entry))) { skipped.add(key); continue; } const base = state.bases[key]; const current = snapshot[id]; - if (current !== undefined && !this.isValidPlain(source, current)) { skipped.add(key); continue; } + if (current !== undefined && !await this.isValidPlain(source, current, id)) { skipped.add(key); continue; } const local = options.isLocalDiscarded ? base : stampRecord(shape, state.pending[key] ?? base, current, tick); const remote = view.records[key]; const result = mergeRecords(shape, base, local, remote, tick); @@ -405,13 +406,19 @@ export class SyncEngine { return isSyncKind(kind) && this.sources.has(kind) && entry.schema <= syncKindInfo(kind).schema; } - private isValid(key: string, record: SyncedRecord): boolean { - const kind = splitRecordKey(key).kind as SyncKind; - const plain = recordPlain(syncKindInfo(kind).shape, record); - return plain === undefined || this.isValidPlain(this.sources.get(kind)!, plain); + private async isValid(key: string, record: SyncedRecord): Promise { + const { kind, id } = splitRecordKey(key); + const plain = recordPlain(syncKindInfo(kind as SyncKind).shape, record); + return plain === undefined || this.isValidPlain(this.sources.get(kind as SyncKind)!, plain, id); } - private isValidPlain(source: SyncSource, plain: Json): boolean { - try { source.validate(plain); return true; } catch { return false; } + private async isValidPlain(source: SyncSource, plain: Json, id: string): Promise { + try { + await source.validate(plain, id); + return true; + } catch (error) { + if (error instanceof SyncSourceUnavailableError) throw error; + return false; + } } } diff --git a/electron/sync-renderer.test.ts b/electron/sync-renderer.test.ts new file mode 100644 index 0000000..d1178ef --- /dev/null +++ b/electron/sync-renderer.test.ts @@ -0,0 +1,64 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { SyncSourceReply, SyncSourceRequest } from '../shared/sync'; +import { SyncSourceUnavailableError } from './sync-engine'; +import { RendererSources } from './sync-renderer'; + +function setup(answer: (request: SyncSourceRequest) => SyncSourceReply | null, timeoutMs = 50) { + const asked: SyncSourceRequest[] = []; + let hasWindow = true; + const host: RendererSources = new RendererSources({ + timeoutMs, + send: (id, request) => { + if (!hasWindow) return false; + asked.push(request); + const reply = answer(request); + if (reply) queueMicrotask(() => host.receive(id, reply)); + return true; + }, + }); + const source = (kind: SyncSourceRequest['kind']) => host.sources().find((candidate) => candidate.kind === kind)!; + return { host, asked, source, closeWindow: () => { hasWindow = false; } }; +} + +test('nothing is asked before the renderer says its stores have loaded', async () => { + const { host, asked, source } = setup(() => ({ result: { values: {}, invalid: [] } })); + await assert.rejects(source('chart').snapshot(), SyncSourceUnavailableError); + host.setReady(true); + assert.deepEqual(await source('chart').snapshot(), {}); + assert.equal(asked.length, 1); +}); + +test('values a snapshot returned are checked without asking again; others are asked about', async () => { + const values = { c1: { symbol: 'EURUSD' }, c2: { symbol: '' } }; + const { host, asked, source } = setup((request) => request.method === 'snapshot' + ? { result: { values, invalid: ['c2'] } } + : request.method === 'validate' && (request.value as { symbol: string }).symbol ? { result: null } : { error: 'Chart c3 is not readable.' }); + host.setReady(true); + const chart = source('chart'); + const snapshot = await chart.snapshot(); + await chart.validate(snapshot.c1, 'c1'); + await assert.rejects(Promise.resolve(chart.validate(snapshot.c2, 'c2')), /failed its checks/); + assert.equal(asked.length, 1); + await chart.validate({ symbol: 'GBPUSD' }, 'c3'); + await assert.rejects(Promise.resolve(chart.validate({ symbol: '' }, 'c3')), (error: Error) => !(error instanceof SyncSourceUnavailableError) && /not readable/.test(error.message)); + assert.deepEqual(asked.slice(1).map((request) => request.method === 'validate' && request.id), ['c3', 'c3']); +}); + +test('a renderer that stops answering, reloads or has no window fails as unavailable', async () => { + const { host, source, closeWindow } = setup(() => null, 20); + host.setReady(true); + await assert.rejects(source('watchlists').snapshot(), SyncSourceUnavailableError); + const pending = source('watchlists').apply({ upserts: {}, deletes: [] }); + host.setReady(false); + await assert.rejects(pending, SyncSourceUnavailableError); + host.setReady(true); + closeWindow(); + await assert.rejects(source('watchlists').snapshot(), SyncSourceUnavailableError); +}); + +test('a malformed snapshot is refused rather than read as empty', async () => { + const { host, source } = setup(() => ({ result: { values: [], invalid: [] } })); + host.setReady(true); + await assert.rejects(source('chart').snapshot(), /malformed/); +}); diff --git a/electron/sync-renderer.ts b/electron/sync-renderer.ts new file mode 100644 index 0000000..d3513c7 --- /dev/null +++ b/electron/sync-renderer.ts @@ -0,0 +1,91 @@ +import { rendererSyncKinds, type Json, type RendererSyncKind, type SyncSnapshot, type SyncSourceReply, type SyncSourceRequest } from '../shared/sync'; +import { SyncSourceUnavailableError, type SyncSource } from './sync-engine'; + +const REQUEST_TIMEOUT_MS = 30_000; + +export interface RendererSourcesDeps { + // False when there is no window to ask. + readonly send: (id: number, request: SyncSourceRequest) => boolean; + readonly timeoutMs?: number; +} + +interface Waiting { + readonly resolve: (result: unknown) => void; + readonly reject: (error: Error) => void; + readonly timer: ReturnType; +} + +const isObject = (value: unknown): value is object => value !== null && typeof value === 'object'; + +// The renderer's stores, reached over IPC. The renderer checks its own values when it takes a +// snapshot, so the engine validating those again needs no round trip; only remote values do. +export class RendererSources { + private next = 1; + private isReady = false; + private readonly waiting = new Map(); + private readonly checked = new WeakSet(); + private readonly refused = new WeakSet(); + + constructor(private readonly deps: RendererSourcesDeps) {} + + // Ready once the renderer's stores have loaded; a reload or a closed window fails what was asked. + setReady(isReady: boolean): void { + this.isReady = isReady; + if (isReady) return; + for (const id of [...this.waiting.keys()]) this.fail(id, new SyncSourceUnavailableError('The window closed before it answered.')); + } + + receive(id: number, reply: SyncSourceReply): void { + const waiting = this.waiting.get(id); + if (!waiting) return; + this.waiting.delete(id); + clearTimeout(waiting.timer); + if ('error' in reply) waiting.reject(new Error(reply.error)); + else waiting.resolve(reply.result); + } + + sources(): SyncSource[] { + return rendererSyncKinds.map((kind) => this.source(kind)); + } + + private source(kind: RendererSyncKind): SyncSource { + return { + kind, + snapshot: async () => { + const snapshot = await this.request({ method: 'snapshot', kind }) as SyncSnapshot; + if (!isObject(snapshot?.values) || Array.isArray(snapshot.values) || !Array.isArray(snapshot.invalid)) throw new Error(`The ${kind} snapshot is malformed.`); + const invalid = new Set(snapshot.invalid); + for (const [id, value] of Object.entries(snapshot.values)) { + if (isObject(value)) (invalid.has(id) ? this.refused : this.checked).add(value); + } + return snapshot.values; + }, + validate: async (value: Json, id: string) => { + if (isObject(value) && this.checked.has(value)) return; + if (isObject(value) && this.refused.has(value)) throw new Error(`This ${kind} record failed its checks.`); + await this.request({ method: 'validate', kind, id, value }); + }, + apply: async (change) => { + await this.request({ method: 'apply', kind, change }); + }, + }; + } + + private request(request: SyncSourceRequest): Promise { + if (!this.isReady) return Promise.reject(new SyncSourceUnavailableError('The app window is still loading.')); + const id = this.next++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => this.fail(id, new SyncSourceUnavailableError('The app window did not answer.')), this.deps.timeoutMs ?? REQUEST_TIMEOUT_MS); + this.waiting.set(id, { resolve, reject, timer }); + if (!this.deps.send(id, request)) this.fail(id, new SyncSourceUnavailableError('There is no app window.')); + }); + } + + private fail(id: number, error: Error): void { + const waiting = this.waiting.get(id); + if (!waiting) return; + this.waiting.delete(id); + clearTimeout(waiting.timer); + waiting.reject(error); + } +} diff --git a/electron/sync-service.test.ts b/electron/sync-service.test.ts index 7ff945f..525c85d 100644 --- a/electron/sync-service.test.ts +++ b/electron/sync-service.test.ts @@ -8,7 +8,8 @@ import type { PasswordCipher } from './account-registry'; import { FakeGitHub } from './github-fixtures'; import { GitHubApi, GitHubRemote } from './github-remote'; import { newRootKey } from './sync-crypto'; -import { FakeTimers } from './sync-engine-fixtures'; +import type { SyncSource } from './sync-engine'; +import { FakeTimers, MemorySource } from './sync-engine-fixtures'; import { inspectRepository, updateKeyring, withDeviceWrap } from './sync-keyring'; import { SyncService } from './sync-service'; @@ -28,7 +29,7 @@ async function until(check: () => boolean): Promise { await settle(); } -async function computer(github: FakeGitHub, name: string, dir?: string, file?: string) { +async function computer(github: FakeGitHub, name: string, dir?: string, file?: string, sources: readonly SyncSource[] = []) { const folder = dir ?? await fs.mkdtemp(path.join(os.tmpdir(), 'openterm-sync-service-')); const timers = new FakeTimers(); const views: SyncView[] = []; @@ -37,7 +38,7 @@ async function computer(github: FakeGitHub, name: string, dir?: string, file?: s const service = new SyncService({ dir: folder, fetch: github.fetch, cipher, platform: 'linux', appVersion: '0.12.0', deviceName: name, send: (view) => views.push(view), journal: (message) => journal.push(message), openExternal: async (url) => { opened.push(url); }, chooseFile: async () => file ?? null, - now: () => 1_800_000_000_000 + timers.time, timers, pause: async () => undefined, passphraseCost: { N: 2 ** 14, r: 8, p: 1 }, + now: () => 1_800_000_000_000 + timers.time, timers, pause: async () => undefined, passphraseCost: { N: 2 ** 14, r: 8, p: 1 }, sources: () => sources, }); await service.init(); const tick = async (ms = 0) => { @@ -56,8 +57,8 @@ async function signIn(device: Awaited>) { await until(() => device.view().userCode === null); } -async function setUpFirst(github: FakeGitHub, passphrase: string | null = null) { - const first = await computer(github, 'Office PC'); +async function setUpFirst(github: FakeGitHub, passphrase: string | null = null, sources: readonly SyncSource[] = []) { + const first = await computer(github, 'Office PC', undefined, undefined, sources); await signIn(first); await first.service.handle('sync.chooseRepository', { id: REPO }); const view = await first.service.handle('sync.create', { passphrase }); @@ -353,3 +354,37 @@ test('without GitHub, two computers sync through a sync file', async () => { assert.equal(first.view().status, 'offline'); assert.equal(first.view().attention, null); }); + +test('a computer joining synced layouts asks whether to merge its own or use the synced ones', async () => { + const github = fakeGitHub(); + const office = new MemorySource('chart'); + office.data = { c1: { symbol: 'XAUUSD' } }; + const { recoveryCode } = await setUpFirst(github, null, [office]); + const laptop = new MemorySource('chart'); + laptop.data = { c9: { symbol: 'EURUSD' } }; + const second = await computer(github, 'Laptop', undefined, undefined, [laptop]); + await signIn(second); + await second.service.handle('sync.chooseRepository', { id: REPO }); + await second.service.handle('sync.join', { recoveryCode }); + await second.tick(60_000); + assert.equal(second.view().attention?.reason, 'join'); + assert.deepEqual(laptop.data, { c9: { symbol: 'EURUSD' } }); + await second.service.handle('sync.resolve', { choice: 'useSynced' }); + await second.tick(); + assert.equal(second.view().status, 'idle'); + assert.deepEqual(laptop.data, { c1: { symbol: 'XAUUSD' } }); +}); + +test('a computer joining a repository with no layouts yet syncs without asking', async () => { + const github = fakeGitHub(); + const { recoveryCode } = await setUpFirst(github, null, [new MemorySource('chart')]); + const laptop = new MemorySource('chart'); + laptop.data = { c9: { symbol: 'EURUSD' } }; + const second = await computer(github, 'Laptop', undefined, undefined, [laptop]); + await signIn(second); + await second.service.handle('sync.chooseRepository', { id: REPO }); + await second.service.handle('sync.join', { recoveryCode }); + await second.tick(); + assert.equal(second.view().attention, null); + assert.equal(second.view().status, 'idle'); +}); diff --git a/electron/sync-service.ts b/electron/sync-service.ts index dc085e5..807e268 100644 --- a/electron/sync-service.ts +++ b/electron/sync-service.ts @@ -1,7 +1,7 @@ import { randomInt } from 'node:crypto'; import path from 'node:path'; import { - normalizeCodeGroup, passphraseProblem, type SyncAttention, type SyncAttentionReason, type SyncDeviceView, type SyncMassChange, type SyncMethod, type SyncMethods, + normalizeCodeGroup, passphraseProblem, rendererSyncKinds, splitRecordKey, type SyncAttention, type SyncAttentionReason, type SyncDeviceView, type SyncMassChange, type SyncMethod, type SyncMethods, type SyncPage, type SyncPairingRequest, type SyncPhase, type SyncRepositoryChoice, type SyncResolution, type SyncStatus, type SyncView, } from '../shared/sync'; import type { PasswordCipher } from './account-registry'; @@ -9,7 +9,7 @@ import { FileRemote, SyncFileDamagedError, SyncFileUnavailableError } from './fi import { GITHUB_APP_SLUG, GITHUB_WEB, GitHubAuth, GitHubAuthError, type DeviceCode, type Fetch } from './github-auth'; import { GitHubApi, GitHubNetworkError, GitHubRateLimitError, GitHubRemote, GitHubRequestError } from './github-remote'; import { deriveKeys, newDeviceKeyPair, SyncCryptoError } from './sync-crypto'; -import { SyncEngine, type SyncOptions, type SyncRemote, type SyncSource } from './sync-engine'; +import { SyncEngine, SyncSourceUnavailableError, type SyncOptions, type SyncRemote, type SyncSource } from './sync-engine'; import { answerPairing, FORMAT_PATH, inspectRepository, newKeyring, PAIRING_LIFETIME_MS, pairingConfirmation, pairingWords, requestPairing, SyncSetupError, unlockKeyring, updateKeyring, withDeviceWrap, withNewRecovery, type RepositorySetup, @@ -48,6 +48,8 @@ const heldMessage = (changes: readonly SyncMassChange[]) => changes.some((change ? 'Much of the data on this computer would be deleted from every computer. Choose whether to delete it everywhere or restore it from sync.' : 'Another computer deleted much of the synced data. Choose whether to delete it here too.'; +const joinMessage = 'This computer has layouts, drawings and lists of its own, and the synced data has some too. Merge adds this computer\'s to the synced ones on every computer; Use synced data replaces this computer\'s with the synced ones.'; + const groupIdOf = (target: SyncTarget) => target.kind === 'github' ? target.repository.id : target.groupId; const labelOf = (target: SyncTarget) => target.kind === 'github' ? target.repository.fullName : target.path; @@ -375,7 +377,25 @@ export class SyncService { await this.credentials.update({ target, rootKey, deviceKeys, epoch, isRecoveryConfirmed: true }); [this.phase, this.pairingWords] = ['on', []]; this.startEngine(target); - this.scheduler.start(); + if (await this.hasSyncedContent(target, rootKey, epoch)) this.attend('join', joinMessage); + else this.scheduler.start(); + } + + // A computer joining for the first time asks before its own data meets the synced data. When the + // repository cannot be read now it asks anyway, so nothing merges without a choice. + private async hasSyncedContent(target: SyncTarget, rootKey: Buffer, epoch: number): Promise { + if (await this.stateStore?.load()) return false; + try { + const remote = this.remoteFor(target); + const head = await remote.head(); + const envelope = head === null ? null : (await remote.read(head, [MANIFEST_PATH])).get(MANIFEST_PATH); + if (!envelope) return false; + const manifest = openManifest(deriveKeys(rootKey), envelope, { epoch, repoId: groupIdOf(target), highWater: 0 }); + const kinds = new Set(rendererSyncKinds); + return Object.entries(manifest.records).some(([key, entry]) => entry.hash !== null && kinds.has(splitRecordKey(key).kind)); + } catch { + return true; + } } // The new computer posts its public key and waits for a linked computer to wrap the key to it. @@ -514,6 +534,7 @@ export class SyncService { private async resolve(choice: SyncResolution): Promise { const options: Record = { + merge: {}, allowDeletions: { isMassChangeAllowed: true }, useSynced: { isLocalDiscarded: true, isMassChangeAllowed: true }, acceptHistory: { isHistoryAccepted: true, isLocalDiscarded: true, isMassChangeAllowed: true }, @@ -619,6 +640,10 @@ export class SyncService { private failed(error: Error): RunResult { if (error instanceof SyncIntegrityError) return this.attend(error.reason, error.message); + if (error instanceof SyncSourceUnavailableError) { + this.status = 'idle'; + return { kind: 'retry' }; + } if (error instanceof GitHubAuthError && error.reason === 'reconnect') return this.attend('reconnect', error.message); if (error instanceof GitHubRateLimitError) { [this.status, this.error] = ['waiting', error.message]; diff --git a/packages/meridian-charts/src/index.ts b/packages/meridian-charts/src/index.ts index 758f01c..1599f6e 100644 --- a/packages/meridian-charts/src/index.ts +++ b/packages/meridian-charts/src/index.ts @@ -24,6 +24,7 @@ export { type MeridianUiTokens, type ThemePatch, } from './themes'; +export { customColorList, onCustomColorsChange, setCustomColors } from './ui/color-picker'; export { iconSvg } from './ui/icons'; export { DrawingToolbar, type CursorMode, type MagnetMode, type ToolbarActions, type ToolbarState } from './ui/toolbar'; export { ensureStyles } from './ui/styles'; diff --git a/packages/meridian-charts/src/ui/color-picker.ts b/packages/meridian-charts/src/ui/color-picker.ts index 74f5b0b..e35ba75 100644 --- a/packages/meridian-charts/src/ui/color-picker.ts +++ b/packages/meridian-charts/src/ui/color-picker.ts @@ -29,6 +29,21 @@ function saveCustom(colors: readonly string[]): void { // Shared by every picker in the window, as the custom row is in TradingView. let customColors = loadCustom(); +const customListeners = new Set<() => void>(); + +// For a host that keeps the custom row elsewhere too (sync between computers). +export const customColorList = (): readonly string[] => customColors; + +export function setCustomColors(colors: readonly string[]): void { + customColors = colors.filter((color) => /^#[0-9a-f]{6}$/i.test(color)).slice(0, CUSTOM_LIMIT); + saveCustom(customColors); +} + +export function onCustomColorsChange(listener: () => void): () => void { + customListeners.add(listener); + return () => customListeners.delete(listener); +} + // Places a popup under `anchor` inside `host`, flipping above it when there is no room below. export function placePopup(popup: HTMLElement, host: HTMLElement, anchor: HTMLElement): void { const hostBox = host.getBoundingClientRect(); @@ -112,6 +127,7 @@ export function openColorPicker(host: HTMLElement, anchor: HTMLElement, current: const color = input.value.toLowerCase(); customColors = [color, ...customColors.filter((candidate) => candidate !== color)].slice(0, CUSTOM_LIMIT); saveCustom(customColors); + customListeners.forEach((listener) => listener()); renderCustom(); choose(color); }; diff --git a/shared/sync.ts b/shared/sync.ts index 6b03ae3..fae33d4 100644 --- a/shared/sync.ts +++ b/shared/sync.ts @@ -12,40 +12,48 @@ export type Json = null | boolean | number | string | readonly Json[] | { readon // - value: one value; the newer write wins. export type SyncShape = 'fields' | 'map' | 'set' | 'list' | 'text' | 'value'; -// Declaration order is the order kinds sync in and are listed in. Raise `schema` when a kind's body -// changes shape; older builds then leave records of the newer schema alone instead of rewriting them. +// Declaration order is the order kinds sync in and are listed in: a kind applies after the kinds it +// refers to (scripts and themes before the charts using them, charts before the layouts holding +// them). Raise `schema` when a kind's body changes shape; older builds then leave records of the +// newer schema alone instead of rewriting them. `side` is the process whose stores hold the kind. export const syncKinds = [ - { key: 'account', label: 'Accounts', shape: 'fields', schema: 1, isSecret: false }, - { key: 'accountSecret', label: 'Saved passwords and API keys', shape: 'value', schema: 1, isSecret: true }, - { key: 'identity', label: 'CID', shape: 'value', schema: 1, isSecret: true }, - { key: 'webhookSecrets', label: 'Webhook settings', shape: 'fields', schema: 1, isSecret: true }, - { key: 'layout', label: 'Layouts', shape: 'fields', schema: 1, isSecret: false }, - { key: 'layoutOrder', label: 'Layout order', shape: 'list', schema: 1, isSecret: false }, - { key: 'chart', label: 'Charts', shape: 'fields', schema: 1, isSecret: false }, - { key: 'chartDrawings', label: 'Chart drawings', shape: 'map', schema: 1, isSecret: false }, - { key: 'symbolDrawings', label: 'Symbol drawings', shape: 'map', schema: 1, isSecret: false }, - { key: 'chartProfile', label: 'Chart profiles', shape: 'fields', schema: 1, isSecret: false }, - { key: 'chartTemplate', label: 'Chart templates', shape: 'fields', schema: 1, isSecret: false }, - { key: 'drawingTemplates', label: 'Drawing templates', shape: 'map', schema: 1, isSecret: false }, - { key: 'drawingLooks', label: 'Drawing looks', shape: 'map', schema: 1, isSecret: false }, - { key: 'theme', label: 'Themes', shape: 'fields', schema: 1, isSecret: false }, - { key: 'watchlists', label: 'Watchlists', shape: 'map', schema: 1, isSecret: false }, - { key: 'indicatorFavorites', label: 'Indicator favourites', shape: 'set', schema: 1, isSecret: false }, - { key: 'timeframeFavorites', label: 'Timeframe favourites', shape: 'set', schema: 1, isSecret: false }, - { key: 'drawingFavorites', label: 'Drawing favourites', shape: 'set', schema: 1, isSecret: false }, - { key: 'customColors', label: 'Custom colours', shape: 'list', schema: 1, isSecret: false }, - { key: 'priceAlert', label: 'Price alerts', shape: 'fields', schema: 1, isSecret: false }, - { key: 'pineScript', label: 'Pine scripts', shape: 'text', schema: 1, isSecret: false }, - { key: 'preferences', label: 'Preferences', shape: 'fields', schema: 1, isSecret: false }, - { key: 'journalingLink', label: 'Journaling connection', shape: 'fields', schema: 1, isSecret: false }, - { key: 'copierConfig', label: 'Copier settings', shape: 'fields', schema: 1, isSecret: false }, - { key: 'automationRule', label: 'Automation rules', shape: 'fields', schema: 1, isSecret: false }, -] as const satisfies readonly { key: string; label: string; shape: SyncShape; schema: number; isSecret: boolean }[]; + { key: 'account', label: 'Accounts', shape: 'fields', schema: 1, isSecret: false, side: 'main' }, + { key: 'accountSecret', label: 'Saved passwords and API keys', shape: 'value', schema: 1, isSecret: true, side: 'main' }, + { key: 'identity', label: 'CID', shape: 'value', schema: 1, isSecret: true, side: 'main' }, + { key: 'webhookSecrets', label: 'Webhook settings', shape: 'fields', schema: 1, isSecret: true, side: 'main' }, + { key: 'theme', label: 'Themes', shape: 'value', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'pineScript', label: 'Pine scripts', shape: 'text', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'chart', label: 'Charts', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'chartDrawings', label: 'Chart drawings', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'layout', label: 'Layouts', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'layoutCharts', label: 'Layout charts', shape: 'list', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'layoutDrawings', label: 'Layout drawings', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'layoutOrder', label: 'Layout order', shape: 'list', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'symbolDrawings', label: 'Symbol drawings', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'chartProfile', label: 'Chart profiles', shape: 'value', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'chartTemplate', label: 'Chart templates', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'drawingTemplates', label: 'Drawing templates', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'drawingLooks', label: 'Drawing looks', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'watchlists', label: 'Watchlists', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'indicatorFavorites', label: 'Indicator favourites', shape: 'set', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'timeframeFavorites', label: 'Timeframe favourites', shape: 'set', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'drawingFavorites', label: 'Drawing favourites', shape: 'set', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'customColors', label: 'Custom colours', shape: 'list', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'priceAlert', label: 'Price alerts', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'preferences', label: 'Preferences', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'journalingLink', label: 'Journaling connection', shape: 'fields', schema: 1, isSecret: false, side: 'main' }, + { key: 'copierConfig', label: 'Copier settings', shape: 'fields', schema: 1, isSecret: false, side: 'main' }, + { key: 'automationRule', label: 'Automation rules', shape: 'fields', schema: 1, isSecret: false, side: 'main' }, +] as const satisfies readonly { key: string; label: string; shape: SyncShape; schema: number; isSecret: boolean; side: 'main' | 'renderer' }[]; export type SyncKind = (typeof syncKinds)[number]['key']; +export type RendererSyncKind = Extract<(typeof syncKinds)[number], { side: 'renderer' }>['key']; + export const isSyncKind = (value: string): value is SyncKind => syncKinds.some((kind) => kind.key === value); +export const rendererSyncKinds: readonly RendererSyncKind[] = syncKinds.flatMap((kind) => kind.side === 'renderer' ? [kind.key] : []); + export const syncKindInfo = (kind: SyncKind) => syncKinds.find((item) => item.key === kind)!; export const recordKey = (kind: SyncKind, id: string) => `${kind}:${id}`; @@ -97,7 +105,7 @@ export type SyncStatus = (typeof syncStatuses)[number]['key']; export type SyncAttentionReason = | 'tampered' | 'rollback' | 'epoch' | 'tooNew' | 'stale' | 'revoked' - | 'held' | 'reconnect' | 'repositoryPublic' | 'repositoryMissing' | 'failed'; + | 'held' | 'join' | 'reconnect' | 'repositoryPublic' | 'repositoryMissing' | 'failed'; export interface SyncMassChange { readonly kind: SyncKind; @@ -163,7 +171,7 @@ export interface SyncView { export type SyncPage = 'install' | 'verification' | 'authorizations' | 'repository' | 'newRepository'; -export type SyncResolution = 'allowDeletions' | 'useSynced' | 'acceptHistory'; +export type SyncResolution = 'merge' | 'allowDeletions' | 'useSynced' | 'acceptHistory'; export interface SyncMethods { 'sync.view': [Record, SyncView]; @@ -195,9 +203,33 @@ export interface SyncEvents { export type SyncEvent = keyof SyncEvents; +export interface SyncSourceChange { + readonly upserts: Readonly>; + readonly deletes: readonly string[]; +} + +// A snapshot also names the records that fail their own checks, so they are left out of sync +// instead of being read as deleted. +export interface SyncSnapshot { + readonly values: Readonly>; + readonly invalid: readonly string[]; +} + +// What the main process asks of the renderer's stores. +export type SyncSourceRequest = + | { readonly method: 'snapshot'; readonly kind: RendererSyncKind } + | { readonly method: 'validate'; readonly kind: RendererSyncKind; readonly id: string; readonly value: Json } + | { readonly method: 'apply'; readonly kind: RendererSyncKind; readonly change: SyncSourceChange }; + +export type SyncSourceReply = { readonly result: unknown } | { readonly error: string }; + export interface SyncApi { request(method: M, params: SyncMethods[M][0]): Promise; on(event: E, listener: (data: SyncEvents[E]) => void): () => void; + // Answers the main process's requests for the renderer's stores; called once they have loaded. + serve(handler: (request: SyncSourceRequest) => Promise): void; + // Tells the main process a synced store changed, so the edit is stamped and pushed. + changed(): void; } // Anyone holding the repository can guess passphrases offline for ever, so they must be long. diff --git a/src/components/chart/ChartWindow.tsx b/src/components/chart/ChartWindow.tsx index 0be5bf0..34ee40b 100644 --- a/src/components/chart/ChartWindow.tsx +++ b/src/components/chart/ChartWindow.tsx @@ -20,7 +20,7 @@ import { drawingsForSymbol, setSymbolDrawings, symbolDrawings } from '../../lib/ import { themeCssVariables } from '@meridian/charts/themes'; import { chartTheme, themeStore } from '../../lib/theme'; import { importedThemes } from '../../lib/custom-themes'; -import { focusChart, layoutDrawings, setLayoutDrawings, updateChart, workspace, type ChartConfig } from '../../lib/workspace'; +import { focusChart, isSyncedDrawings, layoutDrawings, setLayoutDrawings, updateChart, workspace, type ChartConfig } from '../../lib/workspace'; import { chartMenu, liteChartMenu } from '../../lib/chart-menu'; import { openContextMenu } from '../../lib/context-menu'; import { OneClickPanel } from './OneClickPanel'; @@ -256,11 +256,12 @@ export function ChartWindow({ chart, isActive, isLite = false }: ChartWindowProp }, [instance, chart.indicators]); // A change to the chart's own list came from this chart's menus or dialogs and can be undone there; - // a change to its tab's or symbol's list alone came from another chart or an import. + // a change to its tab's or symbol's list alone came from another chart or an import, and one from + // sync came from another computer. const appliedOwn = useRef(chart.drawings); useEffect(() => { if (!instance) return; - const isOwnChange = appliedOwn.current !== chart.drawings; + const isOwnChange = appliedOwn.current !== chart.drawings && !isSyncedDrawings(chart.drawings); appliedOwn.current = chart.drawings; const drawings = [...chart.drawings, ...tabShared, ...shared]; keepOwn(chart.drawings); diff --git a/src/components/dialogs/RiskTicket.tsx b/src/components/dialogs/RiskTicket.tsx index 8842561..f39f237 100644 --- a/src/components/dialogs/RiskTicket.tsx +++ b/src/components/dialogs/RiskTicket.tsx @@ -7,6 +7,7 @@ import { formatMoney, formatPrice } from '../../lib/format'; import { session } from '../../lib/session'; import { useStore } from '../../lib/store'; import { useRiskPreview } from '../../lib/risk-preview'; +import { riskTicketStorageKeys as storageKeys } from '../../lib/risk-preferences'; import { NumberField, Segmented, SelectField, Switch, TextField } from '../ui/Controls'; import { LeverageField } from './LeverageField'; @@ -26,7 +27,6 @@ type EntryKind = (typeof entryKinds)[number]['key']; // Forex and crypto accounts keep separate settings: a pip stop or a per-lot commission means // nothing on a crypto contract. Exchanges share theirs; the crypto key predates Binance. -const storageKeys: Record = { fx: 'openterm.riskTicket.v1', crypto: 'openterm.riskTicket.bybit.v1' }; interface Preferences { mode: RiskMode; diff --git a/src/components/sync/SyncDialog.tsx b/src/components/sync/SyncDialog.tsx index fff9925..2a0c7ba 100644 --- a/src/components/sync/SyncDialog.tsx +++ b/src/components/sync/SyncDialog.tsx @@ -281,6 +281,7 @@ function RecoveryStep({ view }: SyncStepProps) { const resolutionLabels: Partial> = { held: [{ choice: 'useSynced', label: 'Restore from sync' }, { choice: 'allowDeletions', label: 'Delete everywhere' }], + join: [{ choice: 'merge', label: 'Merge' }, { choice: 'useSynced', label: 'Use synced data' }], stale: [{ choice: 'useSynced', label: 'Use the synced data' }], rollback: [{ choice: 'acceptHistory', label: 'I restored it on purpose' }], }; @@ -305,7 +306,7 @@ function AttentionBanner({ attention }: AttentionBannerProps) {
{attention.reason === 'reconnect' && } {choices.map(({ choice, label }) => )} - {attention.reason !== 'reconnect' && } + {attention.reason !== 'reconnect' && attention.reason !== 'join' && }
diff --git a/src/lib/pine/pine-scripts.ts b/src/lib/pine/pine-scripts.ts index 6a628f6..060380c 100644 --- a/src/lib/pine/pine-scripts.ts +++ b/src/lib/pine/pine-scripts.ts @@ -231,6 +231,20 @@ export async function importPineScript(source: string, origin: PineOrigin, isQui return script; } +export const pineScriptSource = (id: string): Promise => sourceOf(id); + +// A script from another computer: saved and registered as it is, keeping its id. Charts showing +// it get their new study ids through their own synced records. +export async function putSyncedPineScript(script: PineScript, source: string): Promise { + await saveScript(script, source); + sources.set(script.id, Promise.resolve(source)); + noticed.delete(script.id); + register(script); + pineScripts.set((state) => ({ scripts: [...state.scripts.filter((candidate) => candidate.id !== script.id), script].sort(byRank) })); +} + +export const restorePineScript = (value: unknown): PineScript | null => restore([value])[0] ?? null; + // Scripts registered but not yet in the list. const pending = new Map(); diff --git a/src/lib/risk-preferences.ts b/src/lib/risk-preferences.ts index c4f09f0..ac43335 100644 --- a/src/lib/risk-preferences.ts +++ b/src/lib/risk-preferences.ts @@ -1,10 +1,14 @@ import { defaultRiskPreferences, type RiskPreferences } from '../../shared/automation'; +import type { Market } from '../../shared/protocol'; import { automation } from './automation'; import { log } from './session'; import { Store } from './store'; export const riskPreferences = new Store(defaultRiskPreferences); +// The Risk Ticket's own settings per market, saved by the ticket itself. +export const riskTicketStorageKeys: Record = { fx: 'openterm.riskTicket.v1', crypto: 'openterm.riskTicket.bybit.v1' }; + let saveTimer: ReturnType | undefined; export function loadRiskPreferences(): void { diff --git a/src/lib/sync-model.test.ts b/src/lib/sync-model.test.ts new file mode 100644 index 0000000..9b2f77b --- /dev/null +++ b/src/lib/sync-model.test.ts @@ -0,0 +1,146 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { deriveKeys, newRootKey } from '../../electron/sync-crypto'; +import { SyncEngine, type SyncSource } from '../../electron/sync-engine'; +import { FakeRemote, MemoryStore, REPO_ID } from '../../electron/sync-engine-fixtures'; +import type { Json } from '../../shared/sync'; +import { makeTab, type ChartTab } from './chart-tabs'; +import type { Drawing } from './chart-model'; +import { + applyPreferencesChange, applyWorkspaceChange, checkWorkspacePreferences, checkWorkspaceRecord, orderedItems, orderedMap, ORDER, workspaceKinds, workspaceRecords, type WorkspaceKind, +} from './sync-model'; +import { makeChart, workspace, type ChartConfig, type WorkspaceState } from './workspace'; + +const line = (id: string, price = 1.1): Drawing => ({ + id, type: 'horizontal-line', paneId: 'price', anchors: [{ time: 1_700_000_000_000, price }], style: { lineColor: '#2962ff', lineWidth: 1, lineStyle: 'solid' }, locked: false, visible: true, zIndex: 0, createdAt: 1, +}); + +function chart(id: string, symbol: string, drawings: Drawing[] = []): ChartConfig { + return { ...makeChart(symbol), id, drawings }; +} + +function stateWith(tabs: ChartTab[], charts: ChartConfig[]): WorkspaceState { + return { ...workspace.get(), tabs, charts, activeTabId: tabs[0]?.id ?? '', profiles: [], templates: [] }; +} + +const empty = () => stateWith([], []); + +const allRecords = (state: WorkspaceState) => Object.fromEntries(workspaceKinds.map((kind) => [kind, workspaceRecords(state, kind)])); + +// Applies every kind's records in sync order, as the engine does for a computer that has none yet. +function applyAll(state: WorkspaceState, records: ReturnType): WorkspaceState { + let next = state; + for (const kind of workspaceKinds) next = { ...next, ...applyWorkspaceChange(next, kind, { upserts: records[kind], deletes: [] }) }; + return next; +} + +function sample(): WorkspaceState { + const charts = [chart('c1', 'EURUSD', [line('d1'), line('d2', 1.2)]), { ...chart('c2', 'XAUUSD'), theme: 'meridian-light', linkGroup: 'red' as const }, chart('c3', 'BTCUSDT')]; + const tabs = [ + { ...makeTab('t1', ['c1', 'c2'], '2h', ['crosshair', 'symbol'], 'fx'), drawings: { EURUSD: [line('s1')] } }, + makeTab('t2', ['c3'], 's', ['crosshair'], 'crypto'), + ]; + return stateWith(tabs, charts); +} + +test('records applied to an empty workspace read back the same', () => { + const state = sample(); + const records = allRecords(state); + const applied = applyAll(empty(), records); + assert.deepEqual(allRecords(applied), records); + assert.deepEqual(applied.charts.find((item) => item.id === 'c1')!.drawings.map((drawing) => drawing.id), ['d1', 'd2']); + assert.deepEqual(applied.tabs.find((tab) => tab.id === 't1')!.drawings, { EURUSD: [line('s1')] }); + for (const kind of workspaceKinds) for (const [id, value] of Object.entries(records[kind])) checkWorkspaceRecord(kind, id, value); +}); + +test('items an order does not list follow it, so two racing orders lose nothing', () => { + const map = { ...orderedMap([{ id: 'b' }, { id: 'a' }]), c: { id: 'c' }, [ORDER]: ['b', 'missing', 'b', 'a'] }; + assert.deepEqual(orderedItems(map).map(([id]) => id), ['b', 'a', 'c']); +}); + +test('a removed chart leaves its layout, and a reordered market keeps the other market\'s tabs in place', () => { + const state = sample(); + const removed = { ...state, ...applyWorkspaceChange(state, 'chart', { upserts: {}, deletes: ['c2'] }) }; + assert.deepEqual(removed.tabs.find((tab) => tab.id === 't1')!.chartIds, ['c1']); + const withThird = { ...state, tabs: [...state.tabs, makeTab('t3', ['c1'], 's', [], 'fx')] }; + const reordered = applyWorkspaceChange(withThird, 'layoutOrder', { upserts: { fx: ['t3', 't1'] }, deletes: [] }); + assert.deepEqual(reordered.tabs!.map((tab) => tab.id), ['t3', 't2', 't1']); +}); + +test('broken records are refused before anything applies them', () => { + const bad: [WorkspaceKind, string, Json][] = [ + ['chart', 'c1', { symbol: 'EURUSD', type: 'nope', timeframe: 'H1', indicators: [], isGridShown: true, isVolumeShown: false }], + ['chartDrawings', 'c1', { d1: { id: 'other', type: 'horizontal-line', anchors: [], style: {} } }], + ['layout', 't1', { layout: 'zz', market: 'fx', syncs: [], sizes: {} }], + ['layoutCharts', 't1', ['c1', 2]], + ['layoutOrder', 'stocks', []], + ['chartTemplate', ' padded ', { settings: {} }], + ]; + for (const [kind, id, value] of bad) assert.throws(() => checkWorkspaceRecord(kind, id, value), undefined, `${kind} ${id}`); +}); + +test('workspace preferences check what they know and keep what a newer build added', () => { + assert.throws(() => checkWorkspacePreferences({ drawingSync: 'everywhere' })); + checkWorkspacePreferences({ theme: 'meridian-light', futureSetting: 3 }); + const state = workspace.get(); + const patch = applyPreferencesChange(state, { theme: 'meridian-light', defaultChartTheme: null, activeProfile: 'Missing here', futureSetting: 3 }); + assert.equal(patch.theme, 'meridian-light'); + assert.ok('defaultChartTheme' in patch && patch.defaultChartTheme === undefined); + assert.ok(!('activeProfile' in patch)); +}); + +// Two computers whose engines read and write the workspace model, as the renderer does. +function computer(id: string, remote: FakeRemote, rootKey: Buffer, clock: { now: number }, state: WorkspaceState) { + const holder = { state }; + const sources: SyncSource[] = workspaceKinds.map((kind) => ({ + kind, + snapshot: async () => structuredClone(workspaceRecords(holder.state, kind)), + validate: (value, recordId) => checkWorkspaceRecord(kind, recordId, value), + apply: async (change) => { holder.state = { ...holder.state, ...applyWorkspaceChange(holder.state, kind, change) }; }, + })); + const engine = new SyncEngine({ + epoch: 1, repoId: REPO_ID, keys: deriveKeys(rootKey), remote, store: new MemoryStore(), sources, now: () => clock.now, pause: async () => undefined, + device: { id, name: id, platform: 'linux', publicKey: 'x', appVersion: '1.0.0' }, + }); + return { holder, engine }; +} + +test('layouts, charts and drawings edited on two computers end the same on both', async () => { + const remote = new FakeRemote(); + const rootKey = newRootKey(); + const clock = { now: 1_800_000_000_000 }; + const a = computer('a', remote, rootKey, clock, sample()); + const b = computer('b', remote, rootKey, clock, empty()); + await a.engine.sync(); + await b.engine.sync({ isLocalDiscarded: true }); + assert.deepEqual(allRecords(b.holder.state), allRecords(a.holder.state)); + + // A moves one drawing and clears a chart's theme; B adds a drawing to the same chart and a layout. + clock.now += 1000; + a.holder.state = { ...a.holder.state, charts: a.holder.state.charts.map((item) => item.id === 'c1' ? { ...item, drawings: [line('d1', 1.15), item.drawings[1]] } : item.id === 'c2' ? { ...item, theme: undefined } : item) }; + b.holder.state = { + ...b.holder.state, + charts: [...b.holder.state.charts.map((item) => item.id === 'c1' ? { ...item, drawings: [...item.drawings, line('d3', 1.3)] } : item), chart('c4', 'GBPUSD')], + tabs: [...b.holder.state.tabs, makeTab('t4', ['c4'], 's', ['crosshair'], 'fx')], + }; + await a.engine.sync(); + await b.engine.sync(); + await a.engine.sync(); + assert.deepEqual(allRecords(a.holder.state), allRecords(b.holder.state)); + const c1 = a.holder.state.charts.find((item) => item.id === 'c1')!; + assert.deepEqual(c1.drawings.map((drawing) => [drawing.id, drawing.anchors[0].price]), [['d1', 1.15], ['d2', 1.2], ['d3', 1.3]]); + assert.equal(b.holder.state.charts.find((item) => item.id === 'c2')!.theme, undefined); + assert.deepEqual(a.holder.state.tabs.filter((tab) => tab.market === 'fx').map((tab) => tab.id), ['t1', 't4']); + + // B closes the new layout with its chart; A loses both, and nothing is left behind. + clock.now += 1000; + b.holder.state = { ...b.holder.state, tabs: b.holder.state.tabs.filter((tab) => tab.id !== 't4'), charts: b.holder.state.charts.filter((item) => item.id !== 'c4') }; + await b.engine.sync(); + await a.engine.sync(); + assert.deepEqual(a.holder.state.tabs.map((tab) => tab.id), ['t1', 't2']); + assert.ok(!a.holder.state.charts.some((item) => item.id === 'c4')); + for (const { engine } of [a, b]) { + const settled = await engine.sync(); + assert.equal(settled.status === 'synced' && settled.pulled + settled.pushed, 0); + } +}); diff --git a/src/lib/sync-model.ts b/src/lib/sync-model.ts new file mode 100644 index 0000000..024075c --- /dev/null +++ b/src/lib/sync-model.ts @@ -0,0 +1,295 @@ +import { restoreSettings } from '@meridian/charts/settings'; +import { markets, type Market } from '../../shared/protocol'; +import type { Json, SyncSourceChange } from '../../shared/sync'; +import { drawingSyncs, isLayoutKey, layoutOf, layoutSyncs, restoreSizes, type LayoutKey, type SplitSizes } from './chart-layouts'; +import { migrateDrawing, type Drawing } from './chart-model'; +import { makeTab, type ChartTab } from './chart-tabs'; +import { presetName, restoreChart, restoreProfiles, restoreTemplates, type ChartProfile, type ChartTemplate } from './presets'; +import { replayModes, replaySpeeds } from './replay-model'; +import type { ChartConfig, WorkspaceState } from './workspace'; + +// How the workspace and the other renderer stores turn into sync records and back. A record +// applied and read again must come out the same, or every computer would keep re-sending it. + +export type Records = Readonly>; + +// Singleton kinds (favourites, templates, looks, colours) hold one record under this id. +export const SINGLETON = 'all'; + +// Maps whose items have an order carry it as one more item: the ids in order. Items it does not +// list (added on another computer at the same time) follow in id order, so nothing is lost when +// two orders race and the newer one wins. +export const ORDER = '#order'; + +type JsonObject = { readonly [key: string]: Json }; + +export const isObject = (value: Json | undefined): value is JsonObject => value !== null && typeof value === 'object' && !Array.isArray(value); + +// What a store holds, as plain JSON: undefined fields go, as they would through storage. +export const toJson = (value: unknown): Json => JSON.parse(JSON.stringify(value)) as Json; + +function fail(message: string): never { + throw new Error(message); +} + +export function orderedMap(items: readonly T[], value: (item: T) => unknown = (item) => item): JsonObject { + return toJson({ ...Object.fromEntries(items.map((item) => [item.id, value(item)])), [ORDER]: items.map((item) => item.id) }) as JsonObject; +} + +export function orderedItems(map: JsonObject): [string, Json][] { + const order = Array.isArray(map[ORDER]) ? (map[ORDER] as readonly Json[]).filter((id): id is string => typeof id === 'string' && id !== ORDER && id in map) : []; + const listed = [...new Set(order)]; + const rest = Object.keys(map).filter((id) => id !== ORDER && !listed.includes(id)).sort(); + return [...listed, ...rest].map((id) => [id, map[id]]); +} + +function checkOrder(map: JsonObject): void { + const order = map[ORDER]; + if (order !== undefined && (!Array.isArray(order) || order.some((id) => typeof id !== 'string'))) fail('The order is not a list of ids.'); +} + +// Profile and template names are their record ids, so they must already be valid names. +function presetId(id: string): string { + const isValid = (() => { try { return presetName(id) === id; } catch { return false; } })(); + return isValid ? id : fail(`"${id}" is not a valid name.`); +} + +function restoreDrawingItem(id: string, value: Json): Drawing { + const drawing = migrateDrawing(value); + if (!drawing || drawing.id !== id) fail(`Drawing ${id} is not readable.`); + return drawing; +} + +// A drawings map in drawing order; throws when any drawing does not read. +export function restoreDrawingMap(value: Json): Drawing[] { + if (!isObject(value)) fail('Drawings must be a map.'); + checkOrder(value); + return orderedItems(value).map(([id, item]) => restoreDrawingItem(id, item)); +} + +export const drawingMap = (drawings: readonly Drawing[]) => orderedMap(drawings); + +// Charts: everything but the id and drawings, which sync apart. Optional fields are written as +// null, because a field missing from a record means "unchanged", not "removed". +export function chartRecord(chart: ChartConfig): Json { + const { id, drawings, settings, theme, linkGroup, ...fields } = chart; + return toJson({ ...fields, settings: settings ?? null, theme: theme ?? null, linkGroup: linkGroup ?? null }); +} + +export function restoreChartRecord(id: string, value: Json, drawings: Drawing[]): ChartConfig { + if (!isObject(value)) fail('A chart must be an object.'); + const fields = Object.fromEntries(Object.entries(value).filter(([, field]) => field !== null)); + return restoreChart({ ...fields, id, drawings }) ?? fail(`Chart ${id} is not readable.`); +} + +// Layouts: the tab's own fields. Its charts, their order and its shared drawings sync apart, and +// which chart is active stays with each computer. +export function layoutRecord(tab: ChartTab): Json { + return toJson({ layout: tab.layout, market: tab.market, syncs: tab.syncs, sizes: tab.sizes }); +} + +export function restoreLayoutRecord(value: Json): Pick { + if (!isObject(value) || !isLayoutKey(value.layout)) fail('A layout needs a known layout.'); + const market = markets.find((candidate) => candidate.key === value.market)?.key ?? fail('A layout needs a known market.'); + const syncs = Array.isArray(value.syncs) ? value.syncs : fail('Layout syncs must be a list.'); + if (syncs.some((sync) => !layoutSyncs.some((candidate) => candidate.key === sync))) fail('A layout sync is unknown.'); + if (!isObject(value.sizes)) fail('Layout sizes must be a map.'); + const sizes = Object.fromEntries(Object.entries(value.sizes).filter(([key]) => isLayoutKey(key)).map(([key, shares]) => [key, restoreSizes(shares, layoutOf(key as LayoutKey).tree)])) as Partial>; + return { layout: value.layout as LayoutKey, market, syncs: syncs as ChartTab['syncs'], sizes }; +} + +export function restoreIdList(value: Json): string[] { + if (!Array.isArray(value) || value.some((id) => typeof id !== 'string')) fail('Expected a list of ids.'); + return [...value] as string[]; +} + +// Drawings shared across a layout, by instrument: one map item per drawing, carrying its instrument. +export function layoutDrawingsRecord(tab: ChartTab): Json { + const items = Object.entries(tab.drawings).flatMap(([symbol, drawings]) => drawings.map((drawing) => ({ id: drawing.id, symbol, drawing }))); + return orderedMap(items, ({ symbol, drawing }) => ({ symbol, drawing })); +} + +export function restoreLayoutDrawings(value: Json): Record { + if (!isObject(value)) fail('Layout drawings must be a map.'); + checkOrder(value); + const bySymbol: Record = {}; + for (const [id, item] of orderedItems(value)) { + if (!isObject(item) || typeof item.symbol !== 'string' || !item.symbol) fail(`Layout drawing ${id} has no instrument.`); + (bySymbol[item.symbol] ??= []).push(restoreDrawingItem(id, item.drawing)); + } + return bySymbol; +} + +export function restoreProfile(name: string, value: Json): ChartProfile { + if (!isObject(value)) fail('A profile must be an object.'); + const [profile] = restoreProfiles([{ ...value, name }]); + return profile ?? fail(`Profile ${name} is not readable.`); +} + +export const profileRecord = ({ name, ...profile }: ChartProfile): Json => toJson(profile); + +export function restoreTemplate(name: string, value: Json): ChartTemplate { + if (!isObject(value)) fail('A template must be an object.'); + const [template] = restoreTemplates([{ name, settings: value.settings }]); + return template ?? fail(`Template ${name} is not readable.`); +} + +export const templateRecord = (template: ChartTemplate): Json => toJson({ settings: template.settings }); + +const workspacePreferenceChecks: Readonly boolean>> = { + theme: (value) => typeof value === 'string' && !!value, + drawingSync: (value) => drawingSyncs.some((candidate) => candidate.key === value), + isCrosshairSyncedByDefault: (value) => typeof value === 'boolean', + replayMode: (value) => replayModes.some((candidate) => candidate.key === value), + replaySpeed: (value) => replaySpeeds.some((candidate) => candidate.key === value), + defaultChartSettings: (value) => value === null || isObject(value), + defaultChartTheme: (value) => value === null || typeof value === 'string', + activeProfile: (value) => typeof value === 'string' && (() => { try { return presetId(value) === value; } catch { return false; } })(), +}; + +// The workspace settings that follow the user rather than the screen. Panels, docks, the page and +// the one-click consent stay with each computer. +export function workspacePreferences(state: WorkspaceState): Json { + return toJson({ + theme: state.theme, + drawingSync: state.drawingSync, + isCrosshairSyncedByDefault: state.isCrosshairSyncedByDefault, + replayMode: state.replayMode, + replaySpeed: state.replaySpeed, + defaultChartSettings: state.defaultChartSettings ?? null, + defaultChartTheme: state.defaultChartTheme ?? null, + activeProfile: state.activeProfile, + }); +} + +// Fields a newer build added are left alone; the known ones must hold valid values. +export function checkWorkspacePreferences(value: Json): void { + if (!isObject(value)) fail('Preferences must be an object.'); + for (const [key, field] of Object.entries(value)) if (workspacePreferenceChecks[key] && !workspacePreferenceChecks[key](field)) fail(`The ${key} preference is invalid.`); +} + +function applyWorkspacePreferences(state: WorkspaceState, value: JsonObject): Partial { + const patch: Partial = {}; + const known = (key: string) => key in value && !!workspacePreferenceChecks[key]; + if (known('theme')) patch.theme = value.theme as string; + if (known('drawingSync')) patch.drawingSync = value.drawingSync as WorkspaceState['drawingSync']; + if (known('isCrosshairSyncedByDefault')) patch.isCrosshairSyncedByDefault = value.isCrosshairSyncedByDefault as boolean; + if (known('replayMode')) patch.replayMode = value.replayMode as WorkspaceState['replayMode']; + if (known('replaySpeed')) patch.replaySpeed = value.replaySpeed as WorkspaceState['replaySpeed']; + if (known('defaultChartSettings')) patch.defaultChartSettings = value.defaultChartSettings === null ? undefined : restoreSettings(value.defaultChartSettings); + if (known('defaultChartTheme')) patch.defaultChartTheme = value.defaultChartTheme === null ? undefined : value.defaultChartTheme as string; + // The layouts of the other profile arrive as their own records; a profile this computer does not + // have yet keeps the current one. + if (known('activeProfile') && state.profiles.some((profile) => profile.name === value.activeProfile)) patch.activeProfile = value.activeProfile as string; + return patch; +} + +export const workspaceKinds = ['chart', 'chartDrawings', 'layout', 'layoutCharts', 'layoutDrawings', 'layoutOrder', 'chartProfile', 'chartTemplate'] as const; + +export type WorkspaceKind = (typeof workspaceKinds)[number]; + +export function workspaceRecords(state: WorkspaceState, kind: WorkspaceKind): Records { + const byId = (items: readonly T[], record: (item: T) => Json) => Object.fromEntries(items.map((item) => [item.id, record(item)])); + if (kind === 'chart') return byId(state.charts, chartRecord); + if (kind === 'chartDrawings') return byId(state.charts, (chart) => drawingMap(chart.drawings)); + if (kind === 'layout') return byId(state.tabs, layoutRecord); + if (kind === 'layoutCharts') return byId(state.tabs, (tab) => toJson(tab.chartIds)); + if (kind === 'layoutDrawings') return byId(state.tabs, layoutDrawingsRecord); + if (kind === 'layoutOrder') return Object.fromEntries(markets.map(({ key }) => [key, state.tabs.filter((tab) => tab.market === key).map((tab) => tab.id)])); + if (kind === 'chartProfile') return Object.fromEntries(state.profiles.map((profile) => [profile.name, profileRecord(profile)])); + return Object.fromEntries(state.templates.map((template) => [template.name, templateRecord(template)])); +} + +export function checkWorkspaceRecord(kind: WorkspaceKind, id: string, value: Json): void { + if (kind === 'chart') restoreChartRecord(id, value, []); + else if (kind === 'chartDrawings') restoreDrawingMap(value); + else if (kind === 'layout') restoreLayoutRecord(value); + else if (kind === 'layoutCharts' || kind === 'layoutOrder') restoreIdList(value); + else if (kind === 'layoutDrawings') restoreLayoutDrawings(value); + else if (kind === 'chartProfile') restoreProfile(presetId(id), value); + else restoreTemplate(presetId(id), value); + if (kind === 'layoutOrder' && !markets.some((market) => market.key === id)) fail(`Unknown market ${id}.`); +} + +// Tabs of `market` in the order given, in the places that market's tabs held; tabs the order does +// not list keep their order after the listed ones. +function reorderTabs(tabs: readonly ChartTab[], market: Market, order: readonly string[]): ChartTab[] { + const rank = (tab: ChartTab) => { const index = order.indexOf(tab.id); return index < 0 ? Infinity : index; }; + const sorted = tabs.filter((tab) => tab.market === market).sort((left, right) => rank(left) - rank(right)); + return tabs.map((tab) => tab.market === market ? sorted.shift()! : tab); +} + +// The state after a change from sync, for one kind. `markSynced` tags drawing lists that came from +// another computer, so the chart does not offer them for undo. +export function applyWorkspaceChange(state: WorkspaceState, kind: WorkspaceKind, change: SyncSourceChange, markSynced: (drawings: Drawing[]) => Drawing[] = (drawings) => drawings): Partial { + const deleted = new Set(change.deletes); + const upserts = Object.entries(change.upserts); + const patchTabs = (patch: (tab: ChartTab) => ChartTab) => state.tabs.map(patch); + if (kind === 'chart') { + const charts = state.charts.filter((chart) => !deleted.has(chart.id)); + for (const [id, value] of upserts) { + const index = charts.findIndex((chart) => chart.id === id); + const chart = restoreChartRecord(id, value, index < 0 ? [] : charts[index].drawings); + if (index < 0) charts.push(chart); + else charts[index] = chart; + } + const tabs = deleted.size ? patchTabs((tab) => tab.chartIds.some((id) => deleted.has(id)) ? { ...tab, chartIds: tab.chartIds.filter((id) => !deleted.has(id)) } : tab) : state.tabs; + return { charts, tabs }; + } + if (kind === 'chartDrawings') { + const values = new Map(upserts); + return { + charts: state.charts.map((chart) => { + if (values.has(chart.id)) return { ...chart, drawings: markSynced(restoreDrawingMap(values.get(chart.id)!)) }; + return deleted.has(chart.id) && chart.drawings.length ? { ...chart, drawings: markSynced([]) } : chart; + }), + }; + } + if (kind === 'layout') { + const tabs = state.tabs.filter((tab) => !deleted.has(tab.id)); + for (const [id, value] of upserts) { + const fields = restoreLayoutRecord(value); + const index = tabs.findIndex((tab) => tab.id === id); + if (index < 0) tabs.push({ ...makeTab(id, [], fields.layout, fields.syncs, fields.market), sizes: fields.sizes }); + else tabs[index] = { ...tabs[index], ...fields }; + } + return { tabs }; + } + if (kind === 'layoutCharts') { + const values = new Map(upserts); + return { tabs: patchTabs((tab) => values.has(tab.id) ? { ...tab, chartIds: restoreIdList(values.get(tab.id)!) } : tab) }; + } + if (kind === 'layoutDrawings') { + const values = new Map(upserts); + return { tabs: patchTabs((tab) => values.has(tab.id) ? { ...tab, drawings: restoreLayoutDrawings(values.get(tab.id)!) } : deleted.has(tab.id) ? { ...tab, drawings: {} } : tab) }; + } + if (kind === 'layoutOrder') { + let tabs = [...state.tabs]; + for (const [market, value] of upserts) tabs = reorderTabs(tabs, market as Market, restoreIdList(value)); + return { tabs }; + } + if (kind === 'chartProfile') { + // The profile in use is the live workspace, so it is never removed from under it. + const profiles = state.profiles.filter((profile) => !deleted.has(profile.name) || profile.name === state.activeProfile); + for (const [name, value] of upserts) { + const profile = restoreProfile(name, value); + const index = profiles.findIndex((candidate) => candidate.name === name); + if (index < 0) profiles.push(profile); + else profiles[index] = profile; + } + return { profiles }; + } + const templates = state.templates.filter((template) => !deleted.has(template.name)); + for (const [name, value] of upserts) { + const template = restoreTemplate(name, value); + const index = templates.findIndex((candidate) => candidate.name === name); + if (index < 0) templates.push(template); + else templates[index] = template; + } + return { templates }; +} + +export function applyPreferencesChange(state: WorkspaceState, value: Json): Partial { + checkWorkspacePreferences(value); + return applyWorkspacePreferences(state, value as JsonObject); +} diff --git a/src/lib/sync-sources.ts b/src/lib/sync-sources.ts new file mode 100644 index 0000000..0f7298f --- /dev/null +++ b/src/lib/sync-sources.ts @@ -0,0 +1,290 @@ +import { customColorList, drawingToolInfo, onCustomColorsChange, saveDrawingFavorites, setCustomColors, type DrawingTemplate } from '@meridian/charts'; +import { parseRiskPreferences } from '../../shared/automation'; +import { markets, type Market } from '../../shared/protocol'; +import type { Json, RendererSyncKind, SyncSourceChange } from '../../shared/sync'; +import { automation } from './automation'; +import { importedThemes, removeImportedTheme, saveImportedTheme } from './custom-themes'; +import { drawingTemplates, restoreDrawingLooks, restoreDrawingTemplates } from './drawing-templates'; +import { drawingTools } from './drawing-tools'; +import { indicatorFavorites } from './indicator-favorites'; +import { isChartTimeframe, type ChartTimeframe } from './mt5-timeframes'; +import { pineSourceProblem } from './pine/pine-script'; +import { pineScripts, pineScriptSource, pineScriptsReady, putSyncedPineScript, removePineScripts, restorePineScript } from './pine/pine-scripts'; +import { priceAlerts, updateAlerts } from './price-alerts'; +import { restoreAlerts } from './price-alerts-model'; +import { riskPreferences, riskTicketStorageKeys } from './risk-preferences'; +import type { Store } from './store'; +import { symbolDrawings } from './symbol-drawings'; +import { + applyPreferencesChange, applyWorkspaceChange, checkWorkspacePreferences, checkWorkspaceRecord, drawingMap, isObject, orderedItems, orderedMap, ORDER, restoreDrawingMap, + SINGLETON, toJson, workspacePreferences, workspaceRecords, type Records, type WorkspaceKind, +} from './sync-model'; +import { timeframeFavorites } from './timeframe-favorites'; +import { assistantPreferences, restoreAssistantPreferences } from './trade-assistant'; +import { restoreThemePatches } from './tradingview-model'; +import { restoreWatchlists, type WatchlistsState } from './watchlist-model'; +import { savedWatchlists, setSavedWatchlists, watchlists } from './watchlists'; +import { markSyncedDrawings, workspace } from './workspace'; + +// The renderer's side of sync: each kind's records read from and written to its store. The main +// process holds the keys and the engine, and asks for these over IPC. + +export interface RendererSource { + readonly snapshot: () => Records | Promise; + // Throws when the record must not be applied under that id. + readonly validate: (value: Json, id: string) => void; + readonly apply: (change: SyncSourceChange) => void | Promise; +} + +function fail(message: string): never { + throw new Error(message); +} + +const workspaceSource = (kind: WorkspaceKind): RendererSource => ({ + snapshot: () => workspaceRecords(workspace.get(), kind), + validate: (value, id) => checkWorkspaceRecord(kind, id, value), + apply: (change) => workspace.set((state) => applyWorkspaceChange(state, kind, change, markSyncedDrawings)), +}); + +// The singleton's new value, [] when it was deleted, or null when the change leaves it alone. +function singletonValue(change: SyncSourceChange): Json | null { + if (SINGLETON in change.upserts) return change.upserts[SINGLETON]; + return change.deletes.includes(SINGLETON) ? [] : null; +} + +function checkSingleton(id: string): void { + if (id !== SINGLETON) fail(`Unknown record ${id}.`); +} + +// Favourites keep the order they were added in here; ones from elsewhere go at the end. +function setSource(read: () => readonly string[], write: (items: string[]) => void, isMember: (item: string) => boolean): RendererSource { + return { + snapshot: () => ({ [SINGLETON]: [...read()] }), + validate: (value, id) => { + checkSingleton(id); + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string' || !isMember(item))) fail('Favourites must be known names.'); + }, + apply: (change) => { + const value = singletonValue(change); + if (value === null) return; + const next = value as string[]; + const current = read(); + write([...current.filter((item) => next.includes(item)), ...next.filter((item) => !current.includes(item))]); + }, + }; +} + +// Drawing templates are one map item per tool and name. +function templatesFromMap(value: Json): Record { + if (!isObject(value)) fail('Drawing templates must be a map.'); + const byType: Record = {}; + for (const [key, item] of Object.entries(value)) { + const split = key.indexOf('/'); + const [type, name] = [key.slice(0, split), key.slice(split + 1)]; + const restored = split > 0 ? restoreDrawingTemplates({ [type]: [item] })[type]?.[0] : undefined; + if (!restored || restored.name !== name) fail(`Drawing template ${key} is not readable.`); + (byType[type] ??= []).push(restored); + } + for (const list of Object.values(byType)) list.sort((left, right) => left.name.localeCompare(right.name)); + return byType; +} + +function looksFromMap(value: Json): Record { + if (!isObject(value)) fail('Drawing looks must be a map.'); + const looks = restoreDrawingLooks(value); + if (Object.keys(looks).length !== Object.keys(value).length) fail('A drawing look is not readable.'); + return looks; +} + +function watchlistsFromMap(market: string, value: Json, activeId: string): WatchlistsState { + if (!markets.some((candidate) => candidate.key === market)) fail(`Unknown market ${market}.`); + if (!isObject(value) || (value[ORDER] !== undefined && !Array.isArray(value[ORDER]))) fail('Watchlists must be a map.'); + const lists = orderedItems(value).map(([id, list]) => ({ ...(isObject(list) ? list : {}), id })); + const restored = restoreWatchlists({ activeId, lists }); + const isWhole = restored?.lists.length === lists.length && restored.lists.every((list, index) => list.entries.length === (lists[index] as { entries?: unknown[] }).entries?.length); + return isWhole ? restored! : fail('A watchlist is not readable.'); +} + +const conflictPattern = /~conflict~[^~]*~(\d+)$/; + +function checkPineScript(value: Json, id: string): void { + if (!isObject(value) || typeof value.source !== 'string' || pineSourceProblem(value.source) || !restorePineScript({ ...value, id })) fail(`Pine script ${id} is not readable.`); +} + +const riskTicketIds = markets.map(({ key }) => `riskTicket:${key}`); + +function readRiskTicket(market: Market): Json | undefined { + try { + const value = JSON.parse(localStorage.getItem(riskTicketStorageKeys[market]) ?? 'null') as Json; + return isObject(value) ? toJson(value) : undefined; + } catch { + return undefined; + } +} + +export const syncSources: Readonly> = { + theme: { + snapshot: () => Object.fromEntries(restoreThemePatches(importedThemes.get().themes).map((theme) => [theme.id, toJson(theme)])), + validate: (value, id) => { + if (restoreThemePatches([value])[0]?.id !== id) fail(`Theme ${id} is not readable.`); + }, + apply: (change) => { + change.deletes.forEach(removeImportedTheme); + for (const value of Object.values(change.upserts)) saveImportedTheme(restoreThemePatches([value])[0]); + }, + }, + pineScript: { + // A script whose source is missing is still listed, so it reads as invalid rather than deleted. + snapshot: async () => { + await pineScriptsReady; + const entries = await Promise.all(pineScripts.get().scripts.map(async ({ id, ...script }) => { + const source = await pineScriptSource(id).catch(() => null); + return [id, toJson(source === null ? script : { ...script, source })] as const; + })); + return Object.fromEntries(entries); + }, + validate: checkPineScript, + // Both computers edited a script: the copy kept from this one is named so it can be told apart. + apply: async (change) => { + if (change.deletes.length) removePineScripts(change.deletes); + for (const [id, value] of Object.entries(change.upserts)) { + checkPineScript(value, id); + const { source, ...fields } = value as { source: string }; + const script = restorePineScript({ ...fields, id })!; + const conflict = conflictPattern.exec(id); + const name = conflict && !script.name.includes('(conflict') ? `${script.name} (conflict, ${new Date(Number(conflict[1])).toISOString().slice(0, 10)})` : script.name; + await putSyncedPineScript({ ...script, name }, source); + } + }, + }, + chart: workspaceSource('chart'), + chartDrawings: workspaceSource('chartDrawings'), + layout: workspaceSource('layout'), + layoutCharts: workspaceSource('layoutCharts'), + layoutDrawings: workspaceSource('layoutDrawings'), + layoutOrder: workspaceSource('layoutOrder'), + symbolDrawings: { + snapshot: () => Object.fromEntries(Object.entries(symbolDrawings.get().bySymbol).map(([key, drawings]) => [key, drawingMap(drawings)])), + validate: (value, id) => { + if (!id) fail('Symbol drawings need an instrument.'); + restoreDrawingMap(value); + }, + apply: (change) => symbolDrawings.set((state) => { + const bySymbol = Object.fromEntries(Object.entries(state.bySymbol).filter(([key]) => !change.deletes.includes(key))); + for (const [key, value] of Object.entries(change.upserts)) { + const drawings = restoreDrawingMap(value); + if (drawings.length) bySymbol[key] = drawings; + else delete bySymbol[key]; + } + return { bySymbol }; + }), + }, + chartProfile: workspaceSource('chartProfile'), + chartTemplate: workspaceSource('chartTemplate'), + drawingTemplates: { + snapshot: () => ({ [SINGLETON]: toJson(Object.fromEntries(Object.entries(drawingTemplates.get().byType).flatMap(([type, list]) => list.map((template) => [`${type}/${template.name}`, template])))) }), + validate: (value, id) => { + checkSingleton(id); + templatesFromMap(value); + }, + apply: (change) => { + const value = singletonValue(change); + if (value !== null) drawingTemplates.set({ byType: Array.isArray(value) ? {} : templatesFromMap(value) }); + }, + }, + drawingLooks: { + snapshot: () => ({ [SINGLETON]: toJson(drawingTemplates.get().lastByType) }), + validate: (value, id) => { + checkSingleton(id); + looksFromMap(value); + }, + apply: (change) => { + const value = singletonValue(change); + if (value !== null) drawingTemplates.set({ lastByType: Array.isArray(value) ? {} : looksFromMap(value) }); + }, + }, + watchlists: { + snapshot: () => Object.fromEntries(markets.flatMap(({ key }) => { + const state = savedWatchlists(key); + return state ? [[key, orderedMap(state.lists, ({ name, entries }) => ({ name, entries }))]] : []; + })), + validate: (value, id) => void watchlistsFromMap(id, value, ''), + // Which list is open stays with each computer. A market's lists are never removed as a whole. + apply: (change) => { + for (const [market, value] of Object.entries(change.upserts)) { + setSavedWatchlists(market as Market, watchlistsFromMap(market, value, savedWatchlists(market as Market)?.activeId ?? '')); + } + }, + }, + indicatorFavorites: setSource(() => indicatorFavorites.get().types, (types) => indicatorFavorites.set({ types }), (type) => type.length > 0), + timeframeFavorites: setSource(() => timeframeFavorites.get().labels, (labels) => timeframeFavorites.set({ labels: labels as ChartTimeframe[] }), isChartTimeframe), + drawingFavorites: setSource(() => drawingTools.get().favorites, (favorites) => { + saveDrawingFavorites(favorites); + drawingTools.set({ favorites }); + }, (type) => drawingToolInfo.has(type)), + customColors: { + snapshot: () => ({ [SINGLETON]: [...customColorList()] }), + validate: (value, id) => { + checkSingleton(id); + if (!Array.isArray(value) || value.length > 10 || value.some((color) => typeof color !== 'string' || !/^#[0-9a-f]{6}$/i.test(color))) fail('Custom colours must be up to ten #rrggbb colours.'); + }, + apply: (change) => { + const value = singletonValue(change); + if (value !== null) setCustomColors(value as string[]); + }, + }, + priceAlert: { + snapshot: () => Object.fromEntries(priceAlerts.get().alerts.map(({ id, ...alert }) => [id, toJson(alert)])), + validate: (value, id) => { + if (!isObject(value) || restoreAlerts([{ ...value, id }]).length !== 1) fail(`Alert ${id} is not readable.`); + }, + apply: (change) => { + const alerts = priceAlerts.get().alerts.filter((alert) => !change.deletes.includes(alert.id)); + for (const [id, value] of Object.entries(change.upserts)) { + const [alert] = restoreAlerts([{ ...(value as object), id }]); + const index = alerts.findIndex((candidate) => candidate.id === id); + if (index < 0) alerts.push(alert); + else alerts[index] = alert; + } + updateAlerts(alerts); + }, + }, + // Preferences are never removed as a whole; a deletion leaves this computer's as they are. + preferences: { + snapshot: async () => { + const records: Record = { + workspace: workspacePreferences(workspace.get()), + tradeAssistant: toJson(assistantPreferences.get()), + risk: toJson(await automation.request('preferences.risk', {})), + }; + for (const { key } of markets) { + const ticket = readRiskTicket(key); + if (ticket !== undefined) records[`riskTicket:${key}`] = ticket; + } + return records; + }, + validate: (value, id) => { + if (!isObject(value)) fail('Preferences must be an object.'); + if (id === 'workspace') checkWorkspacePreferences(value); + else if (id !== 'tradeAssistant' && id !== 'risk' && !riskTicketIds.includes(id)) fail(`Unknown preferences ${id}.`); + }, + apply: async (change) => { + for (const [id, value] of Object.entries(change.upserts)) { + if (id === 'workspace') workspace.set((state) => applyPreferencesChange(state, value)); + else if (id === 'tradeAssistant') assistantPreferences.set(restoreAssistantPreferences({ ...assistantPreferences.get(), ...(value as object) })); + else if (id === 'risk') riskPreferences.set(await automation.request('preferences.setRisk', parseRiskPreferences({ ...riskPreferences.get(), ...(value as object) }))); + else { + const market = markets.find(({ key }) => id === `riskTicket:${key}`)?.key; + if (market) localStorage.setItem(riskTicketStorageKeys[market], JSON.stringify({ ...(readRiskTicket(market) as object | undefined), ...(value as object) })); + } + } + }, + }, +}; + +// Every store sync reads, for telling the main process an edit happened. The Risk Ticket saves its +// settings itself, so they are found at the next sync instead. +export const syncedStores: readonly Pick, 'subscribe'>[] = [ + workspace, symbolDrawings, drawingTemplates, importedThemes, watchlists, indicatorFavorites, timeframeFavorites, drawingTools, priceAlerts, pineScripts, assistantPreferences, riskPreferences, + { subscribe: onCustomColorsChange }, +]; diff --git a/src/lib/sync.ts b/src/lib/sync.ts index 5e0f2be..41ee8bb 100644 --- a/src/lib/sync.ts +++ b/src/lib/sync.ts @@ -1,5 +1,9 @@ -import type { SyncApi, SyncMethod, SyncMethods, SyncView } from '../../shared/sync'; +import type { SyncApi, SyncMethod, SyncMethods, SyncSnapshot, SyncSourceRequest, SyncView } from '../../shared/sync'; import { Store } from './store'; +import { syncedStores, syncSources } from './sync-sources'; +import { toJson } from './sync-model'; + +const CHANGE_NOTICE_MS = 250; declare global { interface Window { @@ -11,9 +15,38 @@ export const sync = window.sync; export const syncStore = new Store<{ view: SyncView | null }>({ view: null }); +// Store changes made by applying synced records are not news to the main process. +let isApplying = false; + +async function serve(request: SyncSourceRequest): Promise { + const source = syncSources[request.kind]; + if (request.method === 'validate') return source.validate(request.value, request.id) ?? null; + if (request.method === 'snapshot') { + const values = toJson(await source.snapshot()) as SyncSnapshot['values']; + const invalid = Object.entries(values).flatMap(([id, value]) => { + try { source.validate(value, id); return []; } catch { return [id]; } + }); + return { values, invalid } satisfies SyncSnapshot; + } + isApplying = true; + try { await source.apply(request.change); } + finally { isApplying = false; } + return null; +} + export function startSync(): void { sync.on('view', (view) => syncStore.set({ view })); void sync.request('sync.view', {}).then((view) => syncStore.set({ view })).catch(() => undefined); + let noticeTimer: ReturnType | undefined; + const notice = () => { + if (isApplying || noticeTimer || syncStore.get().view?.phase !== 'on') return; + noticeTimer = setTimeout(() => { + noticeTimer = undefined; + sync.changed(); + }, CHANGE_NOTICE_MS); + }; + syncedStores.forEach((store) => store.subscribe(notice)); + sync.serve(serve); } // Errors are returned for the dialog to show next to what was tried. diff --git a/src/lib/watchlists.ts b/src/lib/watchlists.ts index 2041e25..f255f89 100644 --- a/src/lib/watchlists.ts +++ b/src/lib/watchlists.ts @@ -33,6 +33,23 @@ let market: Market = 'fx'; export const watchlists = new Store(load(market)); +// A market's lists as saved, for sync: the market in use from the store, another from storage, and +// null for a market whose lists were never saved here (its default list has a new id each load). +export function savedWatchlists(of: Market): WatchlistsState | null { + if (of === market) return watchlists.get(); + try { + const saved = localStorage.getItem(storageKeys[of]); + return saved ? restoreWatchlists(JSON.parse(saved)) : null; + } catch { + return null; + } +} + +export function setSavedWatchlists(of: Market, state: WatchlistsState): void { + if (of === market) watchlists.set(state); + else localStorage.setItem(storageKeys[of], JSON.stringify(state)); +} + // The platform of the account in use; null while logged out or while the account changes. export function activePlatform(): Platform | null { const { accounts, profileId } = session.get(); diff --git a/src/lib/workspace.ts b/src/lib/workspace.ts index ed55ce8..440105f 100644 --- a/src/lib/workspace.ts +++ b/src/lib/workspace.ts @@ -280,6 +280,16 @@ export function activeChart(state: WorkspaceState): ChartConfig | undefined { const noDrawings: readonly Drawing[] = []; +// Drawing lists that arrived from another computer: the chart shows them without offering an undo. +const syncedDrawingLists = new WeakSet(); + +export function markSyncedDrawings(drawings: Drawing[]): Drawing[] { + syncedDrawingLists.add(drawings); + return drawings; +} + +export const isSyncedDrawings = (drawings: readonly Drawing[]) => syncedDrawingLists.has(drawings); + // Drawings shared across the chart's tab for the chart's symbol. export function layoutDrawings(state: WorkspaceState, chart: ChartConfig): readonly Drawing[] { return tabOfChart(state, chart.id)?.drawings[instrumentKey(chart.symbol)] ?? noDrawings; From 7b9a788881770377bf11a03fac4b134c85da09fb Mon Sep 17 00:00:00 2001 From: Aaron Rono Date: Fri, 6 Nov 2026 07:38:32 +0300 Subject: [PATCH 2/2] feat: sync accounts, passwords and the CID; hand logins over between computers Phase 4 of sync, after a security review of Phases 1-3. Accounts and secrets: - Saved account profiles always sync. Passwords, API keys and webhook secrets sync only on computers that opt in and have a system keychain; the most recently saved password wins (secretSavedAt), and a secret that no longer decrypts is skipped, never read as deleted. - "Use one CID on all my computers" syncs the CID; turning it on asks which CID to keep when the synced one differs. - Webhook sources and notification targets sync; whether each source is switched on and the listener settings stay per computer. Presence and login handoff: - Computers list the accounts they hold in the encrypted manifest (30-minute presence, renewed at 15, cleared on logout and quit). - An MT5 login to an account open elsewhere asks Move it here / Log in anyway / Cancel. Move writes a request the other computer honours only if its session predates it, so an asleep computer logs out on waking but a later deliberate login is never ended. Review fixes: - Merged values are validated before apply; a failing merge keeps the remote value instead of failing every sync. - The unchanged-head shortcut re-pulls while any record lacks a base, so a skipped record is never overwritten by local data. - __proto__ refused as ids and keys; own-property lookups in merges. - Malformed renderer replies reject; snapshots must be plain JSON; secret kinds can never be served by the renderer. - The join question survives a restart, and a first merge prefers the synced value so a fresh install's defaults never replace settings. - Closing the window on macOS no longer stops sync; a loading window shows as waiting instead of Synced. --- CHANGELOG.md | 7 +- electron/account-registry.test.ts | 53 ++++++++ electron/account-registry.ts | 144 ++++++++++++++++++--- electron/accounts.ts | 4 +- electron/automation-host.ts | 64 ++++++++-- electron/device-identity.ts | 18 +++ electron/github-remote.ts | 4 + electron/main.ts | 128 ++++++++++++++----- electron/sync-engine.test.ts | 72 +++++++++++ electron/sync-engine.ts | 104 +++++++++++++-- electron/sync-main-sources.test.ts | 77 +++++++++++ electron/sync-main-sources.ts | 90 +++++++++++++ electron/sync-manifest.test.ts | 7 +- electron/sync-manifest.ts | 40 +++++- electron/sync-merge.test.ts | 5 + electron/sync-merge.ts | 39 +++++- electron/sync-renderer.test.ts | 16 ++- electron/sync-renderer.ts | 25 +++- electron/sync-service.test.ts | 112 +++++++++++++++- electron/sync-service.ts | 199 +++++++++++++++++++++++++++-- electron/sync-storage.ts | 25 +++- shared/sync.ts | 16 +++ src/components/sync/SyncDialog.tsx | 40 +++++- src/components/sync/SyncStatus.tsx | 2 +- src/lib/sync-model.ts | 9 +- 25 files changed, 1195 insertions(+), 105 deletions(-) create mode 100644 electron/sync-main-sources.test.ts create mode 100644 electron/sync-main-sources.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b507657..992a9ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,12 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). price alerts, Pine scripts, and the theme, replay, Trade Assistant, Risk Ticket and risk settings. Panels, docks, which layout and chart are in front, and one-click trading consent stay with each computer. A computer joining data that already syncs asks whether to merge its own layouts in or - use the synced ones. + use the synced ones; either way your synced settings are kept. +- **Saved accounts sync too.** In Sync Across Computers, **Passwords and identity** lets each + computer also sync saved passwords, API keys and webhook secrets (the newest saved password wins), + and **Use one CID on all my computers**, which asks which CID to keep if they differ. +- **Logging in to an account that is open on another computer asks first**, and **Move it here** + has that computer log out of it, so the two sessions do not keep disconnecting each other. ## [1.0.0] - 2026-11-04 diff --git a/electron/account-registry.test.ts b/electron/account-registry.test.ts index e8719d3..753aee3 100644 --- a/electron/account-registry.test.ts +++ b/electron/account-registry.test.ts @@ -99,3 +99,56 @@ test('profiles saved before platforms existed load as MetaTrader 5 accounts', as assert.equal(profile.platform, 'mt5'); assert.deepEqual(restarted.credentials(profile.id), { platform: 'mt5', login: 1234, password: 'fixture-secret', endpoint }); }); + +test('logging in again with the same saved password keeps when it was saved', async () => { + let stored: readonly { secretSavedAt?: number }[] = []; + const registry = new AccountRegistry(cipher, async (profiles) => { stored = profiles as typeof stored; }); + await registry.remember(account, endpoint, 'first', undefined, true); + const savedAt = stored[0].secretSavedAt!; + assert.ok(savedAt > 0); + await new Promise((resolve) => setTimeout(resolve, 5)); + await registry.remember(account, endpoint, 'first', undefined, true); + assert.equal(stored[0].secretSavedAt, savedAt); + await registry.remember(account, endpoint, 'changed', undefined, true); + assert.ok(stored[0].secretSavedAt! > savedAt); +}); + +test('synced profiles keep this computer\'s secret and latest use; ids come from the fields', async () => { + const registry = new AccountRegistry(cipher, async () => {}); + const profile = await registry.remember(account, endpoint, 'local-secret', 'Mine', true); + const synced = registry.syncedProfiles(); + assert.ok(!JSON.stringify(synced).includes('encrypted')); + assert.throws(() => registry.checkSyncedProfile('wss://other.example/terminal|1234', synced[profile.id]), /does not match/); + const other = { ...synced[profile.id], label: 'Renamed elsewhere', lastUsed: 1 }; + await registry.applySyncedProfiles({ upserts: { [profile.id]: other }, deletes: [] }); + assert.equal(registry.get(profile.id).label, 'Renamed elsewhere'); + assert.equal(registry.get(profile.id).lastUsed, profile.lastUsed); + assert.equal(registry.password(profile.id), 'local-secret'); + await registry.applySyncedProfiles({ upserts: {}, deletes: [profile.id] }); + assert.equal(registry.list().length, 0); +}); + +test('the newest saved secret wins, and one that no longer decrypts is not read as removed', async () => { + const registry = new AccountRegistry(cipher, async () => {}); + const profile = await registry.remember(account, endpoint, 'local', undefined, true); + const local = registry.syncedSecrets()[profile.id] as { secret: string; savedAt: number }; + assert.equal(local.secret, 'local'); + assert.throws(() => registry.checkSyncedSecret('wss://unknown.example/terminal|1', { secret: 'x', savedAt: 1 }), /does not have/); + await registry.applySyncedSecrets({ upserts: { [profile.id]: { secret: 'older', savedAt: local.savedAt - 1 } }, deletes: [] }); + assert.equal(registry.password(profile.id), 'local'); + await registry.applySyncedSecrets({ upserts: { [profile.id]: { secret: 'newer', savedAt: local.savedAt + 1 } }, deletes: [] }); + assert.equal(registry.password(profile.id), 'newer'); + const broken = new AccountRegistry({ ...cipher, decrypt: () => { throw new Error('keychain changed'); } }, async () => {}); + await broken.remember(account, endpoint, 'x', undefined, true); + assert.deepEqual(Object.values(broken.syncedSecrets()), [{ isUnreadable: true }]); + await registry.applySyncedSecrets({ upserts: {}, deletes: [profile.id] }); + assert.equal(registry.get(profile.id).isPasswordSaved, false); +}); + +test('a synced exchange secret must hold an API key and secret', async () => { + const registry = new AccountRegistry(cipher, async () => {}); + const bybit = { ...account, login: 4242, server: 'Bybit Demo trading', company: 'Bybit' }; + const profile = await registry.rememberExchange(bybit, { platform: 'bybit', environment: 'demo', apiKey: 'k', apiSecret: 's' }); + assert.throws(() => registry.checkSyncedSecret(profile.id, { secret: 'plain password', savedAt: 1 }), /API key/); + registry.checkSyncedSecret(profile.id, { secret: JSON.stringify({ apiKey: 'k2', apiSecret: 's2' }), savedAt: 1 }); +}); diff --git a/electron/account-registry.ts b/electron/account-registry.ts index f18e44e..10f8ec0 100644 --- a/electron/account-registry.ts +++ b/electron/account-registry.ts @@ -4,8 +4,25 @@ import type { SessionCredentials } from './session-pool'; interface StoredProfile extends Omit { readonly encryptedPassword?: string; + // When the saved secret last changed, so the newest password wins between computers. + readonly secretSavedAt?: number; } +// A profile as it syncs: everything but the secret, which syncs apart behind its own opt-in. +export type SyncedProfile = Omit; + +export interface SyncedSecret { + readonly secret: string; + readonly savedAt: number; +} + +export interface ProfileChange { + readonly upserts: Readonly>; + readonly deletes: readonly string[]; +} + +const MAX_SECRET_LENGTH = 8192; + export const exchangeAccountId = (platform: ExchangePlatform, environment: string, login: number) => `${platform}:${environment}|${login}`; export const platformOf = (accountId: string): Platform => platforms.find((item) => item.key !== 'mt5' && accountId.startsWith(`${item.key}:`))?.key ?? 'mt5'; @@ -24,28 +41,123 @@ export class AccountRegistry { load(value: unknown): void { if (!Array.isArray(value) || value.length > 1000) throw new Error('Invalid saved accounts.'); - const profiles = value.map((item): StoredProfile => { - if (!item || typeof item !== 'object') throw new Error('Invalid saved account.'); - const { login, name, label, server, company, tradeMode, lastUsed, encryptedPassword } = item; - const platform = platforms.find((candidate) => candidate.key === (item.platform ?? 'mt5'))?.key; - if (!platform) throw new Error('Invalid saved account.'); - const webEndpoint = isExchangePlatform(platform) ? this.environmentOf(platform, item.webEndpoint).restUrl : parseSettings(item).webEndpoint; - if (!Number.isSafeInteger(login) || login <= 0 || !['demo', 'real', 'contest'].includes(tradeMode) || - ![name, label, server, company].every((field) => typeof field === 'string' && field.length <= 512) || - !Number.isFinite(lastUsed) || (encryptedPassword !== undefined && typeof encryptedPassword !== 'string')) { - throw new Error('Invalid saved account.'); + const profiles = value.map((item) => this.parseProfile(item)); + if (new Set(profiles.map((profile) => profile.id)).size !== profiles.length) throw new Error('Duplicate saved accounts.'); + this.profiles = profiles; + } + + // The id is derived from the fields, never read, so a profile cannot claim another's id. + private parseProfile(item: unknown): StoredProfile { + if (!item || typeof item !== 'object' || Array.isArray(item)) throw new Error('Invalid saved account.'); + const { login, name, label, server, company, tradeMode, lastUsed, encryptedPassword, secretSavedAt, ...rest } = item as Record; + const platform = platforms.find((candidate) => candidate.key === (rest.platform ?? 'mt5'))?.key; + if (!platform) throw new Error('Invalid saved account.'); + const webEndpoint = isExchangePlatform(platform) ? this.environmentOf(platform, rest.webEndpoint).restUrl : parseSettings(item as { webEndpoint?: string }).webEndpoint; + if (!Number.isSafeInteger(login) || (login as number) <= 0 || !['demo', 'real', 'contest'].includes(tradeMode as string) || + ![name, label, server, company].every((field) => typeof field === 'string' && field.length <= 512) || + !Number.isFinite(lastUsed) || (encryptedPassword !== undefined && typeof encryptedPassword !== 'string') || + (secretSavedAt !== undefined && !(Number.isFinite(secretSavedAt) && (secretSavedAt as number) >= 0))) { + throw new Error('Invalid saved account.'); + } + const id = isExchangePlatform(platform) ? exchangeAccountId(platform, this.environmentOf(platform, webEndpoint).key, login as number) : this.key(webEndpoint, login as number); + return { + id, platform, login: login as number, name: name as string, label: label as string, server: server as string, company: company as string, webEndpoint, + tradeMode: tradeMode as AccountProfile['tradeMode'], lastUsed: lastUsed as number, + ...(encryptedPassword ? { encryptedPassword: encryptedPassword as string, ...(secretSavedAt !== undefined ? { secretSavedAt: secretSavedAt as number } : {}) } : {}), + }; + } + + // Keeps the saved time of a secret that did not change, so logging in again does not make an + // old password look like the newest one. + private secretFields(existing: StoredProfile | undefined, secret: string): Pick { + const isSame = !!existing?.encryptedPassword && this.tryDecrypt(existing.encryptedPassword) === secret; + return { encryptedPassword: this.cipher.encrypt(secret), secretSavedAt: isSame ? existing!.secretSavedAt ?? 0 : Date.now() }; + } + + private tryDecrypt(value: string): string | null { + try { return this.cipher.decrypt(value); } catch { return null; } + } + + syncedProfiles(): Record { + return Object.fromEntries(this.profiles.map(({ id, encryptedPassword, secretSavedAt, ...profile }) => [id, profile])); + } + + checkSyncedProfile(id: string, value: unknown): void { + if (this.parseProfile(value).id !== id) throw new Error('The account does not match its id.'); + } + + // Applies profiles from other computers. A profile keeps this computer's saved secret, and the + // most recent use of either side. + async applySyncedProfiles(change: ProfileChange): Promise { + const removed = new Set(change.deletes); + const profiles = this.profiles.filter((profile) => !removed.has(profile.id)); + for (const [id, value] of Object.entries(change.upserts)) { + const { encryptedPassword, secretSavedAt, ...parsed } = this.parseProfile(value); + if (parsed.id !== id) throw new Error('The account does not match its id.'); + const index = profiles.findIndex((profile) => profile.id === id); + const existing = profiles[index]; + const profile: StoredProfile = existing + ? { ...parsed, lastUsed: Math.max(parsed.lastUsed, existing.lastUsed), ...(existing.encryptedPassword ? { encryptedPassword: existing.encryptedPassword, secretSavedAt: existing.secretSavedAt } : {}) } + : parsed; + if (index < 0) profiles.push(profile); + else profiles[index] = profile; + } + await this.persist(profiles); + this.profiles = profiles; + for (const id of removed) this.passwords.delete(id); + } + + // Saved secrets in plain text, for sync to encrypt under its own key. One that no longer + // decrypts here is listed as unreadable, so it is left alone rather than read as removed. + syncedSecrets(): Record { + return Object.fromEntries(this.profiles.flatMap(({ id, encryptedPassword, secretSavedAt }) => { + if (!encryptedPassword) return []; + const secret = this.tryDecrypt(encryptedPassword); + return [[id, secret === null ? { isUnreadable: true } as const : { secret, savedAt: secretSavedAt ?? 0 }]]; + })); + } + + // A secret applies only to a profile already here (profiles sync first), and an exchange + // secret must hold an API key and secret. + checkSyncedSecret(id: string, value: unknown): void { + const { secret, savedAt } = (value ?? {}) as Partial; + const profile = this.profiles.find((candidate) => candidate.id === id); + if (!profile) throw new Error('The secret belongs to an account this computer does not have yet.'); + if (typeof secret !== 'string' || !secret || secret.length > MAX_SECRET_LENGTH || !Number.isFinite(savedAt) || savedAt! < 0) throw new Error('Invalid synced secret.'); + if (!isExchangePlatform(profile.platform)) return; + const parsed = (() => { try { return JSON.parse(secret) as unknown; } catch { return null; } })() as { apiKey?: unknown; apiSecret?: unknown } | null; + if (typeof parsed?.apiKey !== 'string' || typeof parsed.apiSecret !== 'string' || !parsed.apiKey || !parsed.apiSecret) throw new Error('Invalid synced API key.'); + } + + // A secret saved here more recently than the synced one is kept; sync then sends it out, so + // every computer ends with the newest password. + async applySyncedSecrets(change: ProfileChange): Promise { + if (!this.cipher.canEncrypt()) throw new Error('Secure storage is unavailable, so synced passwords cannot be kept here.'); + const removed = new Set(change.deletes); + const profiles = this.profiles.map((profile): StoredProfile => { + const value = change.upserts[profile.id]; + if (value !== undefined) { + this.checkSyncedSecret(profile.id, value); + const { secret, savedAt } = value as SyncedSecret; + const isLocalNewer = !!profile.encryptedPassword && (profile.secretSavedAt ?? 0) > savedAt && this.tryDecrypt(profile.encryptedPassword) !== null; + return isLocalNewer ? profile : { ...profile, encryptedPassword: this.cipher.encrypt(secret), secretSavedAt: savedAt }; } - const id = isExchangePlatform(platform) ? exchangeAccountId(platform, this.environmentOf(platform, webEndpoint).key, login) : this.key(webEndpoint, login); - return { id, platform, login, name, label, server, company, webEndpoint, tradeMode, lastUsed, ...(encryptedPassword ? { encryptedPassword } : {}) }; + if (!removed.has(profile.id) || !profile.encryptedPassword) return profile; + const { encryptedPassword, secretSavedAt, ...rest } = profile; + return rest; }); - if (new Set(profiles.map((profile) => profile.id)).size !== profiles.length) throw new Error('Duplicate saved accounts.'); + await this.persist(profiles); this.profiles = profiles; + for (const profile of profiles) { + if (change.upserts[profile.id] !== undefined && profile.encryptedPassword) this.passwords.set(profile.id, this.cipher.decrypt(profile.encryptedPassword)); + if (removed.has(profile.id)) this.passwords.delete(profile.id); + } } canSavePasswords(): boolean { return this.cipher.canEncrypt(); } list(): AccountProfile[] { - return this.profiles.map(({ encryptedPassword, ...profile }) => ({ + return this.profiles.map(({ encryptedPassword, secretSavedAt, ...profile }) => ({ ...profile, hasPassword: this.passwords.has(profile.id) || !!encryptedPassword && this.cipher.canEncrypt(), isPasswordSaved: !!encryptedPassword, })).sort((left, right) => right.lastUsed - left.lastUsed); @@ -76,7 +188,7 @@ export class AccountRegistry { id, platform: 'mt5', name: account.name, label: label?.trim() || existing?.label || `${account.company} ${account.login}`, server: account.server, company: account.company, webEndpoint, tradeMode: account.tradeMode, login: account.login, lastUsed: Date.now(), - ...(isPasswordSaved ? { encryptedPassword: this.cipher.encrypt(password) } : {}), + ...(isPasswordSaved ? this.secretFields(existing, password) : {}), }; const profiles = [...this.profiles.filter((item) => item.id !== id), profile]; await this.persist(profiles); @@ -98,7 +210,7 @@ export class AccountRegistry { id, platform, name: account.name, label: label?.trim() || existing?.label || `${account.server} ${account.login}`, server: account.server, company: account.company, webEndpoint: restUrl, tradeMode: account.tradeMode, login: account.login, lastUsed: Date.now(), - ...(isPasswordSaved ? { encryptedPassword: this.cipher.encrypt(secret) } : {}), + ...(isPasswordSaved ? this.secretFields(existing, secret) : {}), }; const profiles = [...this.profiles.filter((item) => item.id !== id), profile]; await this.persist(profiles); diff --git a/electron/accounts.ts b/electron/accounts.ts index 0976958..d89c8c1 100644 --- a/electron/accounts.ts +++ b/electron/accounts.ts @@ -10,13 +10,15 @@ export const secretCipher: PasswordCipher = { decrypt: (value) => safeStorage.decryptString(Buffer.from(value, 'base64')), }; -export async function loadAccounts(): Promise { +// `onSaved` runs after every write, so sync hears about saved accounts. +export async function loadAccounts(onSaved: () => void = () => undefined): Promise { const file = path.join(app.getPath('userData'), 'accounts.json'); const registry = new AccountRegistry(secretCipher, async (profiles) => { await fs.mkdir(path.dirname(file), { recursive: true }); const temporary = `${file}.tmp`; await fs.writeFile(temporary, JSON.stringify(profiles, null, 2), { mode: 0o600 }); await fs.rename(temporary, file); + onSaved(); }); try { registry.load(JSON.parse(await fs.readFile(file, 'utf8'))); } catch (error) { diff --git a/electron/automation-host.ts b/electron/automation-host.ts index c5c19e7..3170e08 100644 --- a/electron/automation-host.ts +++ b/electron/automation-host.ts @@ -30,6 +30,27 @@ export interface HostDeps { send(event: E, data: AutomationEvents[E]): void; journal(message: string, source: string): void; saveFile(name: string, contents: string): Promise; + // After webhook settings or notification targets are saved, from here or from sync. + onSecretsSaved?(): void; +} + +// Webhook sources and notification targets as they sync. Whether a source is switched on stays +// with each computer, like the listener's address, port and switches: two computers holding +// broker sessions for one source's account would knock each other out. +export interface SyncedWebhooks { + readonly sources: readonly Omit[]; + readonly notifications: readonly NotificationTarget[]; +} + +function checkTargets(value: unknown): NotificationTarget[] { + const targets = value as NotificationTarget[]; + if (!Array.isArray(targets) || targets.length > 20) throw new Error('At most 20 notification targets'); + return targets.map((target) => { + if (!target || typeof target !== 'object' || !notificationKinds.some((kind) => kind.key === target.kind)) throw new Error('Unknown notification kind'); + const problem = validateTarget(target); + if (problem) throw new Error(`${target.name || 'Target'}: ${problem}`); + return { ...target, events: Array.isArray(target.events) ? target.events.filter((event) => notificationEvents.some((item) => item.key === event)) : [] }; + }); } const MAX_NEWS_MINUTES = 240; @@ -85,6 +106,8 @@ export class AutomationHost { readonly news: NewsCalendar; private settings: WebhookSettings = { host: '127.0.0.1', port: 8787, sources: [], isEnabled: false, isPaused: false }; private notifications: NotificationTarget[] = []; + // Settings that failed to load are not synced, or the empty defaults would replace everyone's. + private isSecretsLoaded = false; private copierPush?: NodeJS.Timeout; private readonly heldSources = new Map(); private readonly secrets: JsonStore; @@ -145,6 +168,7 @@ export class AutomationHost { this.settings = validateWebhookSettings(decode(stored.settings)); this.notifications = decode(stored.notifications); this.notifier.setTargets(this.notifications); + this.isSecretsLoaded = true; await this.webhooks.apply(this.settings); this.syncSourceSessions(); } catch (error) { @@ -192,6 +216,37 @@ export class AutomationHost { const isEncrypted = this.deps.cipher.canEncrypt(); const encode = (value: unknown) => isEncrypted ? this.deps.cipher.encrypt(JSON.stringify(value)) : JSON.stringify(value); await this.secrets.save({ isEncrypted, settings: encode(this.settings), notifications: encode(this.notifications) }); + this.isSecretsLoaded = true; + this.deps.onSecretsSaved?.(); + } + + // null while the saved settings are unreadable here. + syncedWebhooks(): SyncedWebhooks | null { + if (!this.isSecretsLoaded) return null; + return { sources: this.settings.sources.map(({ isEnabled, ...source }) => source), notifications: this.notifications }; + } + + private withSyncedSources(value: unknown): WebhookSettings { + const { sources } = (value ?? {}) as Partial; + if (!Array.isArray(sources)) throw new Error('Invalid synced webhook sources'); + const local = new Map(this.settings.sources.map((source) => [source.id, source.isEnabled])); + return validateWebhookSettings({ ...this.settings, sources: sources.map((source) => ({ ...source, isEnabled: local.get(source?.id) ?? false }) as WebhookSource) }); + } + + checkSyncedWebhooks(value: unknown): void { + this.withSyncedSources(value); + checkTargets((value as Partial).notifications); + } + + // A source new to this computer arrives switched off. + async applySyncedWebhooks(value: unknown): Promise { + const settings = this.withSyncedSources(value); + const notifications = checkTargets((value as Partial).notifications); + [this.settings, this.notifications] = [settings, notifications]; + this.notifier.setTargets(notifications); + await this.persistSecrets(); + await this.webhooks.apply(this.settings); + this.syncSourceSessions(); } canOpen(accountId: string | null, symbol?: string, lots?: number): Promise { @@ -279,14 +334,7 @@ export class AutomationHost { return this.webhooks.test(source, body); } case 'notifications.save': { - const targets = params as NotificationTarget[]; - if (!Array.isArray(targets) || targets.length > 20) throw new Error('At most 20 notification targets'); - for (const target of targets) { - if (!notificationKinds.some((kind) => kind.key === target.kind)) throw new Error('Unknown notification kind'); - const problem = validateTarget(target); - if (problem) throw new Error(`${target.name || 'Target'}: ${problem}`); - target.events = target.events.filter((event) => notificationEvents.some((item) => item.key === event)); - } + const targets = checkTargets(params); this.notifications = targets; this.notifier.setTargets(targets); await this.persistSecrets(); diff --git a/electron/device-identity.ts b/electron/device-identity.ts index 1a89eff..4001faa 100644 --- a/electron/device-identity.ts +++ b/electron/device-identity.ts @@ -69,6 +69,24 @@ export class DeviceIdentity { return true; } + // The CID as it syncs between computers that share one identity. + syncRecord(): { cid: string; isUnique: boolean } { + return { cid: this.clientId.toString('hex'), isUnique: !this.isWeak }; + } + + // Another computer's CID, from sync. Like a regenerated one, it applies from the next login. + async setFromSync(value: unknown): Promise { + const { cid, isUnique } = (value ?? {}) as { cid?: unknown; isUnique?: unknown }; + if (typeof isUnique !== 'boolean') throw new Error('Invalid client ID.'); + const parsed = parseClientId({ cid, ...(isUnique ? { version: SEED_VERSION } : {}) }); + if (this.current?.equals(parsed.cid) && this.isWeak === parsed.hasWeakSeed) return false; + await fs.mkdir(path.dirname(this.file), { recursive: true }); + await fs.writeFile(this.file, JSON.stringify({ cid: parsed.cid.toString('hex'), ...(isUnique ? { version: SEED_VERSION } : {}) }, null, 2), { mode: 0o600 }); + this.current = parsed.cid; + this.isWeak = parsed.hasWeakSeed; + return true; + } + async regenerate(): Promise { const next = deriveClientId(this.traits(), randomBytes(16).toString('hex')); await fs.mkdir(path.dirname(this.file), { recursive: true }); diff --git a/electron/github-remote.ts b/electron/github-remote.ts index 57f66ae..7884044 100644 --- a/electron/github-remote.ts +++ b/electron/github-remote.ts @@ -114,6 +114,10 @@ export class GitHubRemote implements SyncRemote { return `/repos/${this.repository.fullName}`; } + serverNow(): number { + return this.api.serverNow; + } + // Conditional on the last ETag (sent back verbatim: GitHub's are weak), so an unchanged head // costs no rate limit. An empty repository has no branch yet and reads as null. async head(): Promise { diff --git a/electron/main.ts b/electron/main.ts index 4402a9d..24d8d10 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -33,9 +33,10 @@ import { accountKey } from './journaling-store'; import { autoUpdater, DebUpdater } from 'electron-updater'; import { UpdateService } from './updater'; import type { UpdateMethod, UpdateState } from '../shared/updates'; -import type { SyncMethod, SyncSourceReply, SyncView } from '../shared/sync'; +import type { SyncMethod, SyncView } from '../shared/sync'; +import { mainSyncSources } from './sync-main-sources'; import { RendererSources } from './sync-renderer'; -import { SyncService } from './sync-service'; +import { SyncService, type LiveAccount } from './sync-service'; import { SYNC_FILE_EXTENSION } from './file-remote'; migrateLegacyProfile(); @@ -134,6 +135,51 @@ function leaveAccount(): void { uiAccount = null; journaling?.setActive(null); pool.afterUiChange(null, previous); + sync?.presenceChanged(); +} + +// When the account in use logged in here, so a move requested after that logs it out. +let uiLoginAt = 0; + +// Sessions this computer holds, for presence. Background sessions (copiers, signal sources) are +// listed too, but not handed over: copiers and signals get leases first. +function liveAccounts(): LiveAccount[] { + const background = pool.status().filter((link) => link.accountId !== activeAccount && (link.state === 'connected' || link.state === 'connecting')); + return [...(activeAccount && uiAccount ? [{ id: activeAccount, since: uiLoginAt }] : []), ...background.map((link) => ({ id: link.accountId, since: Number.POSITIVE_INFINITY }))]; +} + +async function releaseAccounts(accountIds: readonly string[], to: string): Promise { + if (!activeAccount || !accountIds.includes(activeAccount)) return; + const label = accounts?.list().find((profile) => profile.id === activeAccount)?.label || activeAccount.split('|').pop(); + await changeConnection(async () => { + if (!activeAccount || !accountIds.includes(activeAccount)) return; + leaveAccount(); + send('accountChanged', { id: null }); + await applySettings(); + }); + journal(`${label} moved to ${to}, so this computer logged out of it.`, 'Accounts'); +} + +async function ask(message: string, detail: string, buttons: readonly string[]): Promise { + const options = { type: 'question' as const, buttons: [...buttons], defaultId: 0, cancelId: buttons.length - 1, message, detail }; + const parent = window && !window.isDestroyed() && window.isVisible() ? window : undefined; + return (parent ? await dialog.showMessageBox(parent, options) : await dialog.showMessageBox(options)).response; +} + +// One MT5 account logged in from two computers can knock both sessions out, so a login to an +// account open elsewhere asks first, and can have the other computer log out of it. +async function handOff(accountId: string): Promise { + const holders = await sync?.whereOpen(accountId) ?? []; + if (!holders.length) return; + const label = accounts?.list().find((profile) => profile.id === accountId)?.label || accountId.split('|').pop(); + const where = holders.join(' and '); + const choice = await ask(`${label} is open on ${where}.`, `Move it here: ${where} logs out of it, then this computer logs in. Log in anyway: both stay logged in, which brokers may see as one account on two networks, and the two sessions can disconnect each other.`, ['Move it here', 'Log in anyway', 'Cancel']); + if (choice === 2) throw new Error('Login cancelled.'); + if (choice === 1) return; + journal(`Asking ${where} to log out of ${label}.`, 'Accounts'); + if (await sync!.moveHere(accountId)) return; + const again = await ask(`${where} has not logged out of ${label} yet.`, 'It may be asleep or offline, or running a copier or signal source on this account. Log in anyway, and it logs out of the account when it next syncs, unless a copier or signal source keeps it open.', ['Log in anyway', 'Cancel']); + if (again === 1) throw new Error('Login cancelled.'); } function sendSync(view: SyncView): void { @@ -357,12 +403,16 @@ function loginTarget(params: TerminalMethods['account.login'][0]): SignInTarget async function loginAccount(params: TerminalMethods['account.login'][0]) { if (params.isPasswordSaved && !accounts?.canSavePasswords()) throw new Error('Secure password storage is unavailable. Leave Save password unchecked.'); const next = parseSettings({ webEndpoint: params.webEndpoint ?? settings.webEndpoint }); + await handOff(`${next.webEndpoint}|${params.login}`); leaveAccount(); send('accountChanged', { id: null }); direct.use(mt5); mt5.disconnect(); // Each connection creates its client from device.clientId, so a new ID applies to this login. - if (params.isUniqueClientIdRequested && await device.ensureUnique()) journal('This computer now sends a unique client ID at login.', 'Accounts'); + if (params.isUniqueClientIdRequested && await device.ensureUnique()) { + journal('This computer now sends a unique client ID at login.', 'Accounts'); + sync?.noteLocalChange(); + } await pool.beforeUiLogin(`${next.webEndpoint}|${params.login}`); await saveSettings(next); settings = next; @@ -379,6 +429,8 @@ async function loginAccount(params: TerminalMethods['account.login'][0]) { } send('accountChanged', { id: activeAccount }); uiAccount = account; + uiLoginAt = Date.now(); + sync?.presenceChanged(); journaling?.setActive(account); return account; } @@ -420,6 +472,8 @@ async function loginExchange(params: ExchangeLogin): Promise { } send('accountChanged', { id: activeAccount }); uiAccount = account; + uiLoginAt = Date.now(); + sync?.presenceChanged(); return account; } @@ -519,7 +573,7 @@ ipcMain.handle('sync:request', async (_event, method: SyncMethod, params: unknow // Only the main window serves the stores; pop-outs share its page but have no preload of their own. const isMainWindow = (sender: Electron.WebContents) => !!window && !window.isDestroyed() && sender === window.webContents; ipcMain.on('sync:source-ready', (event) => { if (isMainWindow(event.sender)) syncSources.setReady(true); }); -ipcMain.on('sync:source-reply', (event, id: number, reply: SyncSourceReply) => { if (isMainWindow(event.sender)) syncSources.receive(id, reply); }); +ipcMain.on('sync:source-reply', (event, id: unknown, reply: unknown) => { if (isMainWindow(event.sender) && typeof id === 'number') syncSources.receive(id, reply); }); ipcMain.on('sync:changed', (event) => { if (isMainWindow(event.sender)) sync?.noteLocalChange(); }); ipcMain.handle('updates:request', async (_event, method: UpdateMethod, params: unknown) => { try { @@ -575,7 +629,7 @@ if (!app.requestSingleInstanceLock()) { settings = await loadSettings(); await zoom.load(); await device.load(); - try { accounts = await loadAccounts(); } + try { accounts = await loadAccounts(() => sync?.noteLocalChange()); } catch { journal('Saved accounts could not be loaded. Log in manually.', 'Accounts'); } const journalingSession = session.fromPartition('journaling'); journaling = new JournalingService({ @@ -593,6 +647,40 @@ if (!app.requestSingleInstanceLock()) { }, }); await journaling.init(); + // Before the automatic login, which first asks whether the account is open on another computer. + // Its own in-memory session, so no cookie from any other site ever rides along to GitHub. + const syncSession = session.fromPartition('sync'); + sync = new SyncService({ + dir: app.getPath('userData'), + fetch: (url, init) => syncSession.fetch(url, init), + cipher: secretCipher, + platform: process.platform, + appVersion: app.getVersion(), + deviceName: os.hostname(), + send: sendSync, + journal: (message) => journal(message, 'Sync'), + openExternal: (url) => shell.openExternal(url), + revealFile: (file) => shell.showItemInFolder(file), + sources: () => [ + ...mainSyncSources({ + accounts: () => accounts, automation: () => automation, device, onAccountsChanged: () => send('accounts', accountProfiles()), journal: (message) => journal(message, 'Sync'), + }), + ...syncSources.sources(), + ], + liveAccounts, + releaseAccounts, + chooseFile: async (mode) => { + const filters = [{ name: 'OpenTerm5 sync file', extensions: [SYNC_FILE_EXTENSION] }]; + if (mode === 'open') { + const result = window ? await dialog.showOpenDialog(window, { title: 'Open sync file', filters, properties: ['openFile'] }) : null; + return result && !result.canceled ? result.filePaths[0] ?? null : null; + } + const result = window ? await dialog.showSaveDialog(window, { title: 'Create sync file', defaultPath: `OpenTerm5.${SYNC_FILE_EXTENSION}`, filters }) : null; + return result && !result.canceled ? result.filePath ?? null : null; + }, + }); + await sync.init(); + pool.on('state', () => sync?.presenceChanged()); const recent = accounts?.list()[0]; const automatic = recent?.isPasswordSaved && recent.hasPassword && (isExchangePlatform(recent.platform) || recent.webEndpoint === settings.webEndpoint) ? recent : undefined; // Announced before the window loads and before the endpoint connects, so the window never looks logged out meanwhile. @@ -619,6 +707,7 @@ if (!app.requestSingleInstanceLock()) { automation = new AutomationHost({ pool, userData: app.getPath('userData'), cipher: secretCipher, appVersion: app.getVersion(), accounts: () => accounts, send: sendAutomation, journal, saveFile, clientId: () => device.clientId.toString('hex'), + onSecretsSaved: () => sync?.noteLocalChange(), }); await automation.init(); // Squirrel.Mac only installs signed builds, so macOS checks and links to the release page. @@ -634,31 +723,6 @@ if (!app.requestSingleInstanceLock()) { confirmRestart, }); await updates.init(); - // Its own in-memory session, so no cookie from any other site ever rides along to GitHub. - const syncSession = session.fromPartition('sync'); - sync = new SyncService({ - dir: app.getPath('userData'), - fetch: (url, init) => syncSession.fetch(url, init), - cipher: secretCipher, - platform: process.platform, - appVersion: app.getVersion(), - deviceName: os.hostname(), - send: sendSync, - journal: (message) => journal(message, 'Sync'), - openExternal: (url) => shell.openExternal(url), - revealFile: (file) => shell.showItemInFolder(file), - sources: () => syncSources.sources(), - chooseFile: async (mode) => { - const filters = [{ name: 'OpenTerm5 sync file', extensions: [SYNC_FILE_EXTENSION] }]; - if (mode === 'open') { - const result = window ? await dialog.showOpenDialog(window, { title: 'Open sync file', filters, properties: ['openFile'] }) : null; - return result && !result.canceled ? result.filePaths[0] ?? null : null; - } - const result = window ? await dialog.showSaveDialog(window, { title: 'Create sync file', defaultPath: `OpenTerm5.${SYNC_FILE_EXTENSION}`, filters }) : null; - return result && !result.canceled ? result.filePath ?? null : null; - }, - }); - await sync.init(); // Leaving a computer pushes its edits, and sitting down at one catches it up. powerMonitor.on('suspend', () => sync?.wake()); powerMonitor.on('lock-screen', () => sync?.wake()); @@ -676,10 +740,10 @@ app.on('window-all-closed', () => { // Quitting waits up to 5 s for edits not yet synced, so they are not left on one computer. app.on('before-quit', (event) => { - if (!isSyncFlushed && sync?.hasUnpushedChanges) { + if (!isSyncFlushed && sync?.isLeaveNeeded) { event.preventDefault(); isSyncFlushed = true; - void Promise.race([sync.flush(), new Promise((resolve) => setTimeout(resolve, 5000))]).finally(() => app.quit()); + void Promise.race([sync.leave(), new Promise((resolve) => setTimeout(resolve, 5000))]).finally(() => app.quit()); return; } updates?.shutdown(); diff --git a/electron/sync-engine.test.ts b/electron/sync-engine.test.ts index 169b58b..6c6afe8 100644 --- a/electron/sync-engine.test.ts +++ b/electron/sync-engine.test.ts @@ -326,3 +326,75 @@ test('a source that cannot answer stops the sync instead of reading its values a await b.engine.sync(); assert.deepEqual(source.data, { L1: { name: 'Layout' } }); }); + +test('a merge that breaks a limit keeps the synced value instead of failing every sync', async () => { + const { a, b, tick } = setup(); + for (const computer of [a, b]) computer.source('chartDrawings').isValid = (value) => Object.keys(value as object).length <= 2; + a.source('chartDrawings').data = { c1: { d1: 'line' } }; + await a.engine.sync(); + await b.engine.sync(); + a.source('chartDrawings').data = { c1: { d1: 'line', d2: 'box' } }; + b.source('chartDrawings').data = { c1: { d1: 'line', d3: 'fib' } }; + tick(); + await a.engine.sync(); + const outcome = await b.engine.sync(); + assert.equal(outcome.status, 'synced'); + assert.deepEqual(b.source('chartDrawings').data, { c1: { d1: 'line', d2: 'box' } }); +}); + +test('a record this computer could not read is never overwritten by its own copy', async () => { + const { a, b, tick } = setup(); + b.source('layout').data = { L1: { name: 'old' } }; + b.source('layout').isValid = (value) => (value as { name?: string }).name !== 'new'; + a.source('layout').data = { L1: { name: 'new' } }; + await a.engine.sync(); + tick(); + await b.engine.sync(); + await b.engine.sync(); + await a.engine.sync(); + assert.deepEqual(a.source('layout').data, { L1: { name: 'new' } }); +}); + +test('a first merge keeps what only this computer has and takes the synced value where both have one', async () => { + const { a, b, tick } = setup(['layout', 'preferences']); + a.source('preferences').data = { workspace: { theme: 'dark', drawingSync: 'symbol' } }; + a.source('layout').data = { L1: { name: 'Synced' } }; + await a.engine.sync(); + tick(); + b.source('preferences').data = { workspace: { theme: 'light', replayMode: 'allCharts' } }; + b.source('layout').data = { L2: { name: 'Mine' } }; + await b.engine.sync({ isRemotePreferred: true }); + await a.engine.sync(); + const expected = { workspace: { theme: 'dark', drawingSync: 'symbol', replayMode: 'allCharts' } }; + assert.deepEqual(b.source('preferences').data, expected); + assert.deepEqual(a.source('preferences').data, expected); + assert.deepEqual(Object.keys(a.source('layout').data).sort(), ['L1', 'L2']); +}); + +test('use synced data can be limited to some kinds, the others still merging', async () => { + const { a, b } = setup(['account', 'layout']); + a.source('layout').data = { L1: { name: 'Synced' } }; + await a.engine.sync(); + b.source('layout').data = { L9: { name: 'Default' } }; + b.source('account').data = { A9: { login: 9 } }; + await b.engine.sync({ isLocalDiscarded: true, isMassChangeAllowed: true, discardedKinds: ['layout'] }); + await a.engine.sync(); + assert.deepEqual(b.source('layout').data, { L1: { name: 'Synced' } }); + assert.deepEqual(a.source('account').data, { A9: { login: 9 } }); +}); + +test('kinds apply in declaration order whatever order their sources came in, and __proto__ ids never sync', async () => { + const remote = new FakeRemote(); + const rootKey = newRootKey(); + const clock = { now: 1_800_000_000_000 }; + const a = device('a', remote, rootKey, ['layout', 'chart'], clock); + a.source('chart').data = { c1: { symbol: 'EURUSD' } }; + a.source('layout').data = JSON.parse('{"L1": {"name": "x"}, "__proto__": {"name": "y"}}'); + await a.engine.sync(); + const order: string[] = []; + const b = device('b', remote, rootKey, ['layout', 'chart'], clock); + for (const kind of ['layout', 'chart'] as const) b.source(kind).onApply = () => order.push(kind); + await b.engine.sync(); + assert.deepEqual(order, ['chart', 'layout']); + assert.deepEqual(Object.keys(b.source('layout').data), ['L1']); +}); diff --git a/electron/sync-engine.ts b/electron/sync-engine.ts index aa5532f..4bb0ba3 100644 --- a/electron/sync-engine.ts +++ b/electron/sync-engine.ts @@ -1,8 +1,8 @@ -import { isSameJson, isSyncKind, recordKey, splitRecordKey, syncKindInfo, type Json, type SyncKind, type SyncSourceChange } from '../shared/sync'; +import { isSameJson, isSyncKind, recordKey, splitRecordKey, syncKindInfo, syncKinds, type Json, type SyncKind, type SyncSourceChange } from '../shared/sync'; import { hlcAt, HybridClock } from './sync-clock'; import { decodePayload, encodePayload, envelopeHash, MAX_ENVELOPE_BYTES, objectPath, open, recordCipherKey, seal, type SyncKeys } from './sync-crypto'; import { emptyManifest, MANIFEST_PATH, openManifest, sealManifest, SyncIntegrityError, type Manifest, type ManifestDevice, type ManifestEntry } from './sync-manifest'; -import { mergeRecords, parseSyncedRecord, pruneTombstones, recordPlain, stampRecord, type MergeConflict, type SyncedRecord } from './sync-merge'; +import { hasUnsafeKey, mergeRecords, parseSyncedRecord, preferRemote, pruneTombstones, recordPlain, stampRecord, type MergeConflict, type SyncedRecord } from './sync-merge'; const TOMBSTONE_LIFETIME_MS = 180 * 86_400_000; const MAX_ATTEMPTS = 5; @@ -19,6 +19,8 @@ export interface SyncRemote { head(): Promise; read(commit: string, paths: readonly string[]): Promise>; commit(parent: string | null, writes: ReadonlyMap): Promise; + // The remote's own clock, which presence and leases are measured on; local time when absent. + serverNow?(): number; } export type SourceChange = SyncSourceChange; @@ -82,6 +84,11 @@ export interface SyncOptions { readonly isMassChangeAllowed?: boolean; // "Use synced data": local changes since the last sync are replaced by the remote's. readonly isLocalDiscarded?: boolean; + // Limits `isLocalDiscarded` to these kinds; the others still merge. + readonly discardedKinds?: readonly SyncKind[]; + // A computer merging into synced data for the first time: where both sides hold a value and + // there is no base yet, the synced one wins, so a fresh install's defaults never replace it. + readonly isRemotePreferred?: boolean; // The user confirmed the repository was put back to an older state on purpose. readonly isHistoryAccepted?: boolean; } @@ -118,7 +125,9 @@ export class SyncEngine { this.now = deps.now ?? Date.now; this.clock = new HybridClock(deps.device.id, this.now); this.pause = deps.pause ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); - this.sources = new Map(deps.sources.map((source) => [source.kind, source])); + // Kinds apply in declaration order, whatever order the sources came in. + const rank = (kind: SyncKind) => syncKinds.findIndex((info) => info.key === kind); + this.sources = new Map([...deps.sources].sort((left, right) => rank(left.kind) - rank(right.kind)).map((source) => [source.kind, source])); } // Stamps local edits when they happen rather than at the next sync, so "the newer edit wins" @@ -155,6 +164,65 @@ export class SyncEngine { return this.exclusive(() => this.syncNow(options)); } + // One record as the remote holds it now, for a kind this computer may not sync yet (deciding + // which CID to keep before turning CID sync on). Checked like a pull: hash, then authentication. + peek(key: string): Promise { + return this.exclusive(async () => { + const { keys, epoch, repoId, remote } = this.deps; + const head = await remote.head(); + const envelope = head === null ? null : (await remote.read(head, [MANIFEST_PATH])).get(MANIFEST_PATH) ?? null; + if (head === null || !envelope) return undefined; + const state = await this.deps.store.load(); + const entry = openManifest(keys, envelope, { epoch, repoId, highWater: state?.highWater ?? 0 }).records[key]; + if (!entry?.hash) return undefined; + const object = (await remote.read(head, [objectPath(entry.hash)])).get(objectPath(entry.hash)); + if (!object || envelopeHash(object) !== entry.hash) throw new SyncIntegrityError('tampered', 'A synced record is missing or altered.'); + let record: SyncedRecord; + try { record = parseSyncedRecord(decodePayload(open(recordCipherKey(keys, key), object, this.recordContext(key, entry.generation)))); } + catch { throw new SyncIntegrityError('tampered', 'A synced record failed authentication.'); } + return recordPlain(syncKindInfo(splitRecordKey(key).kind as SyncKind).shape, record); + }); + } + + serverNow(): number { + return this.deps.remote.serverNow?.() ?? this.now(); + } + + // The remote manifest now, checked as a pull checks it, without reading any record or store. + peekManifest(): Promise { + return this.exclusive(async () => (await this.readManifest())?.manifest ?? null); + } + + // A commit of the manifest alone (presence, handoff requests), so it needs none of the stores. + // `edit` returns null when nothing needs writing; a lost race reads again and retries. + editManifest(edit: (manifest: Manifest, serverNow: number) => Manifest | null): Promise { + return this.exclusive(async () => { + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + if (attempt) await this.pause(2 ** attempt * 250 + Math.random() * 250); + const current = await this.readManifest(); + if (!current) return false; + this.clock.observe(current.manifest.hlc); + const next = edit(current.manifest, this.serverNow()); + if (!next) return false; + const manifest: Manifest = { ...next, seq: current.manifest.seq + 1, hlc: this.clock.tick(), writer: this.deps.device.id }; + if (await this.deps.remote.commit(current.head, new Map([[MANIFEST_PATH, sealManifest(this.deps.keys, manifest)]])) === null) continue; + const state = await this.deps.store.load(); + if (state) await this.deps.store.save({ ...state, highWater: Math.max(state.highWater, manifest.seq), clock: this.clock.last }); + return true; + } + return false; + }); + } + + private async readManifest(): Promise<{ head: string; manifest: Manifest } | null> { + const { keys, epoch, repoId, remote } = this.deps; + const head = await remote.head(); + const envelope = head === null ? null : (await remote.read(head, [MANIFEST_PATH])).get(MANIFEST_PATH) ?? null; + if (head === null || !envelope) return null; + const state = await this.deps.store.load(); + return { head, manifest: openManifest(keys, envelope, { epoch, repoId, highWater: state?.highWater ?? 0 }) }; + } + private exclusive(run: () => Promise): Promise { const result = this.queue.then(run); this.queue = result.catch(() => undefined); @@ -168,7 +236,8 @@ export class SyncEngine { for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { if (attempt) await this.pause(2 ** attempt * 250 + Math.random() * 250); const head = await this.deps.remote.head(); - const view = head !== null && head === state.head && state.manifest ? { head, manifest: state.manifest, records: state.bases, invalid: new Set() } : await this.pull(head, state, options); + const isUnchanged = head !== null && head === state.head && !!state.manifest && !this.hasUnreadRecords(state); + const view = isUnchanged ? { head: head!, manifest: state.manifest!, records: state.bases, invalid: new Set() } : await this.pull(head, state, options); const outcome = await this.reconcile(state, view, options); if (outcome.status !== 'retry') return outcome.status === 'synced' ? outcome.outcome : outcome; state = outcome.state; @@ -176,6 +245,13 @@ export class SyncEngine { return { status: 'busy' }; } + // A record the last pull left out (invalid then, perhaps valid now that a record it depends on + // arrived) is in the manifest with no base. Skipping the pull would read it as absent and let + // local data overwrite it, so the pull runs again; it fetches only those records. + private hasUnreadRecords(state: SyncState): boolean { + return Object.entries(state.manifest!.records).some(([key, entry]) => entry.hash !== null && this.isReadable(key, entry) && !state.bases[key]); + } + private async pull(head: string | null, state: SyncState, options: SyncOptions): Promise { const { keys, epoch, repoId, remote } = this.deps; const envelope = head === null ? null : (await remote.read(head, [MANIFEST_PATH])).get(MANIFEST_PATH) ?? null; @@ -244,6 +320,7 @@ export class SyncEngine { for (const [kind, source] of this.sources) { const { shape } = syncKindInfo(kind); const snapshot = snapshots.get(kind)!; + const isDiscarded = !!options.isLocalDiscarded && (!options.discardedKinds || options.discardedKinds.includes(kind)); const keys = new Set([...Object.keys(state.bases), ...Object.keys(view.records), ...Object.keys(state.pending)].filter((key) => splitRecordKey(key).kind === kind)); for (const id of Object.keys(snapshot)) keys.add(recordKey(kind, id)); const counts = { local: 0, remote: 0, items: 0, total: 0 }; @@ -256,10 +333,19 @@ export class SyncEngine { const base = state.bases[key]; const current = snapshot[id]; if (current !== undefined && !await this.isValidPlain(source, current, id)) { skipped.add(key); continue; } - const local = options.isLocalDiscarded ? base : stampRecord(shape, state.pending[key] ?? base, current, tick); + const local = isDiscarded ? base : stampRecord(shape, state.pending[key] ?? base, current, tick); const remote = view.records[key]; - const result = mergeRecords(shape, base, local, remote, tick); + const isFirstMeeting = !!options.isRemotePreferred && !base && !!local && !!remote && !local.deletedHlc && !remote.deletedHlc; + const result = isFirstMeeting ? preferRemote(shape, local, remote, tick) : mergeRecords(shape, base, local, remote, tick); let next = result.merged; + // Each side was checked on its own; the merge of two valid values can still break a limit + // (more lists than allowed, no list at all). Then the remote's stays, rather than an apply + // failing on every sync from now on. + const mergedPlain = recordPlain(shape, next); + if (mergedPlain !== undefined && !isSameJson(mergedPlain, current) && !isSameJson(mergedPlain, recordPlain(shape, remote)) && !await this.isValidPlain(source, mergedPlain, id)) { + if (!isAlive(remote)) { skipped.add(key); continue; } + next = remote; + } if (next && !sameRecord(next, remote)) next = pruneTombstones(next, horizon); if (next) merged[key] = next; if (next && !sameRecord(next, remote)) toPush.add(key); @@ -287,9 +373,9 @@ export class SyncEngine { counts.items = Math.max(counts.items, removedItems(base, local)); } } - const isEmptied = counts.total >= MASS_EMPTY_FROM && !Object.keys(snapshot).length && !options.isLocalDiscarded; + const isEmptied = counts.total >= MASS_EMPTY_FROM && !Object.keys(snapshot).length && !isDiscarded; const isMass = (removed: number) => removed > MASS_RECORDS && removed / Math.max(counts.total, 1) > MASS_SHARE; - if (isEmptied || isMass(counts.local) || (counts.items > MASS_ITEMS && !options.isLocalDiscarded)) mass.push({ kind, side: 'local', removed: Math.max(counts.local, counts.items), total: counts.total }); + if (isEmptied || isMass(counts.local) || (counts.items > MASS_ITEMS && !isDiscarded)) mass.push({ kind, side: 'local', removed: Math.max(counts.local, counts.items), total: counts.total }); if (isMass(counts.remote)) mass.push({ kind, side: 'remote', removed: counts.remote, total: counts.total }); if (change.deletes.length || Object.keys(change.upserts).length) { changes.set(kind, change); @@ -412,7 +498,9 @@ export class SyncEngine { return plain === undefined || this.isValidPlain(this.sources.get(kind as SyncKind)!, plain, id); } + // `__proto__` as an id or a key would set an object's prototype wherever it is assigned. private async isValidPlain(source: SyncSource, plain: Json, id: string): Promise { + if (id === '__proto__' || hasUnsafeKey(syncKindInfo(source.kind).shape, plain)) return false; try { await source.validate(plain, id); return true; diff --git a/electron/sync-main-sources.test.ts b/electron/sync-main-sources.test.ts new file mode 100644 index 0000000..b1d2db2 --- /dev/null +++ b/electron/sync-main-sources.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import type { Account } from '../shared/protocol'; +import { AccountRegistry } from './account-registry'; +import { DeviceIdentity } from './device-identity'; +import { deriveKeys, newRootKey } from './sync-crypto'; +import { SyncEngine } from './sync-engine'; +import { FakeRemote, MemoryStore, REPO_ID } from './sync-engine-fixtures'; +import { mainSyncSources } from './sync-main-sources'; + +const account: Account = { name: 'Fixture', server: 'Demo', company: 'Broker', currency: 'USD', tradeMode: 'demo', login: 1234, credit: 0, margin: 0, equity: 1000, profit: 0, balance: 1000, leverage: 100, marginFree: 1000, marginLevel: 0, isHedging: true, isTradeAllowed: true }; +const endpoint = 'wss://broker.example/terminal'; +const cipher = { canEncrypt: () => true, encrypt: (value: string) => `encrypted:${Buffer.from(value).toString('base64')}`, decrypt: (value: string) => Buffer.from(value.slice(10), 'base64').toString() }; +const traits = () => ({ scale: 1, width: 1920, height: 1080, platform: 'Linux x86_64', language: 'en-US' }); + +async function computer(id: string, remote: FakeRemote, rootKey: Buffer, kinds: readonly string[]) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'openterm-sync-main-')); + const registry = new AccountRegistry(cipher, async () => {}); + const device = new DeviceIdentity(path.join(dir, 'device.json'), traits); + await device.load(); + const journal: string[] = []; + let changed = 0; + const sources = mainSyncSources({ accounts: () => registry, automation: () => undefined, device, onAccountsChanged: () => { changed += 1; }, journal: (message) => journal.push(message) }) + .filter((source) => kinds.includes(source.kind)); + const engine = new SyncEngine({ + epoch: 1, repoId: REPO_ID, keys: deriveKeys(rootKey), remote, store: new MemoryStore(), sources, pause: async () => undefined, + device: { id, name: id, platform: 'linux', publicKey: 'x', appVersion: '1.0.0' }, + }); + return { registry, device, engine, journal, changed: () => changed }; +} + +test('accounts sync everywhere; their passwords only between computers that opted in', async () => { + const remote = new FakeRemote(); + const rootKey = newRootKey(); + const office = await computer('aaaaaaaaaaaaaaaa', remote, rootKey, ['account', 'accountSecret']); + const laptop = await computer('bbbbbbbbbbbbbbbb', remote, rootKey, ['account', 'accountSecret']); + const kiosk = await computer('cccccccccccccccc', remote, rootKey, ['account']); + const profile = await office.registry.remember(account, endpoint, 'broker-password', 'Demo', true); + await office.engine.sync(); + for (const content of remote.files().values()) assert.ok(!content.includes('broker-password')); + // The password is checked before its account arrives, so it applies on the next sync. + await laptop.engine.sync(); + await laptop.engine.sync(); + await kiosk.engine.sync(); + assert.equal(laptop.registry.get(profile.id).label, 'Demo'); + assert.equal(laptop.registry.password(profile.id), 'broker-password'); + assert.ok(laptop.changed() > 0); + assert.equal(kiosk.registry.get(profile.id).isPasswordSaved, false); + + // A password changed later on the laptop reaches the office; the kiosk still never gets one. + await new Promise((resolve) => setTimeout(resolve, 5)); + await laptop.registry.remember(account, endpoint, 'changed-password', 'Demo', true); + await laptop.engine.sync(); + await office.engine.sync(); + await kiosk.engine.sync(); + assert.equal(office.registry.password(profile.id), 'changed-password'); + assert.equal(kiosk.registry.password(profile.id), undefined); +}); + +test('one CID for all: the synced CID replaces this computer\'s and applies from the next login', async () => { + const remote = new FakeRemote(); + const rootKey = newRootKey(); + const office = await computer('aaaaaaaaaaaaaaaa', remote, rootKey, ['identity']); + const laptop = await computer('bbbbbbbbbbbbbbbb', remote, rootKey, ['identity']); + assert.notDeepEqual(laptop.device.clientId, office.device.clientId); + await office.engine.sync(); + await laptop.engine.sync({ isRemotePreferred: true }); + assert.deepEqual(laptop.device.clientId, office.device.clientId); + assert.match(laptop.journal.join(' '), /next login/); + await office.device.regenerate(); + await office.engine.sync(); + await laptop.engine.sync(); + assert.deepEqual(laptop.device.clientId, office.device.clientId); +}); diff --git a/electron/sync-main-sources.ts b/electron/sync-main-sources.ts new file mode 100644 index 0000000..82bcb46 --- /dev/null +++ b/electron/sync-main-sources.ts @@ -0,0 +1,90 @@ +import type { Json } from '../shared/sync'; +import type { AccountRegistry } from './account-registry'; +import type { AutomationHost } from './automation-host'; +import type { DeviceIdentity } from './device-identity'; +import { SyncSourceUnavailableError, type SyncSource } from './sync-engine'; + +// The stores the main process holds, as sync sources. Secrets are handled here and never reach the +// renderer: sync decrypts them from the keychain only to encrypt them under its own key. + +export const IDENTITY_ID = 'cid'; +export const WEBHOOKS_ID = 'webhooks'; + +export interface MainSourcesDeps { + readonly accounts: () => AccountRegistry | undefined; + readonly automation: () => AutomationHost | undefined; + readonly device: DeviceIdentity; + readonly onAccountsChanged: () => void; + readonly journal: (message: string) => void; +} + +const toJson = (value: unknown): Json => JSON.parse(JSON.stringify(value)) as Json; + +// Saved accounts that failed to load read as none at all; syncing that would delete them on +// every computer, so sync stops and says why instead. +function registryOf(deps: MainSourcesDeps): AccountRegistry { + const registry = deps.accounts(); + if (!registry) throw new Error('Saved accounts could not be read on this computer, so sync is paused. Restart OpenTerm5 to try again.'); + return registry; +} + +function automationOf(deps: MainSourcesDeps): AutomationHost { + const automation = deps.automation(); + if (!automation) throw new SyncSourceUnavailableError('Automation is still starting.'); + return automation; +} + +function singleton(id: string, expected: string): void { + if (id !== expected) throw new Error(`Unknown record ${id}.`); +} + +export function mainSyncSources(deps: MainSourcesDeps): SyncSource[] { + return [ + { + kind: 'account', + snapshot: async () => toJson(registryOf(deps).syncedProfiles()) as Record, + validate: (value, id) => registryOf(deps).checkSyncedProfile(id, value), + apply: async (change) => { + await registryOf(deps).applySyncedProfiles(change); + deps.onAccountsChanged(); + }, + }, + { + kind: 'accountSecret', + snapshot: async () => toJson(registryOf(deps).syncedSecrets()) as Record, + validate: (value, id) => registryOf(deps).checkSyncedSecret(id, value), + apply: async (change) => { + await registryOf(deps).applySyncedSecrets(change); + deps.onAccountsChanged(); + }, + }, + { + kind: 'identity', + snapshot: async () => ({ [IDENTITY_ID]: deps.device.syncRecord() }), + validate: (value, id) => { + singleton(id, IDENTITY_ID); + const { cid, isUnique } = (value ?? {}) as { cid?: unknown; isUnique?: unknown }; + if (typeof cid !== 'string' || !/^[0-9a-f]{32}$/.test(cid) || typeof isUnique !== 'boolean') throw new Error('Invalid client ID.'); + }, + // The CID is never removed, only replaced. + apply: async (change) => { + const value = change.upserts[IDENTITY_ID]; + if (value !== undefined && await deps.device.setFromSync(value)) deps.journal('This computer now uses the CID of your other computers. It applies from the next login.'); + }, + }, + { + kind: 'webhookSecrets', + // Unreadable settings are offered as a record that fails its checks, so it is left alone. + snapshot: async () => ({ [WEBHOOKS_ID]: toJson(automationOf(deps).syncedWebhooks() ?? { isUnreadable: true }) }), + validate: (value, id) => { + singleton(id, WEBHOOKS_ID); + automationOf(deps).checkSyncedWebhooks(value); + }, + // Webhook settings are never removed as a whole. + apply: async (change) => { + const value = change.upserts[WEBHOOKS_ID]; + if (value !== undefined) await automationOf(deps).applySyncedWebhooks(value); + }, + }, + ]; +} diff --git a/electron/sync-manifest.test.ts b/electron/sync-manifest.test.ts index d911096..7ba55af 100644 --- a/electron/sync-manifest.test.ts +++ b/electron/sync-manifest.test.ts @@ -35,7 +35,11 @@ test('malformed manifests are refused field by field', () => { { ...manifest, leases: { copier: { fence: 1, deviceId: 'a', expiresAt: 'soon', renewedAt: 1 } } }, { ...manifest, presence: { a: { accounts: [5], expiresAt: 1 } } }, { ...manifest, devices: [{ id: 'a', name: 'x', platform: 'linux', publicKey: 'k', appVersion: '1', lastSeen: 1, firstSeen: 1 }] }, + { ...manifest, handoffs: { 'mt5|1': { from: 'a', to: ['Not An Id'], createdAt: 1, expiresAt: 2 } } }, + { ...manifest, handoffs: { 'mt5|1': { from: 'a', to: ['b'], expiresAt: 2 } } }, + JSON.parse(JSON.stringify({ ...manifest, records: {} }).replace('"records":{}', '"records":{"__proto__":{}}')), ]; + assert.ok(parseManifest({ ...manifest, handoffs: { 'mt5|1': { from: 'a', to: ['b'], createdAt: 1, expiresAt: 2 } } }).handoffs); for (const value of bad) assert.throws(() => parseManifest(value), reason('tampered'), JSON.stringify(value)); }); @@ -77,6 +81,7 @@ test('presence shows where an account is open and expires on its own', () => { presence = withPresence(presence, 'b', ['mt5|456'], 10); assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'b', 20), ['a']); assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'a', 20), []); - assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'b', 400_000), []); + assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'b', 400_000), ['a']); + assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'b', 2_000_000), []); assert.deepEqual(Object.keys(withPresence(presence, 'a', [], 30)), ['b']); }); diff --git a/electron/sync-manifest.ts b/electron/sync-manifest.ts index db42577..126e35d 100644 --- a/electron/sync-manifest.ts +++ b/electron/sync-manifest.ts @@ -11,7 +11,14 @@ export const MAX_MANIFEST_BYTES = 4 * 1024 * 1024; // `fenceAfterMs` without a renewal, before anyone else may take the lease at `lifetimeMs` plus // the margin, so two computers never both have the right to open. export const leaseTiming = { renewEveryMs: 120_000, fenceAfterMs: 420_000, lifetimeMs: 600_000, takeoverMarginMs: 60_000 } as const; -export const PRESENCE_LIFETIME_MS = 300_000; +// Presence costs a commit per renewal, and each commit keeps a new copy of the manifest in the +// repository's history, so it lasts half an hour and renews when less than half is left. A +// computer that logs out removes it at once; one that crashed is covered by "Log in anyway". +export const PRESENCE_LIFETIME_MS = 1_800_000; +export const PRESENCE_RENEW_MS = 900_000; +// A request to move an account stays open long enough for a computer that was asleep to see it +// when it wakes; it then logs out only if its own login is older than the request. +export const HANDOFF_LIFETIME_MS = 12 * 3_600_000; export interface ManifestEntry { // null for a deleted record. @@ -44,6 +51,14 @@ export interface Presence { readonly expiresAt: number; } +// A computer asking the ones that hold an account to log out of it, so it can log in alone. +export interface Handoff { + readonly from: string; + readonly to: readonly string[]; + readonly createdAt: number; + readonly expiresAt: number; +} + export interface Manifest { readonly v: 1; readonly seq: number; @@ -57,6 +72,8 @@ export interface Manifest { readonly leases: Readonly>; readonly records: Readonly>; readonly presence: Readonly>; + // By account id. Missing in manifests written before login handoff; older builds keep it as is. + readonly handoffs?: Readonly>; } export type IntegrityReason = 'tampered' | 'rollback' | 'epoch' | 'tooNew' | 'stale' | 'revoked'; @@ -90,6 +107,8 @@ export function parseManifest(value: unknown): Manifest { if (!isObject(device) || !isDeviceId(device.id) || !isText(device.name) || !isText(device.platform) || !isText(device.publicKey) || !isText(device.appVersion) || !isCount(device.lastSeen) || !isCount(device.firstSeen) || typeof device.isRevoked !== 'boolean') fail('device'); } + // Keys are assigned into plain objects all through the engine, so none may be `__proto__`. + if ([...Object.keys(records as object), ...Object.keys(leases as object)].some((key) => key === '__proto__' || key.endsWith(':__proto__'))) fail('keys'); for (const entry of Object.values(records as Record)) { if (!isObject(entry) || !(entry.hash === null || (typeof entry.hash === 'string' && /^[0-9a-f]{64}$/.test(entry.hash))) || !isHlc(entry.hlc) || !isCount(entry.schema) || !isCount(entry.generation)) fail('record'); @@ -100,6 +119,13 @@ export function parseManifest(value: unknown): Manifest { for (const [deviceId, entry] of Object.entries(presence as Record)) { if (!isDeviceId(deviceId) || !isObject(entry) || !Array.isArray(entry.accounts) || !entry.accounts.every((account) => isText(account, 512)) || !isCount(entry.expiresAt)) fail('presence'); } + const { handoffs } = value; + if (handoffs !== undefined) { + if (!isObject(handoffs)) fail('handoffs'); + for (const [account, entry] of Object.entries(handoffs as Record)) { + if (account === '__proto__' || !isText(account, 512) || !isObject(entry) || !isDeviceId(entry.from) || !Array.isArray(entry.to) || !entry.to.every(isDeviceId) || !isCount(entry.createdAt) || !isCount(entry.expiresAt)) fail('handoff'); + } + } return value as unknown as Manifest; } @@ -164,6 +190,18 @@ export function withPresence(presence: Manifest['presence'], deviceId: string, a return accounts.length ? { ...live, [deviceId]: { accounts: [...accounts].sort(), expiresAt: serverNow + PRESENCE_LIFETIME_MS } } : live; } +// Whether this computer's presence should be written: its accounts changed, or it runs out soon. +export function isPresenceDue(presence: Manifest['presence'], deviceId: string, accounts: readonly string[], serverNow: number): boolean { + const own = presence[deviceId]; + if (!accounts.length) return !!own; + return !own || own.expiresAt - serverNow < PRESENCE_RENEW_MS || own.accounts.join('\n') !== [...accounts].sort().join('\n'); +} + +// Open requests, expired ones dropped. +export function liveHandoffs(handoffs: Manifest['handoffs'], serverNow: number): Record { + return Object.fromEntries(Object.entries(handoffs ?? {}).filter(([, entry]) => entry.expiresAt > serverNow)); +} + export function presentElsewhere(presence: Manifest['presence'], account: string, deviceId: string, serverNow: number): string[] { return Object.entries(presence).filter(([id, entry]) => id !== deviceId && entry.expiresAt > serverNow && entry.accounts.includes(account)).map(([id]) => id).sort(); } diff --git a/electron/sync-merge.test.ts b/electron/sync-merge.test.ts index 44bf093..06232d9 100644 --- a/electron/sync-merge.test.ts +++ b/electron/sync-merge.test.ts @@ -181,3 +181,8 @@ test('random edits on three replicas converge on the newest edit of every key', for (const replica of replicas) assert.deepEqual(replica.plain, expected, `round ${round}`); } }); + +test('an entry key that would set a prototype is refused when a record is read', () => { + const hlc = '1800000000000.0000.a'; + assert.throws(() => parseSyncedRecord(JSON.parse(`{"entries": {"__proto__": {"hlc": "${hlc}", "value": 1}}}`)), /not allowed/); +}); diff --git a/electron/sync-merge.ts b/electron/sync-merge.ts index f254afa..9364704 100644 --- a/electron/sync-merge.ts +++ b/electron/sync-merge.ts @@ -33,6 +33,17 @@ export interface MergeResult { const SINGLE = ''; +// Entries are plain objects keyed by item ids, so lookups must not reach Object.prototype. +const own = (entries: SyncEntries, key: string): SyncEntry | undefined => Object.hasOwn(entries, key) ? entries[key] : undefined; + +const UNSAFE_KEY = '__proto__'; + +// A key that would set an object's prototype when assigned, among a value's entry keys. +export function hasUnsafeKey(shape: SyncShape, plain: Json): boolean { + if (shape === 'set') return Array.isArray(plain) && plain.includes(UNSAFE_KEY); + return (shape === 'fields' || shape === 'map') && plain !== null && typeof plain === 'object' && !Array.isArray(plain) && Object.hasOwn(plain, UNSAFE_KEY); +} + function plainEntries(shape: SyncShape, plain: Json): Record { if (shape === 'fields' || shape === 'map') { if (plain === null || typeof plain !== 'object' || Array.isArray(plain)) throw new Error(`A ${shape} record must be an object.`); @@ -49,7 +60,7 @@ export function fromEntries(shape: SyncShape, entries: SyncEntries): Json { const live = Object.entries(entries).filter(([, entry]) => !entry.isDeleted); if (shape === 'fields' || shape === 'map') return Object.fromEntries(live.map(([key, entry]) => [key, entry.value ?? null])); if (shape === 'set') return live.map(([key]) => key).sort(); - return entries[SINGLE]?.isDeleted ? null : entries[SINGLE]?.value ?? null; + return own(entries, SINGLE)?.isDeleted ? null : own(entries, SINGLE)?.value ?? null; } export const recordPlain = (shape: SyncShape, record: SyncedRecord | undefined): Json | undefined => @@ -65,7 +76,7 @@ export function stampRecord(shape: SyncShape, baseline: SyncedRecord | undefined const entries: Record = {}; let isChanged = !baseline || !!baseline.deletedHlc; for (const [key, value] of Object.entries(current)) { - const old = previous[key]; + const old = own(previous, key); const isKept = !!old && !old.isDeleted && isSameJson(old.value, value); entries[key] = isKept ? old : { hlc: tick(), value }; isChanged ||= !isKept; @@ -83,7 +94,7 @@ const isSameEntry = (left: SyncEntry | undefined, right: SyncEntry | undefined) const isSameRecord = (left: SyncedRecord | undefined, right: SyncedRecord | undefined) => left === right || (!!left && !!right && left.deletedHlc === right.deletedHlc && Object.keys(left.entries).length === Object.keys(right.entries).length && - Object.entries(left.entries).every(([key, entry]) => isSameEntry(entry, right.entries[key]))); + Object.entries(left.entries).every(([key, entry]) => isSameEntry(entry, own(right.entries, key)))); const isSameValue = (left: SyncEntry, right: SyncEntry) => !!left.isDeleted === !!right.isDeleted && (!!left.isDeleted || isSameJson(left.value, right.value)); @@ -91,7 +102,7 @@ const isSameValue = (left: SyncEntry, right: SyncEntry) => !!left.isDeleted === function latestChange(base: SyncedRecord | undefined, side: SyncedRecord): string { if (side.deletedHlc) return side.deletedHlc; const previous = base && !base.deletedHlc ? base.entries : {}; - return Object.entries(side.entries).reduce((latest, [key, entry]) => !isSameEntry(previous[key], entry) && entry.hlc > latest ? entry.hlc : latest, ''); + return Object.entries(side.entries).reduce((latest, [key, entry]) => !isSameEntry(own(previous, key), entry) && entry.hlc > latest ? entry.hlc : latest, ''); } function combineLists(winner: Json, loser: Json, base: Json | undefined): readonly Json[] { @@ -124,10 +135,10 @@ export function mergeRecords(shape: SyncShape, base: SyncedRecord | undefined, l const conflicts: MergeConflict[] = []; const copies: Json[] = []; for (const key of new Set([...Object.keys(previous), ...Object.keys(local.entries), ...Object.keys(remote.entries)])) { - const original = previous[key]; + const original = own(previous, key); // An entry pruned on one side after its deletion reads as the oldest possible deletion. - const mine = local.entries[key] ?? (original ? { hlc: '', isDeleted: true as const } : undefined); - const theirs = remote.entries[key] ?? (original ? { hlc: '', isDeleted: true as const } : undefined); + const mine = own(local.entries, key) ?? (original ? { hlc: '', isDeleted: true as const } : undefined); + const theirs = own(remote.entries, key) ?? (original ? { hlc: '', isDeleted: true as const } : undefined); const isMineChanged = !isSameEntry(original, mine); const isTheirsChanged = !isSameEntry(original, theirs); let merged: SyncEntry | undefined; @@ -151,6 +162,19 @@ export function mergeRecords(shape: SyncShape, base: SyncedRecord | undefined, l return { merged: { entries }, conflicts, copies }; } +// The first merge of a computer joining synced data: it keeps what only it has, and wherever +// both sides hold a value the synced one wins. Text still keeps the local version as a copy. +export function preferRemote(shape: SyncShape, local: SyncedRecord, remote: SyncedRecord, tick: () => string): MergeResult { + if (shape === 'text') return mergeRecords(shape, undefined, local, remote, tick); + if (shape === 'value') return { merged: remote, conflicts: [], copies: [] }; + if (shape === 'list') { + const [mine, theirs] = [own(local.entries, SINGLE), own(remote.entries, SINGLE)]; + if (!mine || mine.isDeleted || !theirs || theirs.isDeleted) return { merged: remote, conflicts: [], copies: [] }; + return { merged: { entries: { [SINGLE]: { hlc: tick(), value: combineLists(theirs.value ?? null, mine.value ?? null, undefined) } } }, conflicts: [], copies: [] }; + } + return { merged: { entries: { ...local.entries, ...remote.entries } }, conflicts: [], copies: [] }; +} + export function pruneTombstones(record: SyncedRecord, before: string): SyncedRecord { const entries = Object.entries(record.entries); const kept = entries.filter(([, entry]) => !entry.isDeleted || entry.hlc >= before); @@ -165,6 +189,7 @@ export function parseSyncedRecord(value: unknown): SyncedRecord { if (!entries || typeof entries !== 'object' || Array.isArray(entries)) throw new Error('Sync record has no entries.'); const parsed: Record = {}; for (const [key, entry] of Object.entries(entries as Record)) { + if (key === UNSAFE_KEY) throw new Error('Sync record entry key is not allowed.'); const { hlc, value: entryValue, isDeleted } = (entry ?? {}) as { hlc?: unknown; value?: Json; isDeleted?: unknown }; if (!isHlc(hlc) || (isDeleted !== undefined && isDeleted !== true) || (isDeleted === true) === (entryValue !== undefined)) throw new Error(`Sync record entry ${key} is invalid.`); parsed[key] = isDeleted ? { hlc, isDeleted: true } : { hlc, value: entryValue }; diff --git a/electron/sync-renderer.test.ts b/electron/sync-renderer.test.ts index d1178ef..f4ca724 100644 --- a/electron/sync-renderer.test.ts +++ b/electron/sync-renderer.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; -import type { SyncSourceReply, SyncSourceRequest } from '../shared/sync'; +import { syncKindInfo, type SyncSourceReply, type SyncSourceRequest } from '../shared/sync'; import { SyncSourceUnavailableError } from './sync-engine'; import { RendererSources } from './sync-renderer'; @@ -62,3 +62,17 @@ test('a malformed snapshot is refused rather than read as empty', async () => { host.setReady(true); await assert.rejects(source('chart').snapshot(), /malformed/); }); + +test('a malformed reply rejects instead of hanging, and snapshots must be plain JSON', async () => { + const replies: unknown[] = [null, { result: { values: { c1: { at: new Date() } }, invalid: [] } }, { result: { values: { c1: { price: Number.NaN } }, invalid: [] } }]; + let index = 0; + const host: RendererSources = new RendererSources({ send: (id) => { const reply = replies[index++]; queueMicrotask(() => host.receive(id, reply)); return true; } }); + host.setReady(true); + const chart = host.sources().find((source) => source.kind === 'chart')!; + for (let attempt = 0; attempt < 3; attempt++) await assert.rejects(chart.snapshot(), /malformed/); +}); + +test('no secret kind is ever served by the renderer', () => { + const host = new RendererSources({ send: () => true }); + for (const source of host.sources()) assert.equal(syncKindInfo(source.kind).isSecret, false); +}); diff --git a/electron/sync-renderer.ts b/electron/sync-renderer.ts index d3513c7..6d1c550 100644 --- a/electron/sync-renderer.ts +++ b/electron/sync-renderer.ts @@ -1,7 +1,8 @@ -import { rendererSyncKinds, type Json, type RendererSyncKind, type SyncSnapshot, type SyncSourceReply, type SyncSourceRequest } from '../shared/sync'; +import { rendererSyncKinds, syncKindInfo, type Json, type RendererSyncKind, type SyncSnapshot, type SyncSourceReply, type SyncSourceRequest } from '../shared/sync'; import { SyncSourceUnavailableError, type SyncSource } from './sync-engine'; const REQUEST_TIMEOUT_MS = 30_000; +const MAX_DEPTH = 64; export interface RendererSourcesDeps { // False when there is no window to ask. @@ -17,6 +18,15 @@ interface Waiting { const isObject = (value: unknown): value is object => value !== null && typeof value === 'object'; +// Structured clone carries Maps, Dates, NaN and cycles that JSON cannot; only plain JSON passes. +function isPlainJson(value: unknown, depth = 0): value is Json { + if (value === null || typeof value === 'string' || typeof value === 'boolean') return true; + if (typeof value === 'number') return Number.isFinite(value); + if (depth > MAX_DEPTH || typeof value !== 'object') return false; + if (Array.isArray(value)) return value.every((item) => isPlainJson(item, depth + 1)); + return Object.getPrototypeOf(value) === Object.prototype && Object.values(value).every((item) => isPlainJson(item, depth + 1)); +} + // The renderer's stores, reached over IPC. The renderer checks its own values when it takes a // snapshot, so the engine validating those again needs no round trip; only remote values do. export class RendererSources { @@ -35,17 +45,20 @@ export class RendererSources { for (const id of [...this.waiting.keys()]) this.fail(id, new SyncSourceUnavailableError('The window closed before it answered.')); } - receive(id: number, reply: SyncSourceReply): void { + receive(id: number, reply: unknown): void { const waiting = this.waiting.get(id); if (!waiting) return; this.waiting.delete(id); clearTimeout(waiting.timer); - if ('error' in reply) waiting.reject(new Error(reply.error)); - else waiting.resolve(reply.result); + const answer = reply as Partial | null; + if (!isObject(answer)) waiting.reject(new Error('The app window sent a malformed answer.')); + else if ('error' in answer) waiting.reject(new Error(String(answer.error))); + else waiting.resolve('result' in answer ? answer.result : undefined); } + // Secret kinds never go to the renderer, whatever the kind list says. sources(): SyncSource[] { - return rendererSyncKinds.map((kind) => this.source(kind)); + return rendererSyncKinds.filter((kind) => !syncKindInfo(kind).isSecret).map((kind) => this.source(kind)); } private source(kind: RendererSyncKind): SyncSource { @@ -53,7 +66,7 @@ export class RendererSources { kind, snapshot: async () => { const snapshot = await this.request({ method: 'snapshot', kind }) as SyncSnapshot; - if (!isObject(snapshot?.values) || Array.isArray(snapshot.values) || !Array.isArray(snapshot.invalid)) throw new Error(`The ${kind} snapshot is malformed.`); + if (!isObject(snapshot?.values) || Array.isArray(snapshot.values) || !Array.isArray(snapshot.invalid) || !isPlainJson(snapshot.values)) throw new Error(`The ${kind} snapshot is malformed.`); const invalid = new Set(snapshot.invalid); for (const [id, value] of Object.entries(snapshot.values)) { if (isObject(value)) (invalid.has(id) ? this.refused : this.checked).add(value); diff --git a/electron/sync-service.test.ts b/electron/sync-service.test.ts index 525c85d..5436f5c 100644 --- a/electron/sync-service.test.ts +++ b/electron/sync-service.test.ts @@ -11,7 +11,7 @@ import { newRootKey } from './sync-crypto'; import type { SyncSource } from './sync-engine'; import { FakeTimers, MemorySource } from './sync-engine-fixtures'; import { inspectRepository, updateKeyring, withDeviceWrap } from './sync-keyring'; -import { SyncService } from './sync-service'; +import { SyncService, type SyncServiceDeps } from './sync-service'; const REPO = 42; const cipher: PasswordCipher = { @@ -29,7 +29,7 @@ async function until(check: () => boolean): Promise { await settle(); } -async function computer(github: FakeGitHub, name: string, dir?: string, file?: string, sources: readonly SyncSource[] = []) { +async function computer(github: FakeGitHub, name: string, dir?: string, file?: string, sources: readonly SyncSource[] = [], extra: Partial = {}) { const folder = dir ?? await fs.mkdtemp(path.join(os.tmpdir(), 'openterm-sync-service-')); const timers = new FakeTimers(); const views: SyncView[] = []; @@ -38,7 +38,7 @@ async function computer(github: FakeGitHub, name: string, dir?: string, file?: s const service = new SyncService({ dir: folder, fetch: github.fetch, cipher, platform: 'linux', appVersion: '0.12.0', deviceName: name, send: (view) => views.push(view), journal: (message) => journal.push(message), openExternal: async (url) => { opened.push(url); }, chooseFile: async () => file ?? null, - now: () => 1_800_000_000_000 + timers.time, timers, pause: async () => undefined, passphraseCost: { N: 2 ** 14, r: 8, p: 1 }, sources: () => sources, + now: () => 1_800_000_000_000 + timers.time, timers, pause: async () => undefined, passphraseCost: { N: 2 ** 14, r: 8, p: 1 }, sources: () => sources, ...extra, }); await service.init(); const tick = async (ms = 0) => { @@ -57,8 +57,8 @@ async function signIn(device: Awaited>) { await until(() => device.view().userCode === null); } -async function setUpFirst(github: FakeGitHub, passphrase: string | null = null, sources: readonly SyncSource[] = []) { - const first = await computer(github, 'Office PC', undefined, undefined, sources); +async function setUpFirst(github: FakeGitHub, passphrase: string | null = null, sources: readonly SyncSource[] = [], extra: Partial = {}) { + const first = await computer(github, 'Office PC', undefined, undefined, sources, extra); await signIn(first); await first.service.handle('sync.chooseRepository', { id: REPO }); const view = await first.service.handle('sync.create', { passphrase }); @@ -388,3 +388,105 @@ test('a computer joining a repository with no layouts yet syncs without asking', assert.equal(second.view().attention, null); assert.equal(second.view().status, 'idle'); }); + +async function joinSecond(github: FakeGitHub, recoveryCode: string, sources: readonly SyncSource[], dir?: string) { + const second = await computer(github, 'Laptop', dir, undefined, sources); + await signIn(second); + await second.service.handle('sync.chooseRepository', { id: REPO }); + await second.service.handle('sync.join', { recoveryCode }); + await second.tick(); + return second; +} + +test('turning on one CID asks which to keep when the synced one differs, then adopts it', async () => { + const github = fakeGitHub(); + const officeCid = new MemorySource('identity'); + officeCid.data = { cid: { cid: 'a'.repeat(32), isUnique: true } }; + const { first, recoveryCode } = await setUpFirst(github, null, [officeCid]); + await first.service.handle('sync.setSharing', { isSecretsSynced: false, isIdentitySynced: true }); + await first.tick(); + assert.equal(first.view().isIdentitySynced, true); + const laptopCid = new MemorySource('identity'); + laptopCid.data = { cid: { cid: 'b'.repeat(32), isUnique: true } }; + const second = await joinSecond(github, recoveryCode, [laptopCid]); + await second.service.handle('sync.setSharing', { isSecretsSynced: false, isIdentitySynced: true }); + assert.deepEqual([second.view().isIdentityChoiceNeeded, second.view().isIdentitySynced], [true, false]); + await second.service.handle('sync.setSharing', { isSecretsSynced: false, isIdentitySynced: true, identityChoice: 'synced' }); + await second.tick(); + assert.deepEqual(laptopCid.data, officeCid.data); + assert.equal(second.view().isIdentityChoiceNeeded, false); +}); + +test('passwords stay on a computer until it opts in, and only computers that opted in receive them', async () => { + const github = fakeGitHub(); + const officeSecrets = new MemorySource('accountSecret'); + officeSecrets.data = { 'wss://broker.example/terminal|1': { secret: 'pw', savedAt: 1 } }; + const { first, recoveryCode } = await setUpFirst(github, null, [officeSecrets]); + const laptopSecrets = new MemorySource('accountSecret'); + const second = await joinSecond(github, recoveryCode, [laptopSecrets]); + await second.service.handle('sync.setSharing', { isSecretsSynced: true, isIdentitySynced: false }); + await second.tick(); + assert.deepEqual(laptopSecrets.data, {}); + await first.service.handle('sync.setSharing', { isSecretsSynced: true, isIdentitySynced: false }); + await first.tick(); + await second.service.handle('sync.syncNow', {}); + await second.tick(); + assert.deepEqual(laptopSecrets.data, officeSecrets.data); +}); + +test('the merge question survives a restart, and Merge keeps the synced settings', async () => { + const github = fakeGitHub(); + const office = new MemorySource('preferences'); + office.data = { workspace: { theme: 'dark' } }; + const { recoveryCode } = await setUpFirst(github, null, [office]); + const laptop = new MemorySource('preferences'); + laptop.data = { workspace: { theme: 'light', replayMode: 'allCharts' } }; + const second = await joinSecond(github, recoveryCode, [laptop]); + assert.equal(second.view().attention?.reason, 'join'); + second.service.shutdown(); + const restarted = await computer(github, 'Laptop', second.dir, undefined, [laptop]); + await restarted.tick(60_000); + assert.equal(restarted.view().attention?.reason, 'join'); + assert.deepEqual(laptop.data, { workspace: { theme: 'light', replayMode: 'allCharts' } }); + await restarted.service.handle('sync.resolve', { choice: 'merge' }); + await restarted.tick(); + assert.deepEqual(laptop.data, { workspace: { theme: 'dark', replayMode: 'allCharts' } }); + assert.equal(restarted.view().attention, null); +}); + +test('an account open on one computer shows there, and moving it here logs that computer out', async () => { + const github = fakeGitHub(); + const account = 'wss://broker.example/terminal|1234'; + let live = [{ id: account, since: 0 }]; + const released: [readonly string[], string][] = []; + const { first, recoveryCode } = await setUpFirst(github, null, [], { + liveAccounts: () => live, + releaseAccounts: async (ids, to) => { released.push([ids, to]); live = []; }, + }); + first.service.presenceChanged(); + await first.tick(5000); + const second = await joinSecond(github, recoveryCode, []); + assert.deepEqual(await second.service.whereOpen(account), ['Office PC']); + assert.equal(await second.service.moveHere(account), false); + await first.service.handle('sync.syncNow', {}); + await first.tick(); + assert.deepEqual(released, [[[account], 'Laptop']]); + assert.deepEqual(await second.service.whereOpen(account), []); + assert.equal(await second.service.moveHere(account), true); +}); + +test('a login made after the move request is a choice to come back, so it stays', async () => { + const github = fakeGitHub(); + const account = 'wss://broker.example/terminal|1234'; + let live = [{ id: account, since: 0 }]; + const released: string[] = []; + const { first, recoveryCode } = await setUpFirst(github, null, [], { liveAccounts: () => live, releaseAccounts: async (ids) => { released.push(...ids); } }); + first.service.presenceChanged(); + await first.tick(5000); + const second = await joinSecond(github, recoveryCode, []); + await second.service.moveHere(account); + live = [{ id: account, since: Number.MAX_SAFE_INTEGER }]; + await first.service.handle('sync.syncNow', {}); + await first.tick(); + assert.deepEqual(released, []); +}); diff --git a/electron/sync-service.ts b/electron/sync-service.ts index 807e268..76a2b0d 100644 --- a/electron/sync-service.ts +++ b/electron/sync-service.ts @@ -1,7 +1,7 @@ import { randomInt } from 'node:crypto'; import path from 'node:path'; import { - normalizeCodeGroup, passphraseProblem, rendererSyncKinds, splitRecordKey, type SyncAttention, type SyncAttentionReason, type SyncDeviceView, type SyncMassChange, type SyncMethod, type SyncMethods, + isSameJson, normalizeCodeGroup, passphraseProblem, recordKey, rendererSyncKinds, splitRecordKey, syncKindInfo, type SyncSharing, type SyncAttention, type SyncAttentionReason, type SyncDeviceView, type SyncMassChange, type SyncMethod, type SyncMethods, type SyncPage, type SyncPairingRequest, type SyncPhase, type SyncRepositoryChoice, type SyncResolution, type SyncStatus, type SyncView, } from '../shared/sync'; import type { PasswordCipher } from './account-registry'; @@ -14,9 +14,12 @@ import { answerPairing, FORMAT_PATH, inspectRepository, newKeyring, PAIRING_LIFETIME_MS, pairingConfirmation, pairingWords, requestPairing, SyncSetupError, unlockKeyring, updateKeyring, withDeviceWrap, withNewRecovery, type RepositorySetup, } from './sync-keyring'; -import { MANIFEST_PATH, openManifest, SyncIntegrityError } from './sync-manifest'; +import { IDENTITY_ID } from './sync-main-sources'; +import { + HANDOFF_LIFETIME_MS, isPresenceDue, liveHandoffs, MANIFEST_PATH, openManifest, presentElsewhere, SyncIntegrityError, withPresence, type Manifest, +} from './sync-manifest'; import { realTimers, SyncScheduler, type RunResult, type Timers } from './sync-scheduler'; -import { CredentialStore, EncryptedStateStore, type SyncTarget } from './sync-storage'; +import { CredentialStore, EncryptedStateStore, type PendingJoin, type SyncTarget } from './sync-storage'; const REPOSITORY_CHECK_MS = 3_600_000; const PAIRING_POLL_MS = 5000; @@ -24,6 +27,17 @@ const PAIRING_ATTEMPTS = 3; const CODE_GROUPS = 7; // A sync file has no repository id, so it gets a random one that binds its data the same way. const MAX_GROUP_ID = 2 ** 47; +const PRESENCE_DELAY_MS = 2000; +const PEEK_TIMEOUT_MS = 5000; +const HANDOFF_POLL_MS = 3000; +const HANDOFF_POLLS = 20; + +// An account with a live session here, and since when (local time). Background sessions report +// Infinity: they are not handed over until copiers and signals have leases. +export interface LiveAccount { + readonly id: string; + readonly since: number; +} export interface SyncServiceDeps { readonly dir: string; @@ -38,6 +52,9 @@ export interface SyncServiceDeps { readonly revealFile?: (file: string) => void; readonly chooseFile?: (mode: 'new' | 'open') => Promise; readonly sources?: () => readonly SyncSource[]; + readonly liveAccounts?: () => readonly LiveAccount[]; + // Another computer moved these accounts to itself: log out of them here. + readonly releaseAccounts?: (accountIds: readonly string[], to: string) => Promise; readonly now?: () => number; readonly timers?: Timers; readonly pause?: (ms: number, signal?: AbortSignal) => Promise; @@ -50,6 +67,13 @@ const heldMessage = (changes: readonly SyncMassChange[]) => changes.some((change const joinMessage = 'This computer has layouts, drawings and lists of its own, and the synced data has some too. Merge adds this computer\'s to the synced ones on every computer; Use synced data replaces this computer\'s with the synced ones.'; +// A joining computer's first merge: the synced value wins where both hold one. Use synced data +// replaces layouts and lists; saved accounts and their passwords always merge. +const joinOptions: Record, SyncOptions> = { + merge: { isRemotePreferred: true }, + useSynced: { isRemotePreferred: true, isLocalDiscarded: true, isMassChangeAllowed: true, discardedKinds: rendererSyncKinds }, +}; + const groupIdOf = (target: SyncTarget) => target.kind === 'github' ? target.repository.id : target.groupId; const labelOf = (target: SyncTarget) => target.kind === 'github' ? target.repository.fullName : target.path; @@ -74,6 +98,10 @@ export class SyncService { private pairingRequests: SyncPairingRequest[] = []; private pairingConfirmation: string[] = []; private pairingAttempts = 0; + private isIdentityChoiceNeeded = false; + private presenceTimer: unknown = null; + private hasPresence = false; + private readonly answeredHandoffs = new Set(); // A key that arrived for this computer, held until the confirmation words prove it is the real one. private pairedKey: { readonly rootKey: Buffer; readonly epoch: number } | null = null; // The public key behind each request whose words were shown, so approval binds to what was checked. @@ -110,6 +138,7 @@ export class SyncService { this.phase = 'on'; this.startEngine(target); if (target.kind === 'github' && !this.auth.isSignedIn) this.attend('reconnect', 'Reconnect GitHub to keep syncing.'); + else if (this.credentials.get().pendingJoin === 'ask') this.attend('join', joinMessage); else this.scheduler.start(); } else if (target && (target.kind === 'file' || this.auth.isSignedIn)) { [this.choice, this.phase] = [target, 'joining']; @@ -122,6 +151,7 @@ export class SyncService { shutdown(): void { this.scheduler.stop(); + this.timers.clear(this.presenceTimer); this.stopPairingPoll(); this.signIn?.abort(); } @@ -132,6 +162,54 @@ export class SyncService { this.scheduler.localChange(); } + // A session started or ended here; presence is written shortly after, without the stores. + presenceChanged(): void { + if (!this.engine || this.presenceTimer) return; + this.presenceTimer = this.timers.set(() => { + this.presenceTimer = null; + void this.writePresence(); + }, PRESENCE_DELAY_MS); + } + + // Names of the other computers with a live session on the account, from the remote when it + // answers in time, else from the last sync. + async whereOpen(accountId: string): Promise { + const manifest = await this.currentManifest(); + if (!manifest || !this.engine) return []; + const ids = presentElsewhere(manifest.presence, accountId, this.credentials.get().deviceId, this.engine.serverNow()); + return ids.map((id) => manifest.devices.find((device) => device.id === id)?.name || 'another computer'); + } + + // Asks the computers holding the account to log out of it, and waits a minute for them to. + async moveHere(accountId: string): Promise { + const engine = this.engine; + if (!engine) return true; + const { deviceId } = this.credentials.get(); + const manifest = await this.currentManifest(); + const holders = manifest ? presentElsewhere(manifest.presence, accountId, deviceId, engine.serverNow()) : []; + if (!holders.length) return true; + await engine.editManifest((current, now) => ({ + ...current, handoffs: { ...liveHandoffs(current.handoffs, now), [accountId]: { from: deviceId, to: holders, createdAt: now, expiresAt: now + HANDOFF_LIFETIME_MS } }, + })); + const pause = this.deps.pause ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + for (let poll = 0; poll < HANDOFF_POLLS; poll++) { + await pause(HANDOFF_POLL_MS); + const latest = await engine.peekManifest().catch(() => null); + if (latest && !presentElsewhere(latest.presence, accountId, deviceId, engine.serverNow()).length) return true; + } + return false; + } + + // At quit: edits pushed, and this computer no longer listed as holding any account. + async leave(): Promise { + if (this.hasUnpushedChanges) await this.flush(); + if (this.hasPresence) await this.writePresence([]); + } + + get isLeaveNeeded(): boolean { + return this.hasUnpushedChanges || this.hasPresence; + } + get hasUnpushedChanges(): boolean { return !!this.engine && !this.attention && this.changedAt > 0 && this.changedAt >= this.pushedAt; } @@ -141,7 +219,8 @@ export class SyncService { if (!this.engine || this.attention) return; this.scheduler.stop(); await this.engine.recordLocalChanges().catch(() => undefined); - await this.run({}); + // Closing the window on macOS leaves the app running, so syncing carries on afterwards. + if ((await this.run({})).kind !== 'stop') this.scheduler.start(); } setFocused(isFocused: boolean): void { @@ -173,6 +252,7 @@ export class SyncService { else if (method === 'sync.syncNow') this.syncNow(); else if (method === 'sync.resolve') await this.resolve((params as SyncMethods['sync.resolve'][0]).choice); else if (method === 'sync.newRecoveryCode') await this.newRecoveryCode(); + else if (method === 'sync.setSharing') await this.setSharing(params as SyncMethods['sync.setSharing'][0]); else if (method === 'sync.disconnect') await this.disconnect(); else throw new Error(`Unknown sync method ${method}`); this.emit(); @@ -195,6 +275,8 @@ export class SyncService { repositories: this.repositories, pairingWords: this.pairingWords, pairingRequests: this.pairingRequests, canKeepSecrets: this.credentials.canKeepSecrets, isPairingApproved: !!this.pairedKey, pairingConfirmation: this.pairingConfirmation, isRecoveryConfirmed: credentials.isRecoveryConfirmed, + isSecretsSynced: credentials.isSecretsSynced && this.credentials.canKeepSecrets, isIdentitySynced: credentials.isIdentitySynced, + isIdentityChoiceNeeded: this.isIdentityChoiceNeeded, }; } @@ -377,8 +459,10 @@ export class SyncService { await this.credentials.update({ target, rootKey, deviceKeys, epoch, isRecoveryConfirmed: true }); [this.phase, this.pairingWords] = ['on', []]; this.startEngine(target); - if (await this.hasSyncedContent(target, rootKey, epoch)) this.attend('join', joinMessage); - else this.scheduler.start(); + if (await this.hasSyncedContent(target, rootKey, epoch)) { + await this.credentials.update({ pendingJoin: 'ask' }); + this.attend('join', joinMessage); + } else this.scheduler.start(); } // A computer joining for the first time asks before its own data meets the synced data. When the @@ -533,6 +617,12 @@ export class SyncService { } private async resolve(choice: SyncResolution): Promise { + if (this.attention?.reason === 'join' || this.credentials.get().pendingJoin === 'ask') { + await this.credentials.update({ pendingJoin: choice === 'useSynced' ? 'useSynced' : 'merge' }); + this.attention = null; + if ((await this.run({})).kind !== 'stop') this.scheduler.start(); + return; + } const options: Record = { merge: {}, allowDeletions: { isMassChangeAllowed: true }, @@ -565,11 +655,96 @@ export class SyncService { this.choice = null; this.stateStore = new EncryptedStateStore(path.join(this.deps.dir, 'sync-state.json'), keys.state, { epoch, repoId }); this.engine = new SyncEngine({ - epoch, repoId, keys, remote: this.remoteFor(target), store: this.stateStore, sources: this.deps.sources?.() ?? [], now: this.now, + epoch, repoId, keys, remote: this.remoteFor(target), store: this.stateStore, sources: this.engineSources(), now: this.now, device: { id: deviceId, name: this.deps.deviceName.slice(0, 256), platform: this.deps.platform, publicKey: deviceKeys?.publicKey ?? '', appVersion: this.deps.appVersion }, }); } + // Kinds this computer opted into. Secret kinds also need the keychain; while the synced CID is + // being adopted, this computer offers none of its own, so the synced one is taken as it is. + private engineSources(): SyncSource[] { + const { isSecretsSynced, isIdentitySynced, isIdentityAdopting } = this.credentials.get(); + return (this.deps.sources?.() ?? []).flatMap((source): SyncSource[] => { + if (source.kind === 'identity') { + if (!isIdentitySynced) return []; + return [isIdentityAdopting ? { kind: source.kind, snapshot: async () => ({}), validate: (value, id) => source.validate(value, id), apply: (change) => source.apply(change) } : source]; + } + return syncKindInfo(source.kind).isSecret && !(isSecretsSynced && this.credentials.canKeepSecrets) ? [] : [source]; + }); + } + + // Turning on one CID when another one is already synced asks which to keep, unless the choice + // came with the request. Each change restarts the engine with the kinds now chosen. + private async setSharing({ isSecretsSynced, isIdentitySynced, identityChoice }: SyncSharing): Promise { + if (!this.engine) throw new Error('Sync is not set up on this computer.'); + if (isSecretsSynced && !this.credentials.canKeepSecrets) throw new Error('This computer has no secure storage for passwords, so they cannot sync here.'); + const current = this.credentials.get(); + let isIdentityAdopting = current.isIdentityAdopting && isIdentitySynced; + if (isIdentitySynced && !current.isIdentitySynced) { + const source = this.deps.sources?.().find((candidate) => candidate.kind === 'identity'); + if (!source) throw new Error('This computer cannot sync its CID.'); + const synced = await this.engine.peek(recordKey('identity', IDENTITY_ID)); + const own = (await source.snapshot())[IDENTITY_ID]; + const isDifferent = synced !== undefined && !isSameJson(synced, own); + if (isDifferent && !identityChoice) { + this.isIdentityChoiceNeeded = true; + return; + } + isIdentityAdopting = isDifferent && identityChoice === 'synced'; + } + this.isIdentityChoiceNeeded = false; + this.scheduler.stop(); + await this.running?.catch(() => undefined); + await this.credentials.update({ isSecretsSynced, isIdentitySynced, isIdentityAdopting }); + this.startEngine(this.target()); + if (!this.attention) this.scheduler.start(); + } + + // The synced CID has arrived once it is a merge base; from then on this computer's CID syncs. + private async finishAdopting(): Promise { + if (!this.credentials.get().isIdentityAdopting) return; + const state = await this.stateStore?.load(); + if (!state?.bases[recordKey('identity', IDENTITY_ID)]) return; + await this.credentials.update({ isIdentityAdopting: false }); + this.startEngine(this.target()); + } + + private async currentManifest(): Promise { + if (!this.engine) return null; + const late = new Promise((resolve) => this.timers.set(() => resolve(null), PEEK_TIMEOUT_MS)); + const fresh = await Promise.race([this.engine.peekManifest().catch(() => null), late]); + return fresh ?? (await this.stateStore?.load())?.manifest ?? null; + } + + private async writePresence(accounts = (this.deps.liveAccounts?.() ?? []).map((account) => account.id)): Promise { + if (!this.engine || this.attention) return; + const { deviceId } = this.credentials.get(); + const isWritten = await this.engine.editManifest((manifest, now) => + isPresenceDue(manifest.presence, deviceId, accounts, now) ? { ...manifest, presence: withPresence(manifest.presence, deviceId, accounts, now) } : null).catch(() => false); + if (isWritten || !accounts.length) this.hasPresence = accounts.length > 0; + } + + // Requests from computers that moved an account to themselves after this one logged in to it. + // A login here made after the request was a choice to come back, so it stays. + private async answerHandoffs(): Promise { + const engine = this.engine; + const manifest = (await this.stateStore?.load())?.manifest; + if (!engine || !manifest || !this.deps.releaseAccounts) return; + const { deviceId } = this.credentials.get(); + const offset = engine.serverNow() - this.now(); + const live = new Map((this.deps.liveAccounts?.() ?? []).map((account) => [account.id, account.since + offset])); + const byComputer = new Map(); + for (const [account, request] of Object.entries(liveHandoffs(manifest.handoffs, engine.serverNow()))) { + const key = `${account}|${request.from}|${request.createdAt}`; + if (!request.to.includes(deviceId) || !(live.get(account)! < request.createdAt) || this.answeredHandoffs.has(key)) continue; + this.answeredHandoffs.add(key); + const name = manifest.devices.find((device) => device.id === request.from)?.name || 'another computer'; + byComputer.set(name, [...byComputer.get(name) ?? [], account]); + } + for (const [name, released] of byComputer) await this.deps.releaseAccounts(released, name); + if (byComputer.size) await this.writePresence(); + } + private async stamp(): Promise { await this.engine?.recordLocalChanges(); } @@ -583,6 +758,8 @@ export class SyncService { private async runOnce(options: SyncOptions): Promise { if (!this.engine) return { kind: 'stop' }; + const join = this.credentials.get().pendingJoin; + if (join === 'ask') return this.attend('join', joinMessage); const startedAt = this.now(); const target = this.target(); this.status = 'syncing'; @@ -590,14 +767,18 @@ export class SyncService { try { const problem = await this.checkRepository(target); if (problem) return this.attend(problem.reason, problem.message); - const outcome = await this.engine.sync(options); + const outcome = await this.engine.sync(join ? { ...options, ...joinOptions[join] } : options); if (outcome.status === 'held') return this.attend('held', heldMessage(outcome.changes), outcome.changes); if (outcome.status === 'busy') { this.status = 'idle'; return { kind: 'retry' }; } + if (join) await this.credentials.update({ pendingJoin: null }); + await this.answerHandoffs(); + await this.writePresence(); await this.refreshDevices(); await this.refreshPairing(target); + await this.finishAdopting(); this.pushedAt = startedAt + 1; [this.status, this.attention, this.error, this.conflicts, this.lastSyncedAt] = ['idle', null, null, outcome.conflicts.length, this.now()]; if (outcome.conflicts.length) this.deps.journal(`${outcome.conflicts.length} edit(s) made on two computers at once were settled in favour of the newer one.`); @@ -641,7 +822,7 @@ export class SyncService { private failed(error: Error): RunResult { if (error instanceof SyncIntegrityError) return this.attend(error.reason, error.message); if (error instanceof SyncSourceUnavailableError) { - this.status = 'idle'; + [this.status, this.error] = ['loading', error.message]; return { kind: 'retry' }; } if (error instanceof GitHubAuthError && error.reason === 'reconnect') return this.attend('reconnect', error.message); diff --git a/electron/sync-storage.ts b/electron/sync-storage.ts index 77b89b0..61e6584 100644 --- a/electron/sync-storage.ts +++ b/electron/sync-storage.ts @@ -24,8 +24,20 @@ export interface SyncCredentials { readonly deviceKeys: DeviceKeyPair | null; readonly target: SyncTarget | null; readonly isRecoveryConfirmed: boolean; + // This computer's opt-ins: saved passwords, API keys and webhook secrets; one CID for all. + readonly isSecretsSynced: boolean; + readonly isIdentitySynced: boolean; + // Set when turning on one CID chose the synced one: this computer offers none until it arrives. + readonly isIdentityAdopting: boolean; + // A first join into synced content: 'ask' until the user chooses, then the choice until a + // sync with it succeeds, so a restart in between neither skips the question nor the choice. + readonly pendingJoin: PendingJoin | null; } +export type PendingJoin = 'ask' | 'merge' | 'useSynced'; + +const pendingJoins: readonly PendingJoin[] = ['ask', 'merge', 'useSynced']; + interface StoredCredentials { readonly epoch: number; readonly deviceId: string; @@ -33,6 +45,10 @@ interface StoredCredentials { readonly secrets: string | null; readonly target: SyncTarget | null; readonly isRecoveryConfirmed: boolean; + readonly isSecretsSynced: boolean; + readonly isIdentitySynced: boolean; + readonly isIdentityAdopting: boolean; + readonly pendingJoin: PendingJoin | null; } interface Secrets { @@ -62,6 +78,10 @@ function parseStored(value: unknown): StoredCredentials { secrets: typeof stored.secrets === 'string' ? stored.secrets : null, target: parseTarget(stored.target), isRecoveryConfirmed: stored.isRecoveryConfirmed !== false, + isSecretsSynced: stored.isSecretsSynced === true, + isIdentitySynced: stored.isIdentitySynced === true, + isIdentityAdopting: stored.isIdentityAdopting === true, + pendingJoin: pendingJoins.find((join) => join === stored.pendingJoin) ?? null, }; } @@ -111,7 +131,10 @@ export class CredentialStore { // Forgets everything but this computer's id, so it stays one device to the repository. async clear(): Promise { const { deviceId } = this.get(); - this.current = { epoch: 1, deviceId, login: null, rootKey: null, tokens: null, deviceKeys: null, target: null, isRecoveryConfirmed: true }; + this.current = { + epoch: 1, deviceId, login: null, rootKey: null, tokens: null, deviceKeys: null, target: null, isRecoveryConfirmed: true, + isSecretsSynced: false, isIdentitySynced: false, isIdentityAdopting: false, pendingJoin: null, + }; await this.write(this.current); return this.current; } diff --git a/shared/sync.ts b/shared/sync.ts index fae33d4..ec4e380 100644 --- a/shared/sync.ts +++ b/shared/sync.ts @@ -98,6 +98,7 @@ export const syncStatuses = [ { key: 'syncing', label: 'Syncing' }, { key: 'offline', label: 'Offline' }, { key: 'waiting', label: 'Waiting for GitHub' }, + { key: 'loading', label: 'Waiting for the app window' }, { key: 'attention', label: 'Needs attention' }, ] as const; @@ -167,12 +168,26 @@ export interface SyncView { readonly pairingConfirmation: readonly string[]; readonly canKeepSecrets: boolean; readonly isRecoveryConfirmed: boolean; + // Opt-ins of this computer: saved passwords, API keys and webhook secrets; one CID for all. + readonly isSecretsSynced: boolean; + readonly isIdentitySynced: boolean; + // Turning on one CID found a different one already synced: which to keep waits for the user. + readonly isIdentityChoiceNeeded: boolean; } export type SyncPage = 'install' | 'verification' | 'authorizations' | 'repository' | 'newRepository'; export type SyncResolution = 'merge' | 'allowDeletions' | 'useSynced' | 'acceptHistory'; +// Which CID to keep when this computer's and the synced one differ. +export type SyncIdentityChoice = 'synced' | 'this'; + +export interface SyncSharing { + readonly isSecretsSynced: boolean; + readonly isIdentitySynced: boolean; + readonly identityChoice?: SyncIdentityChoice; +} + export interface SyncMethods { 'sync.view': [Record, SyncView]; 'sync.connect': [Record, SyncView]; @@ -192,6 +207,7 @@ export interface SyncMethods { 'sync.syncNow': [Record, SyncView]; 'sync.resolve': [{ choice: SyncResolution }, SyncView]; 'sync.newRecoveryCode': [Record, SyncView]; + 'sync.setSharing': [SyncSharing, SyncView]; 'sync.disconnect': [Record, SyncView]; } diff --git a/src/components/sync/SyncDialog.tsx b/src/components/sync/SyncDialog.tsx index 2a0c7ba..0714ba8 100644 --- a/src/components/sync/SyncDialog.tsx +++ b/src/components/sync/SyncDialog.tsx @@ -3,7 +3,7 @@ import { passphraseProblem, syncKindInfo, syncStatuses, type SyncAttention, type import { requestSync, syncStore } from '../../lib/sync'; import { useStore } from '../../lib/store'; import { Dialog } from '../dialogs/Dialog'; -import { Section } from '../ui/Controls'; +import { Section, Switch } from '../ui/Controls'; function useAction() { const [error, setError] = useState(null); @@ -337,6 +337,40 @@ function DeviceList({ devices }: DeviceListProps) { ); } +// Secrets and the CID sync only on request, and only where the system keychain can hold them. +function SharingPanel({ view }: SyncStepProps) { + const { error, isBusy, run } = useAction(); + const share = (change: Partial) => void run('sync.setSharing', { isSecretsSynced: view.isSecretsSynced, isIdentitySynced: view.isIdentitySynced, ...change }); + return ( +
+ share({ isSecretsSynced })} + /> + share({ isIdentitySynced })} + /> + {view.isIdentityChoiceNeeded && ( +
+

Your other computers already share a different CID. Which one should all your computers use? A change applies from each computer's next login.

+
+ + +
+
+ )} + +
+ ); +} + function OnPanel({ view }: SyncStepProps) { const { error, isBusy, run } = useAction(); const [isConfirmingDisconnect, setIsConfirmingDisconnect] = useState(false); @@ -349,7 +383,7 @@ function OnPanel({ view }: SyncStepProps) { {view.attention && } {!view.isRecoveryConfirmed &&

The recovery code for this sync was never confirmed. Make a new one and save it.

} {view.status === 'offline' && view.error &&

{view.error} OpenTerm5 keeps your edits and tries again.

} -

Charts, drawings and other data start syncing in a coming update. For now this connects your computers and checks the encrypted {view.backend === 'file' ? 'sync file' : 'repository'}.

+

Layouts, charts, drawings, watchlists, templates, alerts, Pine scripts, saved accounts and settings stay the same on your computers. Panels, and which layout is in front, stay with each one.

@@ -379,7 +413,7 @@ function SyncStep({ view }: SyncStepProps) { case 'joining': return ; case 'pairing': return ; case 'savingCode': return ; - case 'on': return ; + case 'on': return <>; } } diff --git a/src/components/sync/SyncStatus.tsx b/src/components/sync/SyncStatus.tsx index 7e0dacc..2b9f898 100644 --- a/src/components/sync/SyncStatus.tsx +++ b/src/components/sync/SyncStatus.tsx @@ -3,7 +3,7 @@ import { syncStore } from '../../lib/sync'; import { useStore } from '../../lib/store'; import { openDialog } from '../../lib/workspace'; -const tones = { idle: 'is-good', syncing: '', offline: 'is-warn', waiting: 'is-warn', attention: 'is-bad' } as const; +const tones = { idle: 'is-good', syncing: '', offline: 'is-warn', waiting: 'is-warn', loading: 'is-warn', attention: 'is-bad' } as const; export function SyncStatus() { const view = useStore(syncStore, (state) => state.view); diff --git a/src/lib/sync-model.ts b/src/lib/sync-model.ts index 024075c..7b491b4 100644 --- a/src/lib/sync-model.ts +++ b/src/lib/sync-model.ts @@ -112,12 +112,13 @@ export function layoutDrawingsRecord(tab: ChartTab): Json { export function restoreLayoutDrawings(value: Json): Record { if (!isObject(value)) fail('Layout drawings must be a map.'); checkOrder(value); - const bySymbol: Record = {}; + // A Map, so an instrument named like an Object.prototype member is just a name. + const bySymbol = new Map(); for (const [id, item] of orderedItems(value)) { - if (!isObject(item) || typeof item.symbol !== 'string' || !item.symbol) fail(`Layout drawing ${id} has no instrument.`); - (bySymbol[item.symbol] ??= []).push(restoreDrawingItem(id, item.drawing)); + if (!isObject(item) || typeof item.symbol !== 'string' || !item.symbol || item.symbol === '__proto__') fail(`Layout drawing ${id} has no instrument.`); + bySymbol.set(item.symbol, [...bySymbol.get(item.symbol) ?? [], restoreDrawingItem(id, item.drawing)]); } - return bySymbol; + return Object.fromEntries(bySymbol); } export function restoreProfile(name: string, value: Json): ChartProfile {