diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c314e8..992a9ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Sync Across Computers now syncs your content**: layouts and their charts (symbol, timeframe, + indicators, chart settings and themes), every drawing (on a chart, shared across a layout, or on a + symbol for all layouts), chart profiles and templates, drawing templates and last-used looks, + imported themes, watchlists, favourite indicators, timeframes and drawing tools, custom colours, + price alerts, Pine scripts, and the theme, replay, Trade Assistant, Risk Ticket and risk settings. + Panels, docks, which layout and chart are in front, and one-click trading consent stay with each + computer. A computer joining data that already syncs asks whether to merge its own layouts in or + use the synced ones; either way your synced settings are kept. +- **Saved accounts sync too.** In Sync Across Computers, **Passwords and identity** lets each + computer also sync saved passwords, API keys and webhook secrets (the newest saved password wins), + and **Use one CID on all my computers**, which asks which CID to keep if they differ. +- **Logging in to an account that is open on another computer asks first**, and **Move it here** + has that computer log out of it, so the two sessions do not keep disconnecting each other. + ## [1.0.0] - 2026-11-04 ### Added diff --git a/electron/account-registry.test.ts b/electron/account-registry.test.ts index e8719d3..753aee3 100644 --- a/electron/account-registry.test.ts +++ b/electron/account-registry.test.ts @@ -99,3 +99,56 @@ test('profiles saved before platforms existed load as MetaTrader 5 accounts', as assert.equal(profile.platform, 'mt5'); assert.deepEqual(restarted.credentials(profile.id), { platform: 'mt5', login: 1234, password: 'fixture-secret', endpoint }); }); + +test('logging in again with the same saved password keeps when it was saved', async () => { + let stored: readonly { secretSavedAt?: number }[] = []; + const registry = new AccountRegistry(cipher, async (profiles) => { stored = profiles as typeof stored; }); + await registry.remember(account, endpoint, 'first', undefined, true); + const savedAt = stored[0].secretSavedAt!; + assert.ok(savedAt > 0); + await new Promise((resolve) => setTimeout(resolve, 5)); + await registry.remember(account, endpoint, 'first', undefined, true); + assert.equal(stored[0].secretSavedAt, savedAt); + await registry.remember(account, endpoint, 'changed', undefined, true); + assert.ok(stored[0].secretSavedAt! > savedAt); +}); + +test('synced profiles keep this computer\'s secret and latest use; ids come from the fields', async () => { + const registry = new AccountRegistry(cipher, async () => {}); + const profile = await registry.remember(account, endpoint, 'local-secret', 'Mine', true); + const synced = registry.syncedProfiles(); + assert.ok(!JSON.stringify(synced).includes('encrypted')); + assert.throws(() => registry.checkSyncedProfile('wss://other.example/terminal|1234', synced[profile.id]), /does not match/); + const other = { ...synced[profile.id], label: 'Renamed elsewhere', lastUsed: 1 }; + await registry.applySyncedProfiles({ upserts: { [profile.id]: other }, deletes: [] }); + assert.equal(registry.get(profile.id).label, 'Renamed elsewhere'); + assert.equal(registry.get(profile.id).lastUsed, profile.lastUsed); + assert.equal(registry.password(profile.id), 'local-secret'); + await registry.applySyncedProfiles({ upserts: {}, deletes: [profile.id] }); + assert.equal(registry.list().length, 0); +}); + +test('the newest saved secret wins, and one that no longer decrypts is not read as removed', async () => { + const registry = new AccountRegistry(cipher, async () => {}); + const profile = await registry.remember(account, endpoint, 'local', undefined, true); + const local = registry.syncedSecrets()[profile.id] as { secret: string; savedAt: number }; + assert.equal(local.secret, 'local'); + assert.throws(() => registry.checkSyncedSecret('wss://unknown.example/terminal|1', { secret: 'x', savedAt: 1 }), /does not have/); + await registry.applySyncedSecrets({ upserts: { [profile.id]: { secret: 'older', savedAt: local.savedAt - 1 } }, deletes: [] }); + assert.equal(registry.password(profile.id), 'local'); + await registry.applySyncedSecrets({ upserts: { [profile.id]: { secret: 'newer', savedAt: local.savedAt + 1 } }, deletes: [] }); + assert.equal(registry.password(profile.id), 'newer'); + const broken = new AccountRegistry({ ...cipher, decrypt: () => { throw new Error('keychain changed'); } }, async () => {}); + await broken.remember(account, endpoint, 'x', undefined, true); + assert.deepEqual(Object.values(broken.syncedSecrets()), [{ isUnreadable: true }]); + await registry.applySyncedSecrets({ upserts: {}, deletes: [profile.id] }); + assert.equal(registry.get(profile.id).isPasswordSaved, false); +}); + +test('a synced exchange secret must hold an API key and secret', async () => { + const registry = new AccountRegistry(cipher, async () => {}); + const bybit = { ...account, login: 4242, server: 'Bybit Demo trading', company: 'Bybit' }; + const profile = await registry.rememberExchange(bybit, { platform: 'bybit', environment: 'demo', apiKey: 'k', apiSecret: 's' }); + assert.throws(() => registry.checkSyncedSecret(profile.id, { secret: 'plain password', savedAt: 1 }), /API key/); + registry.checkSyncedSecret(profile.id, { secret: JSON.stringify({ apiKey: 'k2', apiSecret: 's2' }), savedAt: 1 }); +}); diff --git a/electron/account-registry.ts b/electron/account-registry.ts index f18e44e..10f8ec0 100644 --- a/electron/account-registry.ts +++ b/electron/account-registry.ts @@ -4,8 +4,25 @@ import type { SessionCredentials } from './session-pool'; interface StoredProfile extends Omit { readonly encryptedPassword?: string; + // When the saved secret last changed, so the newest password wins between computers. + readonly secretSavedAt?: number; } +// A profile as it syncs: everything but the secret, which syncs apart behind its own opt-in. +export type SyncedProfile = Omit; + +export interface SyncedSecret { + readonly secret: string; + readonly savedAt: number; +} + +export interface ProfileChange { + readonly upserts: Readonly>; + readonly deletes: readonly string[]; +} + +const MAX_SECRET_LENGTH = 8192; + export const exchangeAccountId = (platform: ExchangePlatform, environment: string, login: number) => `${platform}:${environment}|${login}`; export const platformOf = (accountId: string): Platform => platforms.find((item) => item.key !== 'mt5' && accountId.startsWith(`${item.key}:`))?.key ?? 'mt5'; @@ -24,28 +41,123 @@ export class AccountRegistry { load(value: unknown): void { if (!Array.isArray(value) || value.length > 1000) throw new Error('Invalid saved accounts.'); - const profiles = value.map((item): StoredProfile => { - if (!item || typeof item !== 'object') throw new Error('Invalid saved account.'); - const { login, name, label, server, company, tradeMode, lastUsed, encryptedPassword } = item; - const platform = platforms.find((candidate) => candidate.key === (item.platform ?? 'mt5'))?.key; - if (!platform) throw new Error('Invalid saved account.'); - const webEndpoint = isExchangePlatform(platform) ? this.environmentOf(platform, item.webEndpoint).restUrl : parseSettings(item).webEndpoint; - if (!Number.isSafeInteger(login) || login <= 0 || !['demo', 'real', 'contest'].includes(tradeMode) || - ![name, label, server, company].every((field) => typeof field === 'string' && field.length <= 512) || - !Number.isFinite(lastUsed) || (encryptedPassword !== undefined && typeof encryptedPassword !== 'string')) { - throw new Error('Invalid saved account.'); + const profiles = value.map((item) => this.parseProfile(item)); + if (new Set(profiles.map((profile) => profile.id)).size !== profiles.length) throw new Error('Duplicate saved accounts.'); + this.profiles = profiles; + } + + // The id is derived from the fields, never read, so a profile cannot claim another's id. + private parseProfile(item: unknown): StoredProfile { + if (!item || typeof item !== 'object' || Array.isArray(item)) throw new Error('Invalid saved account.'); + const { login, name, label, server, company, tradeMode, lastUsed, encryptedPassword, secretSavedAt, ...rest } = item as Record; + const platform = platforms.find((candidate) => candidate.key === (rest.platform ?? 'mt5'))?.key; + if (!platform) throw new Error('Invalid saved account.'); + const webEndpoint = isExchangePlatform(platform) ? this.environmentOf(platform, rest.webEndpoint).restUrl : parseSettings(item as { webEndpoint?: string }).webEndpoint; + if (!Number.isSafeInteger(login) || (login as number) <= 0 || !['demo', 'real', 'contest'].includes(tradeMode as string) || + ![name, label, server, company].every((field) => typeof field === 'string' && field.length <= 512) || + !Number.isFinite(lastUsed) || (encryptedPassword !== undefined && typeof encryptedPassword !== 'string') || + (secretSavedAt !== undefined && !(Number.isFinite(secretSavedAt) && (secretSavedAt as number) >= 0))) { + throw new Error('Invalid saved account.'); + } + const id = isExchangePlatform(platform) ? exchangeAccountId(platform, this.environmentOf(platform, webEndpoint).key, login as number) : this.key(webEndpoint, login as number); + return { + id, platform, login: login as number, name: name as string, label: label as string, server: server as string, company: company as string, webEndpoint, + tradeMode: tradeMode as AccountProfile['tradeMode'], lastUsed: lastUsed as number, + ...(encryptedPassword ? { encryptedPassword: encryptedPassword as string, ...(secretSavedAt !== undefined ? { secretSavedAt: secretSavedAt as number } : {}) } : {}), + }; + } + + // Keeps the saved time of a secret that did not change, so logging in again does not make an + // old password look like the newest one. + private secretFields(existing: StoredProfile | undefined, secret: string): Pick { + const isSame = !!existing?.encryptedPassword && this.tryDecrypt(existing.encryptedPassword) === secret; + return { encryptedPassword: this.cipher.encrypt(secret), secretSavedAt: isSame ? existing!.secretSavedAt ?? 0 : Date.now() }; + } + + private tryDecrypt(value: string): string | null { + try { return this.cipher.decrypt(value); } catch { return null; } + } + + syncedProfiles(): Record { + return Object.fromEntries(this.profiles.map(({ id, encryptedPassword, secretSavedAt, ...profile }) => [id, profile])); + } + + checkSyncedProfile(id: string, value: unknown): void { + if (this.parseProfile(value).id !== id) throw new Error('The account does not match its id.'); + } + + // Applies profiles from other computers. A profile keeps this computer's saved secret, and the + // most recent use of either side. + async applySyncedProfiles(change: ProfileChange): Promise { + const removed = new Set(change.deletes); + const profiles = this.profiles.filter((profile) => !removed.has(profile.id)); + for (const [id, value] of Object.entries(change.upserts)) { + const { encryptedPassword, secretSavedAt, ...parsed } = this.parseProfile(value); + if (parsed.id !== id) throw new Error('The account does not match its id.'); + const index = profiles.findIndex((profile) => profile.id === id); + const existing = profiles[index]; + const profile: StoredProfile = existing + ? { ...parsed, lastUsed: Math.max(parsed.lastUsed, existing.lastUsed), ...(existing.encryptedPassword ? { encryptedPassword: existing.encryptedPassword, secretSavedAt: existing.secretSavedAt } : {}) } + : parsed; + if (index < 0) profiles.push(profile); + else profiles[index] = profile; + } + await this.persist(profiles); + this.profiles = profiles; + for (const id of removed) this.passwords.delete(id); + } + + // Saved secrets in plain text, for sync to encrypt under its own key. One that no longer + // decrypts here is listed as unreadable, so it is left alone rather than read as removed. + syncedSecrets(): Record { + return Object.fromEntries(this.profiles.flatMap(({ id, encryptedPassword, secretSavedAt }) => { + if (!encryptedPassword) return []; + const secret = this.tryDecrypt(encryptedPassword); + return [[id, secret === null ? { isUnreadable: true } as const : { secret, savedAt: secretSavedAt ?? 0 }]]; + })); + } + + // A secret applies only to a profile already here (profiles sync first), and an exchange + // secret must hold an API key and secret. + checkSyncedSecret(id: string, value: unknown): void { + const { secret, savedAt } = (value ?? {}) as Partial; + const profile = this.profiles.find((candidate) => candidate.id === id); + if (!profile) throw new Error('The secret belongs to an account this computer does not have yet.'); + if (typeof secret !== 'string' || !secret || secret.length > MAX_SECRET_LENGTH || !Number.isFinite(savedAt) || savedAt! < 0) throw new Error('Invalid synced secret.'); + if (!isExchangePlatform(profile.platform)) return; + const parsed = (() => { try { return JSON.parse(secret) as unknown; } catch { return null; } })() as { apiKey?: unknown; apiSecret?: unknown } | null; + if (typeof parsed?.apiKey !== 'string' || typeof parsed.apiSecret !== 'string' || !parsed.apiKey || !parsed.apiSecret) throw new Error('Invalid synced API key.'); + } + + // A secret saved here more recently than the synced one is kept; sync then sends it out, so + // every computer ends with the newest password. + async applySyncedSecrets(change: ProfileChange): Promise { + if (!this.cipher.canEncrypt()) throw new Error('Secure storage is unavailable, so synced passwords cannot be kept here.'); + const removed = new Set(change.deletes); + const profiles = this.profiles.map((profile): StoredProfile => { + const value = change.upserts[profile.id]; + if (value !== undefined) { + this.checkSyncedSecret(profile.id, value); + const { secret, savedAt } = value as SyncedSecret; + const isLocalNewer = !!profile.encryptedPassword && (profile.secretSavedAt ?? 0) > savedAt && this.tryDecrypt(profile.encryptedPassword) !== null; + return isLocalNewer ? profile : { ...profile, encryptedPassword: this.cipher.encrypt(secret), secretSavedAt: savedAt }; } - const id = isExchangePlatform(platform) ? exchangeAccountId(platform, this.environmentOf(platform, webEndpoint).key, login) : this.key(webEndpoint, login); - return { id, platform, login, name, label, server, company, webEndpoint, tradeMode, lastUsed, ...(encryptedPassword ? { encryptedPassword } : {}) }; + if (!removed.has(profile.id) || !profile.encryptedPassword) return profile; + const { encryptedPassword, secretSavedAt, ...rest } = profile; + return rest; }); - if (new Set(profiles.map((profile) => profile.id)).size !== profiles.length) throw new Error('Duplicate saved accounts.'); + await this.persist(profiles); this.profiles = profiles; + for (const profile of profiles) { + if (change.upserts[profile.id] !== undefined && profile.encryptedPassword) this.passwords.set(profile.id, this.cipher.decrypt(profile.encryptedPassword)); + if (removed.has(profile.id)) this.passwords.delete(profile.id); + } } canSavePasswords(): boolean { return this.cipher.canEncrypt(); } list(): AccountProfile[] { - return this.profiles.map(({ encryptedPassword, ...profile }) => ({ + return this.profiles.map(({ encryptedPassword, secretSavedAt, ...profile }) => ({ ...profile, hasPassword: this.passwords.has(profile.id) || !!encryptedPassword && this.cipher.canEncrypt(), isPasswordSaved: !!encryptedPassword, })).sort((left, right) => right.lastUsed - left.lastUsed); @@ -76,7 +188,7 @@ export class AccountRegistry { id, platform: 'mt5', name: account.name, label: label?.trim() || existing?.label || `${account.company} ${account.login}`, server: account.server, company: account.company, webEndpoint, tradeMode: account.tradeMode, login: account.login, lastUsed: Date.now(), - ...(isPasswordSaved ? { encryptedPassword: this.cipher.encrypt(password) } : {}), + ...(isPasswordSaved ? this.secretFields(existing, password) : {}), }; const profiles = [...this.profiles.filter((item) => item.id !== id), profile]; await this.persist(profiles); @@ -98,7 +210,7 @@ export class AccountRegistry { id, platform, name: account.name, label: label?.trim() || existing?.label || `${account.server} ${account.login}`, server: account.server, company: account.company, webEndpoint: restUrl, tradeMode: account.tradeMode, login: account.login, lastUsed: Date.now(), - ...(isPasswordSaved ? { encryptedPassword: this.cipher.encrypt(secret) } : {}), + ...(isPasswordSaved ? this.secretFields(existing, secret) : {}), }; const profiles = [...this.profiles.filter((item) => item.id !== id), profile]; await this.persist(profiles); diff --git a/electron/accounts.ts b/electron/accounts.ts index 0976958..d89c8c1 100644 --- a/electron/accounts.ts +++ b/electron/accounts.ts @@ -10,13 +10,15 @@ export const secretCipher: PasswordCipher = { decrypt: (value) => safeStorage.decryptString(Buffer.from(value, 'base64')), }; -export async function loadAccounts(): Promise { +// `onSaved` runs after every write, so sync hears about saved accounts. +export async function loadAccounts(onSaved: () => void = () => undefined): Promise { const file = path.join(app.getPath('userData'), 'accounts.json'); const registry = new AccountRegistry(secretCipher, async (profiles) => { await fs.mkdir(path.dirname(file), { recursive: true }); const temporary = `${file}.tmp`; await fs.writeFile(temporary, JSON.stringify(profiles, null, 2), { mode: 0o600 }); await fs.rename(temporary, file); + onSaved(); }); try { registry.load(JSON.parse(await fs.readFile(file, 'utf8'))); } catch (error) { diff --git a/electron/automation-host.ts b/electron/automation-host.ts index c5c19e7..3170e08 100644 --- a/electron/automation-host.ts +++ b/electron/automation-host.ts @@ -30,6 +30,27 @@ export interface HostDeps { send(event: E, data: AutomationEvents[E]): void; journal(message: string, source: string): void; saveFile(name: string, contents: string): Promise; + // After webhook settings or notification targets are saved, from here or from sync. + onSecretsSaved?(): void; +} + +// Webhook sources and notification targets as they sync. Whether a source is switched on stays +// with each computer, like the listener's address, port and switches: two computers holding +// broker sessions for one source's account would knock each other out. +export interface SyncedWebhooks { + readonly sources: readonly Omit[]; + readonly notifications: readonly NotificationTarget[]; +} + +function checkTargets(value: unknown): NotificationTarget[] { + const targets = value as NotificationTarget[]; + if (!Array.isArray(targets) || targets.length > 20) throw new Error('At most 20 notification targets'); + return targets.map((target) => { + if (!target || typeof target !== 'object' || !notificationKinds.some((kind) => kind.key === target.kind)) throw new Error('Unknown notification kind'); + const problem = validateTarget(target); + if (problem) throw new Error(`${target.name || 'Target'}: ${problem}`); + return { ...target, events: Array.isArray(target.events) ? target.events.filter((event) => notificationEvents.some((item) => item.key === event)) : [] }; + }); } const MAX_NEWS_MINUTES = 240; @@ -85,6 +106,8 @@ export class AutomationHost { readonly news: NewsCalendar; private settings: WebhookSettings = { host: '127.0.0.1', port: 8787, sources: [], isEnabled: false, isPaused: false }; private notifications: NotificationTarget[] = []; + // Settings that failed to load are not synced, or the empty defaults would replace everyone's. + private isSecretsLoaded = false; private copierPush?: NodeJS.Timeout; private readonly heldSources = new Map(); private readonly secrets: JsonStore; @@ -145,6 +168,7 @@ export class AutomationHost { this.settings = validateWebhookSettings(decode(stored.settings)); this.notifications = decode(stored.notifications); this.notifier.setTargets(this.notifications); + this.isSecretsLoaded = true; await this.webhooks.apply(this.settings); this.syncSourceSessions(); } catch (error) { @@ -192,6 +216,37 @@ export class AutomationHost { const isEncrypted = this.deps.cipher.canEncrypt(); const encode = (value: unknown) => isEncrypted ? this.deps.cipher.encrypt(JSON.stringify(value)) : JSON.stringify(value); await this.secrets.save({ isEncrypted, settings: encode(this.settings), notifications: encode(this.notifications) }); + this.isSecretsLoaded = true; + this.deps.onSecretsSaved?.(); + } + + // null while the saved settings are unreadable here. + syncedWebhooks(): SyncedWebhooks | null { + if (!this.isSecretsLoaded) return null; + return { sources: this.settings.sources.map(({ isEnabled, ...source }) => source), notifications: this.notifications }; + } + + private withSyncedSources(value: unknown): WebhookSettings { + const { sources } = (value ?? {}) as Partial; + if (!Array.isArray(sources)) throw new Error('Invalid synced webhook sources'); + const local = new Map(this.settings.sources.map((source) => [source.id, source.isEnabled])); + return validateWebhookSettings({ ...this.settings, sources: sources.map((source) => ({ ...source, isEnabled: local.get(source?.id) ?? false }) as WebhookSource) }); + } + + checkSyncedWebhooks(value: unknown): void { + this.withSyncedSources(value); + checkTargets((value as Partial).notifications); + } + + // A source new to this computer arrives switched off. + async applySyncedWebhooks(value: unknown): Promise { + const settings = this.withSyncedSources(value); + const notifications = checkTargets((value as Partial).notifications); + [this.settings, this.notifications] = [settings, notifications]; + this.notifier.setTargets(notifications); + await this.persistSecrets(); + await this.webhooks.apply(this.settings); + this.syncSourceSessions(); } canOpen(accountId: string | null, symbol?: string, lots?: number): Promise { @@ -279,14 +334,7 @@ export class AutomationHost { return this.webhooks.test(source, body); } case 'notifications.save': { - const targets = params as NotificationTarget[]; - if (!Array.isArray(targets) || targets.length > 20) throw new Error('At most 20 notification targets'); - for (const target of targets) { - if (!notificationKinds.some((kind) => kind.key === target.kind)) throw new Error('Unknown notification kind'); - const problem = validateTarget(target); - if (problem) throw new Error(`${target.name || 'Target'}: ${problem}`); - target.events = target.events.filter((event) => notificationEvents.some((item) => item.key === event)); - } + const targets = checkTargets(params); this.notifications = targets; this.notifier.setTargets(targets); await this.persistSecrets(); diff --git a/electron/device-identity.ts b/electron/device-identity.ts index 1a89eff..4001faa 100644 --- a/electron/device-identity.ts +++ b/electron/device-identity.ts @@ -69,6 +69,24 @@ export class DeviceIdentity { return true; } + // The CID as it syncs between computers that share one identity. + syncRecord(): { cid: string; isUnique: boolean } { + return { cid: this.clientId.toString('hex'), isUnique: !this.isWeak }; + } + + // Another computer's CID, from sync. Like a regenerated one, it applies from the next login. + async setFromSync(value: unknown): Promise { + const { cid, isUnique } = (value ?? {}) as { cid?: unknown; isUnique?: unknown }; + if (typeof isUnique !== 'boolean') throw new Error('Invalid client ID.'); + const parsed = parseClientId({ cid, ...(isUnique ? { version: SEED_VERSION } : {}) }); + if (this.current?.equals(parsed.cid) && this.isWeak === parsed.hasWeakSeed) return false; + await fs.mkdir(path.dirname(this.file), { recursive: true }); + await fs.writeFile(this.file, JSON.stringify({ cid: parsed.cid.toString('hex'), ...(isUnique ? { version: SEED_VERSION } : {}) }, null, 2), { mode: 0o600 }); + this.current = parsed.cid; + this.isWeak = parsed.hasWeakSeed; + return true; + } + async regenerate(): Promise { const next = deriveClientId(this.traits(), randomBytes(16).toString('hex')); await fs.mkdir(path.dirname(this.file), { recursive: true }); diff --git a/electron/github-remote.ts b/electron/github-remote.ts index 57f66ae..7884044 100644 --- a/electron/github-remote.ts +++ b/electron/github-remote.ts @@ -114,6 +114,10 @@ export class GitHubRemote implements SyncRemote { return `/repos/${this.repository.fullName}`; } + serverNow(): number { + return this.api.serverNow; + } + // Conditional on the last ETag (sent back verbatim: GitHub's are weak), so an unchanged head // costs no rate limit. An empty repository has no branch yet and reads as null. async head(): Promise { diff --git a/electron/main.ts b/electron/main.ts index dcc0336..24d8d10 100644 --- a/electron/main.ts +++ b/electron/main.ts @@ -34,7 +34,9 @@ import { autoUpdater, DebUpdater } from 'electron-updater'; import { UpdateService } from './updater'; import type { UpdateMethod, UpdateState } from '../shared/updates'; import type { SyncMethod, SyncView } from '../shared/sync'; -import { SyncService } from './sync-service'; +import { mainSyncSources } from './sync-main-sources'; +import { RendererSources } from './sync-renderer'; +import { SyncService, type LiveAccount } from './sync-service'; import { SYNC_FILE_EXTENSION } from './file-remote'; migrateLegacyProfile(); @@ -54,6 +56,14 @@ let automation: AutomationHost | undefined; let journaling: JournalingService | undefined; let updates: UpdateService | undefined; let sync: SyncService | undefined; +const syncSources = new RendererSources({ + send: (id, request) => { + if (!window || window.isDestroyed()) return false; + window.webContents.send('sync:source', id, request); + return true; + }, +}); +let isSyncFlushed = false; let isChanging = false; let isWindowLoaded = false; let uiAccount: Account | null = null; @@ -125,6 +135,51 @@ function leaveAccount(): void { uiAccount = null; journaling?.setActive(null); pool.afterUiChange(null, previous); + sync?.presenceChanged(); +} + +// When the account in use logged in here, so a move requested after that logs it out. +let uiLoginAt = 0; + +// Sessions this computer holds, for presence. Background sessions (copiers, signal sources) are +// listed too, but not handed over: copiers and signals get leases first. +function liveAccounts(): LiveAccount[] { + const background = pool.status().filter((link) => link.accountId !== activeAccount && (link.state === 'connected' || link.state === 'connecting')); + return [...(activeAccount && uiAccount ? [{ id: activeAccount, since: uiLoginAt }] : []), ...background.map((link) => ({ id: link.accountId, since: Number.POSITIVE_INFINITY }))]; +} + +async function releaseAccounts(accountIds: readonly string[], to: string): Promise { + if (!activeAccount || !accountIds.includes(activeAccount)) return; + const label = accounts?.list().find((profile) => profile.id === activeAccount)?.label || activeAccount.split('|').pop(); + await changeConnection(async () => { + if (!activeAccount || !accountIds.includes(activeAccount)) return; + leaveAccount(); + send('accountChanged', { id: null }); + await applySettings(); + }); + journal(`${label} moved to ${to}, so this computer logged out of it.`, 'Accounts'); +} + +async function ask(message: string, detail: string, buttons: readonly string[]): Promise { + const options = { type: 'question' as const, buttons: [...buttons], defaultId: 0, cancelId: buttons.length - 1, message, detail }; + const parent = window && !window.isDestroyed() && window.isVisible() ? window : undefined; + return (parent ? await dialog.showMessageBox(parent, options) : await dialog.showMessageBox(options)).response; +} + +// One MT5 account logged in from two computers can knock both sessions out, so a login to an +// account open elsewhere asks first, and can have the other computer log out of it. +async function handOff(accountId: string): Promise { + const holders = await sync?.whereOpen(accountId) ?? []; + if (!holders.length) return; + const label = accounts?.list().find((profile) => profile.id === accountId)?.label || accountId.split('|').pop(); + const where = holders.join(' and '); + const choice = await ask(`${label} is open on ${where}.`, `Move it here: ${where} logs out of it, then this computer logs in. Log in anyway: both stay logged in, which brokers may see as one account on two networks, and the two sessions can disconnect each other.`, ['Move it here', 'Log in anyway', 'Cancel']); + if (choice === 2) throw new Error('Login cancelled.'); + if (choice === 1) return; + journal(`Asking ${where} to log out of ${label}.`, 'Accounts'); + if (await sync!.moveHere(accountId)) return; + const again = await ask(`${where} has not logged out of ${label} yet.`, 'It may be asleep or offline, or running a copier or signal source on this account. Log in anyway, and it logs out of the account when it next syncs, unless a copier or signal source keeps it open.', ['Log in anyway', 'Cancel']); + if (again === 1) throw new Error('Login cancelled.'); } function sendSync(view: SyncView): void { @@ -170,6 +225,7 @@ function closePopouts(): void { } function createWindow(): void { + isSyncFlushed = false; window = new BrowserWindow({ width: 1440, height: 900, @@ -212,9 +268,20 @@ function createWindow(): void { : [{ role: 'copy' }]; Menu.buildFromTemplate(template).popup({ window }); }); - // A reload or a closed main window takes the pop-outs' page with it. - window.webContents.on('did-start-loading', closePopouts); + // A reload or a closed main window takes the pop-outs' page with it, and the stores sync reads. + window.webContents.on('did-start-loading', () => { + closePopouts(); + syncSources.setReady(false); + }); + // Edits not yet synced are pushed while the window can still be asked for them. + window.on('close', (event) => { + if (isSyncFlushed || !sync?.hasUnpushedChanges) return; + event.preventDefault(); + isSyncFlushed = true; + void Promise.race([sync.flush(), new Promise((resolve) => setTimeout(resolve, 5000))]).finally(() => window?.close()); + }); window.on('closed', () => { + syncSources.setReady(false); closePopouts(); window = undefined; if (process.platform !== 'darwin') app.quit(); @@ -336,12 +403,16 @@ function loginTarget(params: TerminalMethods['account.login'][0]): SignInTarget async function loginAccount(params: TerminalMethods['account.login'][0]) { if (params.isPasswordSaved && !accounts?.canSavePasswords()) throw new Error('Secure password storage is unavailable. Leave Save password unchecked.'); const next = parseSettings({ webEndpoint: params.webEndpoint ?? settings.webEndpoint }); + await handOff(`${next.webEndpoint}|${params.login}`); leaveAccount(); send('accountChanged', { id: null }); direct.use(mt5); mt5.disconnect(); // Each connection creates its client from device.clientId, so a new ID applies to this login. - if (params.isUniqueClientIdRequested && await device.ensureUnique()) journal('This computer now sends a unique client ID at login.', 'Accounts'); + if (params.isUniqueClientIdRequested && await device.ensureUnique()) { + journal('This computer now sends a unique client ID at login.', 'Accounts'); + sync?.noteLocalChange(); + } await pool.beforeUiLogin(`${next.webEndpoint}|${params.login}`); await saveSettings(next); settings = next; @@ -358,6 +429,8 @@ async function loginAccount(params: TerminalMethods['account.login'][0]) { } send('accountChanged', { id: activeAccount }); uiAccount = account; + uiLoginAt = Date.now(); + sync?.presenceChanged(); journaling?.setActive(account); return account; } @@ -399,6 +472,8 @@ async function loginExchange(params: ExchangeLogin): Promise { } send('accountChanged', { id: activeAccount }); uiAccount = account; + uiLoginAt = Date.now(); + sync?.presenceChanged(); return account; } @@ -495,6 +570,11 @@ ipcMain.handle('sync:request', async (_event, method: SyncMethod, params: unknow return { error: { code: 0, message: (error as Error).message } }; } }); +// Only the main window serves the stores; pop-outs share its page but have no preload of their own. +const isMainWindow = (sender: Electron.WebContents) => !!window && !window.isDestroyed() && sender === window.webContents; +ipcMain.on('sync:source-ready', (event) => { if (isMainWindow(event.sender)) syncSources.setReady(true); }); +ipcMain.on('sync:source-reply', (event, id: unknown, reply: unknown) => { if (isMainWindow(event.sender) && typeof id === 'number') syncSources.receive(id, reply); }); +ipcMain.on('sync:changed', (event) => { if (isMainWindow(event.sender)) sync?.noteLocalChange(); }); ipcMain.handle('updates:request', async (_event, method: UpdateMethod, params: unknown) => { try { if (!updates) throw new Error('Updates are still starting'); @@ -549,7 +629,7 @@ if (!app.requestSingleInstanceLock()) { settings = await loadSettings(); await zoom.load(); await device.load(); - try { accounts = await loadAccounts(); } + try { accounts = await loadAccounts(() => sync?.noteLocalChange()); } catch { journal('Saved accounts could not be loaded. Log in manually.', 'Accounts'); } const journalingSession = session.fromPartition('journaling'); journaling = new JournalingService({ @@ -567,6 +647,40 @@ if (!app.requestSingleInstanceLock()) { }, }); await journaling.init(); + // Before the automatic login, which first asks whether the account is open on another computer. + // Its own in-memory session, so no cookie from any other site ever rides along to GitHub. + const syncSession = session.fromPartition('sync'); + sync = new SyncService({ + dir: app.getPath('userData'), + fetch: (url, init) => syncSession.fetch(url, init), + cipher: secretCipher, + platform: process.platform, + appVersion: app.getVersion(), + deviceName: os.hostname(), + send: sendSync, + journal: (message) => journal(message, 'Sync'), + openExternal: (url) => shell.openExternal(url), + revealFile: (file) => shell.showItemInFolder(file), + sources: () => [ + ...mainSyncSources({ + accounts: () => accounts, automation: () => automation, device, onAccountsChanged: () => send('accounts', accountProfiles()), journal: (message) => journal(message, 'Sync'), + }), + ...syncSources.sources(), + ], + liveAccounts, + releaseAccounts, + chooseFile: async (mode) => { + const filters = [{ name: 'OpenTerm5 sync file', extensions: [SYNC_FILE_EXTENSION] }]; + if (mode === 'open') { + const result = window ? await dialog.showOpenDialog(window, { title: 'Open sync file', filters, properties: ['openFile'] }) : null; + return result && !result.canceled ? result.filePaths[0] ?? null : null; + } + const result = window ? await dialog.showSaveDialog(window, { title: 'Create sync file', defaultPath: `OpenTerm5.${SYNC_FILE_EXTENSION}`, filters }) : null; + return result && !result.canceled ? result.filePath ?? null : null; + }, + }); + await sync.init(); + pool.on('state', () => sync?.presenceChanged()); const recent = accounts?.list()[0]; const automatic = recent?.isPasswordSaved && recent.hasPassword && (isExchangePlatform(recent.platform) || recent.webEndpoint === settings.webEndpoint) ? recent : undefined; // Announced before the window loads and before the endpoint connects, so the window never looks logged out meanwhile. @@ -593,6 +707,7 @@ if (!app.requestSingleInstanceLock()) { automation = new AutomationHost({ pool, userData: app.getPath('userData'), cipher: secretCipher, appVersion: app.getVersion(), accounts: () => accounts, send: sendAutomation, journal, saveFile, clientId: () => device.clientId.toString('hex'), + onSecretsSaved: () => sync?.noteLocalChange(), }); await automation.init(); // Squirrel.Mac only installs signed builds, so macOS checks and links to the release page. @@ -608,30 +723,6 @@ if (!app.requestSingleInstanceLock()) { confirmRestart, }); await updates.init(); - // Its own in-memory session, so no cookie from any other site ever rides along to GitHub. - const syncSession = session.fromPartition('sync'); - sync = new SyncService({ - dir: app.getPath('userData'), - fetch: (url, init) => syncSession.fetch(url, init), - cipher: secretCipher, - platform: process.platform, - appVersion: app.getVersion(), - deviceName: os.hostname(), - send: sendSync, - journal: (message) => journal(message, 'Sync'), - openExternal: (url) => shell.openExternal(url), - revealFile: (file) => shell.showItemInFolder(file), - chooseFile: async (mode) => { - const filters = [{ name: 'OpenTerm5 sync file', extensions: [SYNC_FILE_EXTENSION] }]; - if (mode === 'open') { - const result = window ? await dialog.showOpenDialog(window, { title: 'Open sync file', filters, properties: ['openFile'] }) : null; - return result && !result.canceled ? result.filePaths[0] ?? null : null; - } - const result = window ? await dialog.showSaveDialog(window, { title: 'Create sync file', defaultPath: `OpenTerm5.${SYNC_FILE_EXTENSION}`, filters }) : null; - return result && !result.canceled ? result.filePath ?? null : null; - }, - }); - await sync.init(); // Leaving a computer pushes its edits, and sitting down at one catches it up. powerMonitor.on('suspend', () => sync?.wake()); powerMonitor.on('lock-screen', () => sync?.wake()); @@ -647,14 +738,12 @@ app.on('window-all-closed', () => { } }); -let isSyncFlushed = false; - // Quitting waits up to 5 s for edits not yet synced, so they are not left on one computer. app.on('before-quit', (event) => { - if (!isSyncFlushed && sync?.hasUnpushedChanges) { + if (!isSyncFlushed && sync?.isLeaveNeeded) { event.preventDefault(); isSyncFlushed = true; - void Promise.race([sync.flush(), new Promise((resolve) => setTimeout(resolve, 5000))]).finally(() => app.quit()); + void Promise.race([sync.leave(), new Promise((resolve) => setTimeout(resolve, 5000))]).finally(() => app.quit()); return; } updates?.shutdown(); diff --git a/electron/preload.ts b/electron/preload.ts index 8ec224b..ce7a527 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -3,7 +3,7 @@ import type { TerminalEvent, TerminalMethod, ConnectionSettings, TerminalApi } f import type { AutomationApi, AutomationEvent, AutomationMethod } from '../shared/automation'; import type { JournalingApi, JournalingEvent, JournalingMethod } from '../shared/journaling'; import type { UpdateApi, UpdateEvent, UpdateMethod } from '../shared/updates'; -import type { SyncApi, SyncEvent, SyncMethod } from '../shared/sync'; +import type { SyncApi, SyncEvent, SyncMethod, SyncSourceRequest } from '../shared/sync'; import type { ZoomStep } from '../shared/zoom'; class RequestError extends Error { @@ -67,6 +67,17 @@ const updates: UpdateApi = { const sync: SyncApi = { request: (method: SyncMethod, params: unknown) => invoke('sync:request', method, params), on: (event: SyncEvent, listener: (data: never) => void) => onSync(event, listener), + serve: (handler: (request: SyncSourceRequest) => Promise) => { + ipcRenderer.removeAllListeners('sync:source'); + ipcRenderer.on('sync:source', (_: IpcRendererEvent, id: number, request: SyncSourceRequest) => { + handler(request).then( + (result) => ipcRenderer.send('sync:source-reply', id, { result }), + (error: unknown) => ipcRenderer.send('sync:source-reply', id, { error: error instanceof Error ? error.message : String(error) }), + ); + }); + ipcRenderer.send('sync:source-ready'); + }, + changed: () => ipcRenderer.send('sync:changed'), } as SyncApi; contextBridge.exposeInMainWorld('terminal', api); diff --git a/electron/sync-engine-fixtures.ts b/electron/sync-engine-fixtures.ts index 9e9f91a..e0befa9 100644 --- a/electron/sync-engine-fixtures.ts +++ b/electron/sync-engine-fixtures.ts @@ -68,7 +68,7 @@ export class MemorySource implements SyncSource { async snapshot(): Promise>> { return structuredClone(this.data); } - validate(value: Json): void { + validate(value: Json): void | Promise { if (!this.isValid(value)) throw new Error('invalid'); } diff --git a/electron/sync-engine.test.ts b/electron/sync-engine.test.ts index 6c14284..6c6afe8 100644 --- a/electron/sync-engine.test.ts +++ b/electron/sync-engine.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { randomBytes } from 'node:crypto'; import { test } from 'node:test'; import { deriveKeys, newRootKey } from './sync-crypto'; +import { SyncSourceUnavailableError } from './sync-engine'; import { device, FakeRemote, REPO_ID } from './sync-engine-fixtures'; import { MANIFEST_PATH, openManifest, sealManifest, SyncIntegrityError } from './sync-manifest'; @@ -312,3 +313,88 @@ test('two computers syncing many random edits end up identical', async () => { for (const computer of [a, b, a]) await computer.engine.sync({ isMassChangeAllowed: true }); assert.deepEqual(a.source('chartDrawings').data, b.source('chartDrawings').data); }); + +test('a source that cannot answer stops the sync instead of reading its values as invalid', async () => { + const { a, b } = setup(); + a.source('layout').data = { L1: { name: 'Layout' } }; + await a.engine.sync(); + const source = b.source('layout'); + source.validate = async () => { throw new SyncSourceUnavailableError('still loading'); }; + await assert.rejects(b.engine.sync(), SyncSourceUnavailableError); + assert.deepEqual(source.data, {}); + source.validate = async () => undefined; + await b.engine.sync(); + assert.deepEqual(source.data, { L1: { name: 'Layout' } }); +}); + +test('a merge that breaks a limit keeps the synced value instead of failing every sync', async () => { + const { a, b, tick } = setup(); + for (const computer of [a, b]) computer.source('chartDrawings').isValid = (value) => Object.keys(value as object).length <= 2; + a.source('chartDrawings').data = { c1: { d1: 'line' } }; + await a.engine.sync(); + await b.engine.sync(); + a.source('chartDrawings').data = { c1: { d1: 'line', d2: 'box' } }; + b.source('chartDrawings').data = { c1: { d1: 'line', d3: 'fib' } }; + tick(); + await a.engine.sync(); + const outcome = await b.engine.sync(); + assert.equal(outcome.status, 'synced'); + assert.deepEqual(b.source('chartDrawings').data, { c1: { d1: 'line', d2: 'box' } }); +}); + +test('a record this computer could not read is never overwritten by its own copy', async () => { + const { a, b, tick } = setup(); + b.source('layout').data = { L1: { name: 'old' } }; + b.source('layout').isValid = (value) => (value as { name?: string }).name !== 'new'; + a.source('layout').data = { L1: { name: 'new' } }; + await a.engine.sync(); + tick(); + await b.engine.sync(); + await b.engine.sync(); + await a.engine.sync(); + assert.deepEqual(a.source('layout').data, { L1: { name: 'new' } }); +}); + +test('a first merge keeps what only this computer has and takes the synced value where both have one', async () => { + const { a, b, tick } = setup(['layout', 'preferences']); + a.source('preferences').data = { workspace: { theme: 'dark', drawingSync: 'symbol' } }; + a.source('layout').data = { L1: { name: 'Synced' } }; + await a.engine.sync(); + tick(); + b.source('preferences').data = { workspace: { theme: 'light', replayMode: 'allCharts' } }; + b.source('layout').data = { L2: { name: 'Mine' } }; + await b.engine.sync({ isRemotePreferred: true }); + await a.engine.sync(); + const expected = { workspace: { theme: 'dark', drawingSync: 'symbol', replayMode: 'allCharts' } }; + assert.deepEqual(b.source('preferences').data, expected); + assert.deepEqual(a.source('preferences').data, expected); + assert.deepEqual(Object.keys(a.source('layout').data).sort(), ['L1', 'L2']); +}); + +test('use synced data can be limited to some kinds, the others still merging', async () => { + const { a, b } = setup(['account', 'layout']); + a.source('layout').data = { L1: { name: 'Synced' } }; + await a.engine.sync(); + b.source('layout').data = { L9: { name: 'Default' } }; + b.source('account').data = { A9: { login: 9 } }; + await b.engine.sync({ isLocalDiscarded: true, isMassChangeAllowed: true, discardedKinds: ['layout'] }); + await a.engine.sync(); + assert.deepEqual(b.source('layout').data, { L1: { name: 'Synced' } }); + assert.deepEqual(a.source('account').data, { A9: { login: 9 } }); +}); + +test('kinds apply in declaration order whatever order their sources came in, and __proto__ ids never sync', async () => { + const remote = new FakeRemote(); + const rootKey = newRootKey(); + const clock = { now: 1_800_000_000_000 }; + const a = device('a', remote, rootKey, ['layout', 'chart'], clock); + a.source('chart').data = { c1: { symbol: 'EURUSD' } }; + a.source('layout').data = JSON.parse('{"L1": {"name": "x"}, "__proto__": {"name": "y"}}'); + await a.engine.sync(); + const order: string[] = []; + const b = device('b', remote, rootKey, ['layout', 'chart'], clock); + for (const kind of ['layout', 'chart'] as const) b.source(kind).onApply = () => order.push(kind); + await b.engine.sync(); + assert.deepEqual(order, ['chart', 'layout']); + assert.deepEqual(Object.keys(b.source('layout').data), ['L1']); +}); diff --git a/electron/sync-engine.ts b/electron/sync-engine.ts index b1a5fde..4bb0ba3 100644 --- a/electron/sync-engine.ts +++ b/electron/sync-engine.ts @@ -1,8 +1,8 @@ -import { isSameJson, isSyncKind, recordKey, splitRecordKey, syncKindInfo, type Json, type SyncKind } from '../shared/sync'; +import { isSameJson, isSyncKind, recordKey, splitRecordKey, syncKindInfo, syncKinds, type Json, type SyncKind, type SyncSourceChange } from '../shared/sync'; import { hlcAt, HybridClock } from './sync-clock'; import { decodePayload, encodePayload, envelopeHash, MAX_ENVELOPE_BYTES, objectPath, open, recordCipherKey, seal, type SyncKeys } from './sync-crypto'; import { emptyManifest, MANIFEST_PATH, openManifest, sealManifest, SyncIntegrityError, type Manifest, type ManifestDevice, type ManifestEntry } from './sync-manifest'; -import { mergeRecords, parseSyncedRecord, pruneTombstones, recordPlain, stampRecord, type MergeConflict, type SyncedRecord } from './sync-merge'; +import { hasUnsafeKey, mergeRecords, parseSyncedRecord, preferRemote, pruneTombstones, recordPlain, stampRecord, type MergeConflict, type SyncedRecord } from './sync-merge'; const TOMBSTONE_LIFETIME_MS = 180 * 86_400_000; const MAX_ATTEMPTS = 5; @@ -19,18 +19,21 @@ export interface SyncRemote { head(): Promise; read(commit: string, paths: readonly string[]): Promise>; commit(parent: string | null, writes: ReadonlyMap): Promise; + // The remote's own clock, which presence and leases are measured on; local time when absent. + serverNow?(): number; } -export interface SourceChange { - readonly upserts: Readonly>; - readonly deletes: readonly string[]; -} +export type SourceChange = SyncSourceChange; + +// A source that cannot answer at all (its window is still loading) stops the sync, rather than +// having its values read as invalid or missing. +export class SyncSourceUnavailableError extends Error {} export interface SyncSource { readonly kind: SyncKind; snapshot(): Promise>>; - // Throws when a value would be unsafe to apply; it never changes the value. - validate(value: Json): void; + // Throws (or rejects) when a value would be unsafe to apply under that id; it never changes the value. + validate(value: Json, id: string): void | Promise; apply(change: SourceChange): Promise; } @@ -81,6 +84,11 @@ export interface SyncOptions { readonly isMassChangeAllowed?: boolean; // "Use synced data": local changes since the last sync are replaced by the remote's. readonly isLocalDiscarded?: boolean; + // Limits `isLocalDiscarded` to these kinds; the others still merge. + readonly discardedKinds?: readonly SyncKind[]; + // A computer merging into synced data for the first time: where both sides hold a value and + // there is no base yet, the synced one wins, so a fresh install's defaults never replace it. + readonly isRemotePreferred?: boolean; // The user confirmed the repository was put back to an older state on purpose. readonly isHistoryAccepted?: boolean; } @@ -117,7 +125,9 @@ export class SyncEngine { this.now = deps.now ?? Date.now; this.clock = new HybridClock(deps.device.id, this.now); this.pause = deps.pause ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms))); - this.sources = new Map(deps.sources.map((source) => [source.kind, source])); + // Kinds apply in declaration order, whatever order the sources came in. + const rank = (kind: SyncKind) => syncKinds.findIndex((info) => info.key === kind); + this.sources = new Map([...deps.sources].sort((left, right) => rank(left.kind) - rank(right.kind)).map((source) => [source.kind, source])); } // Stamps local edits when they happen rather than at the next sync, so "the newer edit wins" @@ -137,7 +147,7 @@ export class SyncEngine { for (const id of Object.keys(snapshot)) keys.add(recordKey(kind, id)); for (const key of keys) { const current = snapshot[splitRecordKey(key).id]; - if (current !== undefined && !this.isValidPlain(source, current)) continue; + if (current !== undefined && !await this.isValidPlain(source, current, splitRecordKey(key).id)) continue; const baseline = state.pending[key] ?? state.bases[key]; const stamped = stampRecord(shape, baseline, current, () => this.clock.tick()); if (stamped === baseline || !stamped) continue; @@ -154,6 +164,65 @@ export class SyncEngine { return this.exclusive(() => this.syncNow(options)); } + // One record as the remote holds it now, for a kind this computer may not sync yet (deciding + // which CID to keep before turning CID sync on). Checked like a pull: hash, then authentication. + peek(key: string): Promise { + return this.exclusive(async () => { + const { keys, epoch, repoId, remote } = this.deps; + const head = await remote.head(); + const envelope = head === null ? null : (await remote.read(head, [MANIFEST_PATH])).get(MANIFEST_PATH) ?? null; + if (head === null || !envelope) return undefined; + const state = await this.deps.store.load(); + const entry = openManifest(keys, envelope, { epoch, repoId, highWater: state?.highWater ?? 0 }).records[key]; + if (!entry?.hash) return undefined; + const object = (await remote.read(head, [objectPath(entry.hash)])).get(objectPath(entry.hash)); + if (!object || envelopeHash(object) !== entry.hash) throw new SyncIntegrityError('tampered', 'A synced record is missing or altered.'); + let record: SyncedRecord; + try { record = parseSyncedRecord(decodePayload(open(recordCipherKey(keys, key), object, this.recordContext(key, entry.generation)))); } + catch { throw new SyncIntegrityError('tampered', 'A synced record failed authentication.'); } + return recordPlain(syncKindInfo(splitRecordKey(key).kind as SyncKind).shape, record); + }); + } + + serverNow(): number { + return this.deps.remote.serverNow?.() ?? this.now(); + } + + // The remote manifest now, checked as a pull checks it, without reading any record or store. + peekManifest(): Promise { + return this.exclusive(async () => (await this.readManifest())?.manifest ?? null); + } + + // A commit of the manifest alone (presence, handoff requests), so it needs none of the stores. + // `edit` returns null when nothing needs writing; a lost race reads again and retries. + editManifest(edit: (manifest: Manifest, serverNow: number) => Manifest | null): Promise { + return this.exclusive(async () => { + for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { + if (attempt) await this.pause(2 ** attempt * 250 + Math.random() * 250); + const current = await this.readManifest(); + if (!current) return false; + this.clock.observe(current.manifest.hlc); + const next = edit(current.manifest, this.serverNow()); + if (!next) return false; + const manifest: Manifest = { ...next, seq: current.manifest.seq + 1, hlc: this.clock.tick(), writer: this.deps.device.id }; + if (await this.deps.remote.commit(current.head, new Map([[MANIFEST_PATH, sealManifest(this.deps.keys, manifest)]])) === null) continue; + const state = await this.deps.store.load(); + if (state) await this.deps.store.save({ ...state, highWater: Math.max(state.highWater, manifest.seq), clock: this.clock.last }); + return true; + } + return false; + }); + } + + private async readManifest(): Promise<{ head: string; manifest: Manifest } | null> { + const { keys, epoch, repoId, remote } = this.deps; + const head = await remote.head(); + const envelope = head === null ? null : (await remote.read(head, [MANIFEST_PATH])).get(MANIFEST_PATH) ?? null; + if (head === null || !envelope) return null; + const state = await this.deps.store.load(); + return { head, manifest: openManifest(keys, envelope, { epoch, repoId, highWater: state?.highWater ?? 0 }) }; + } + private exclusive(run: () => Promise): Promise { const result = this.queue.then(run); this.queue = result.catch(() => undefined); @@ -167,7 +236,8 @@ export class SyncEngine { for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) { if (attempt) await this.pause(2 ** attempt * 250 + Math.random() * 250); const head = await this.deps.remote.head(); - const view = head !== null && head === state.head && state.manifest ? { head, manifest: state.manifest, records: state.bases, invalid: new Set() } : await this.pull(head, state, options); + const isUnchanged = head !== null && head === state.head && !!state.manifest && !this.hasUnreadRecords(state); + const view = isUnchanged ? { head: head!, manifest: state.manifest!, records: state.bases, invalid: new Set() } : await this.pull(head, state, options); const outcome = await this.reconcile(state, view, options); if (outcome.status !== 'retry') return outcome.status === 'synced' ? outcome.outcome : outcome; state = outcome.state; @@ -175,6 +245,13 @@ export class SyncEngine { return { status: 'busy' }; } + // A record the last pull left out (invalid then, perhaps valid now that a record it depends on + // arrived) is in the manifest with no base. Skipping the pull would read it as absent and let + // local data overwrite it, so the pull runs again; it fetches only those records. + private hasUnreadRecords(state: SyncState): boolean { + return Object.entries(state.manifest!.records).some(([key, entry]) => entry.hash !== null && this.isReadable(key, entry) && !state.bases[key]); + } + private async pull(head: string | null, state: SyncState, options: SyncOptions): Promise { const { keys, epoch, repoId, remote } = this.deps; const envelope = head === null ? null : (await remote.read(head, [MANIFEST_PATH])).get(MANIFEST_PATH) ?? null; @@ -213,7 +290,7 @@ export class SyncEngine { catch { throw new SyncIntegrityError('tampered', `A synced record (${splitRecordKey(key).kind}) failed authentication.`); } // An authentic record that fails validation came from a buggy build, not an attacker: it is // left alone on this computer rather than blocking every other record. - if (!this.isValid(key, record)) invalid.add(key); + if (!await this.isValid(key, record)) invalid.add(key); else records[key] = record; } for (const record of Object.values(records)) { @@ -243,6 +320,7 @@ export class SyncEngine { for (const [kind, source] of this.sources) { const { shape } = syncKindInfo(kind); const snapshot = snapshots.get(kind)!; + const isDiscarded = !!options.isLocalDiscarded && (!options.discardedKinds || options.discardedKinds.includes(kind)); const keys = new Set([...Object.keys(state.bases), ...Object.keys(view.records), ...Object.keys(state.pending)].filter((key) => splitRecordKey(key).kind === kind)); for (const id of Object.keys(snapshot)) keys.add(recordKey(kind, id)); const counts = { local: 0, remote: 0, items: 0, total: 0 }; @@ -254,11 +332,20 @@ export class SyncEngine { if (view.invalid.has(key) || (entry && !this.isReadable(key, entry))) { skipped.add(key); continue; } const base = state.bases[key]; const current = snapshot[id]; - if (current !== undefined && !this.isValidPlain(source, current)) { skipped.add(key); continue; } - const local = options.isLocalDiscarded ? base : stampRecord(shape, state.pending[key] ?? base, current, tick); + if (current !== undefined && !await this.isValidPlain(source, current, id)) { skipped.add(key); continue; } + const local = isDiscarded ? base : stampRecord(shape, state.pending[key] ?? base, current, tick); const remote = view.records[key]; - const result = mergeRecords(shape, base, local, remote, tick); + const isFirstMeeting = !!options.isRemotePreferred && !base && !!local && !!remote && !local.deletedHlc && !remote.deletedHlc; + const result = isFirstMeeting ? preferRemote(shape, local, remote, tick) : mergeRecords(shape, base, local, remote, tick); let next = result.merged; + // Each side was checked on its own; the merge of two valid values can still break a limit + // (more lists than allowed, no list at all). Then the remote's stays, rather than an apply + // failing on every sync from now on. + const mergedPlain = recordPlain(shape, next); + if (mergedPlain !== undefined && !isSameJson(mergedPlain, current) && !isSameJson(mergedPlain, recordPlain(shape, remote)) && !await this.isValidPlain(source, mergedPlain, id)) { + if (!isAlive(remote)) { skipped.add(key); continue; } + next = remote; + } if (next && !sameRecord(next, remote)) next = pruneTombstones(next, horizon); if (next) merged[key] = next; if (next && !sameRecord(next, remote)) toPush.add(key); @@ -286,9 +373,9 @@ export class SyncEngine { counts.items = Math.max(counts.items, removedItems(base, local)); } } - const isEmptied = counts.total >= MASS_EMPTY_FROM && !Object.keys(snapshot).length && !options.isLocalDiscarded; + const isEmptied = counts.total >= MASS_EMPTY_FROM && !Object.keys(snapshot).length && !isDiscarded; const isMass = (removed: number) => removed > MASS_RECORDS && removed / Math.max(counts.total, 1) > MASS_SHARE; - if (isEmptied || isMass(counts.local) || (counts.items > MASS_ITEMS && !options.isLocalDiscarded)) mass.push({ kind, side: 'local', removed: Math.max(counts.local, counts.items), total: counts.total }); + if (isEmptied || isMass(counts.local) || (counts.items > MASS_ITEMS && !isDiscarded)) mass.push({ kind, side: 'local', removed: Math.max(counts.local, counts.items), total: counts.total }); if (isMass(counts.remote)) mass.push({ kind, side: 'remote', removed: counts.remote, total: counts.total }); if (change.deletes.length || Object.keys(change.upserts).length) { changes.set(kind, change); @@ -405,13 +492,21 @@ export class SyncEngine { return isSyncKind(kind) && this.sources.has(kind) && entry.schema <= syncKindInfo(kind).schema; } - private isValid(key: string, record: SyncedRecord): boolean { - const kind = splitRecordKey(key).kind as SyncKind; - const plain = recordPlain(syncKindInfo(kind).shape, record); - return plain === undefined || this.isValidPlain(this.sources.get(kind)!, plain); + private async isValid(key: string, record: SyncedRecord): Promise { + const { kind, id } = splitRecordKey(key); + const plain = recordPlain(syncKindInfo(kind as SyncKind).shape, record); + return plain === undefined || this.isValidPlain(this.sources.get(kind as SyncKind)!, plain, id); } - private isValidPlain(source: SyncSource, plain: Json): boolean { - try { source.validate(plain); return true; } catch { return false; } + // `__proto__` as an id or a key would set an object's prototype wherever it is assigned. + private async isValidPlain(source: SyncSource, plain: Json, id: string): Promise { + if (id === '__proto__' || hasUnsafeKey(syncKindInfo(source.kind).shape, plain)) return false; + try { + await source.validate(plain, id); + return true; + } catch (error) { + if (error instanceof SyncSourceUnavailableError) throw error; + return false; + } } } diff --git a/electron/sync-main-sources.test.ts b/electron/sync-main-sources.test.ts new file mode 100644 index 0000000..b1d2db2 --- /dev/null +++ b/electron/sync-main-sources.test.ts @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { test } from 'node:test'; +import type { Account } from '../shared/protocol'; +import { AccountRegistry } from './account-registry'; +import { DeviceIdentity } from './device-identity'; +import { deriveKeys, newRootKey } from './sync-crypto'; +import { SyncEngine } from './sync-engine'; +import { FakeRemote, MemoryStore, REPO_ID } from './sync-engine-fixtures'; +import { mainSyncSources } from './sync-main-sources'; + +const account: Account = { name: 'Fixture', server: 'Demo', company: 'Broker', currency: 'USD', tradeMode: 'demo', login: 1234, credit: 0, margin: 0, equity: 1000, profit: 0, balance: 1000, leverage: 100, marginFree: 1000, marginLevel: 0, isHedging: true, isTradeAllowed: true }; +const endpoint = 'wss://broker.example/terminal'; +const cipher = { canEncrypt: () => true, encrypt: (value: string) => `encrypted:${Buffer.from(value).toString('base64')}`, decrypt: (value: string) => Buffer.from(value.slice(10), 'base64').toString() }; +const traits = () => ({ scale: 1, width: 1920, height: 1080, platform: 'Linux x86_64', language: 'en-US' }); + +async function computer(id: string, remote: FakeRemote, rootKey: Buffer, kinds: readonly string[]) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'openterm-sync-main-')); + const registry = new AccountRegistry(cipher, async () => {}); + const device = new DeviceIdentity(path.join(dir, 'device.json'), traits); + await device.load(); + const journal: string[] = []; + let changed = 0; + const sources = mainSyncSources({ accounts: () => registry, automation: () => undefined, device, onAccountsChanged: () => { changed += 1; }, journal: (message) => journal.push(message) }) + .filter((source) => kinds.includes(source.kind)); + const engine = new SyncEngine({ + epoch: 1, repoId: REPO_ID, keys: deriveKeys(rootKey), remote, store: new MemoryStore(), sources, pause: async () => undefined, + device: { id, name: id, platform: 'linux', publicKey: 'x', appVersion: '1.0.0' }, + }); + return { registry, device, engine, journal, changed: () => changed }; +} + +test('accounts sync everywhere; their passwords only between computers that opted in', async () => { + const remote = new FakeRemote(); + const rootKey = newRootKey(); + const office = await computer('aaaaaaaaaaaaaaaa', remote, rootKey, ['account', 'accountSecret']); + const laptop = await computer('bbbbbbbbbbbbbbbb', remote, rootKey, ['account', 'accountSecret']); + const kiosk = await computer('cccccccccccccccc', remote, rootKey, ['account']); + const profile = await office.registry.remember(account, endpoint, 'broker-password', 'Demo', true); + await office.engine.sync(); + for (const content of remote.files().values()) assert.ok(!content.includes('broker-password')); + // The password is checked before its account arrives, so it applies on the next sync. + await laptop.engine.sync(); + await laptop.engine.sync(); + await kiosk.engine.sync(); + assert.equal(laptop.registry.get(profile.id).label, 'Demo'); + assert.equal(laptop.registry.password(profile.id), 'broker-password'); + assert.ok(laptop.changed() > 0); + assert.equal(kiosk.registry.get(profile.id).isPasswordSaved, false); + + // A password changed later on the laptop reaches the office; the kiosk still never gets one. + await new Promise((resolve) => setTimeout(resolve, 5)); + await laptop.registry.remember(account, endpoint, 'changed-password', 'Demo', true); + await laptop.engine.sync(); + await office.engine.sync(); + await kiosk.engine.sync(); + assert.equal(office.registry.password(profile.id), 'changed-password'); + assert.equal(kiosk.registry.password(profile.id), undefined); +}); + +test('one CID for all: the synced CID replaces this computer\'s and applies from the next login', async () => { + const remote = new FakeRemote(); + const rootKey = newRootKey(); + const office = await computer('aaaaaaaaaaaaaaaa', remote, rootKey, ['identity']); + const laptop = await computer('bbbbbbbbbbbbbbbb', remote, rootKey, ['identity']); + assert.notDeepEqual(laptop.device.clientId, office.device.clientId); + await office.engine.sync(); + await laptop.engine.sync({ isRemotePreferred: true }); + assert.deepEqual(laptop.device.clientId, office.device.clientId); + assert.match(laptop.journal.join(' '), /next login/); + await office.device.regenerate(); + await office.engine.sync(); + await laptop.engine.sync(); + assert.deepEqual(laptop.device.clientId, office.device.clientId); +}); diff --git a/electron/sync-main-sources.ts b/electron/sync-main-sources.ts new file mode 100644 index 0000000..82bcb46 --- /dev/null +++ b/electron/sync-main-sources.ts @@ -0,0 +1,90 @@ +import type { Json } from '../shared/sync'; +import type { AccountRegistry } from './account-registry'; +import type { AutomationHost } from './automation-host'; +import type { DeviceIdentity } from './device-identity'; +import { SyncSourceUnavailableError, type SyncSource } from './sync-engine'; + +// The stores the main process holds, as sync sources. Secrets are handled here and never reach the +// renderer: sync decrypts them from the keychain only to encrypt them under its own key. + +export const IDENTITY_ID = 'cid'; +export const WEBHOOKS_ID = 'webhooks'; + +export interface MainSourcesDeps { + readonly accounts: () => AccountRegistry | undefined; + readonly automation: () => AutomationHost | undefined; + readonly device: DeviceIdentity; + readonly onAccountsChanged: () => void; + readonly journal: (message: string) => void; +} + +const toJson = (value: unknown): Json => JSON.parse(JSON.stringify(value)) as Json; + +// Saved accounts that failed to load read as none at all; syncing that would delete them on +// every computer, so sync stops and says why instead. +function registryOf(deps: MainSourcesDeps): AccountRegistry { + const registry = deps.accounts(); + if (!registry) throw new Error('Saved accounts could not be read on this computer, so sync is paused. Restart OpenTerm5 to try again.'); + return registry; +} + +function automationOf(deps: MainSourcesDeps): AutomationHost { + const automation = deps.automation(); + if (!automation) throw new SyncSourceUnavailableError('Automation is still starting.'); + return automation; +} + +function singleton(id: string, expected: string): void { + if (id !== expected) throw new Error(`Unknown record ${id}.`); +} + +export function mainSyncSources(deps: MainSourcesDeps): SyncSource[] { + return [ + { + kind: 'account', + snapshot: async () => toJson(registryOf(deps).syncedProfiles()) as Record, + validate: (value, id) => registryOf(deps).checkSyncedProfile(id, value), + apply: async (change) => { + await registryOf(deps).applySyncedProfiles(change); + deps.onAccountsChanged(); + }, + }, + { + kind: 'accountSecret', + snapshot: async () => toJson(registryOf(deps).syncedSecrets()) as Record, + validate: (value, id) => registryOf(deps).checkSyncedSecret(id, value), + apply: async (change) => { + await registryOf(deps).applySyncedSecrets(change); + deps.onAccountsChanged(); + }, + }, + { + kind: 'identity', + snapshot: async () => ({ [IDENTITY_ID]: deps.device.syncRecord() }), + validate: (value, id) => { + singleton(id, IDENTITY_ID); + const { cid, isUnique } = (value ?? {}) as { cid?: unknown; isUnique?: unknown }; + if (typeof cid !== 'string' || !/^[0-9a-f]{32}$/.test(cid) || typeof isUnique !== 'boolean') throw new Error('Invalid client ID.'); + }, + // The CID is never removed, only replaced. + apply: async (change) => { + const value = change.upserts[IDENTITY_ID]; + if (value !== undefined && await deps.device.setFromSync(value)) deps.journal('This computer now uses the CID of your other computers. It applies from the next login.'); + }, + }, + { + kind: 'webhookSecrets', + // Unreadable settings are offered as a record that fails its checks, so it is left alone. + snapshot: async () => ({ [WEBHOOKS_ID]: toJson(automationOf(deps).syncedWebhooks() ?? { isUnreadable: true }) }), + validate: (value, id) => { + singleton(id, WEBHOOKS_ID); + automationOf(deps).checkSyncedWebhooks(value); + }, + // Webhook settings are never removed as a whole. + apply: async (change) => { + const value = change.upserts[WEBHOOKS_ID]; + if (value !== undefined) await automationOf(deps).applySyncedWebhooks(value); + }, + }, + ]; +} diff --git a/electron/sync-manifest.test.ts b/electron/sync-manifest.test.ts index d911096..7ba55af 100644 --- a/electron/sync-manifest.test.ts +++ b/electron/sync-manifest.test.ts @@ -35,7 +35,11 @@ test('malformed manifests are refused field by field', () => { { ...manifest, leases: { copier: { fence: 1, deviceId: 'a', expiresAt: 'soon', renewedAt: 1 } } }, { ...manifest, presence: { a: { accounts: [5], expiresAt: 1 } } }, { ...manifest, devices: [{ id: 'a', name: 'x', platform: 'linux', publicKey: 'k', appVersion: '1', lastSeen: 1, firstSeen: 1 }] }, + { ...manifest, handoffs: { 'mt5|1': { from: 'a', to: ['Not An Id'], createdAt: 1, expiresAt: 2 } } }, + { ...manifest, handoffs: { 'mt5|1': { from: 'a', to: ['b'], expiresAt: 2 } } }, + JSON.parse(JSON.stringify({ ...manifest, records: {} }).replace('"records":{}', '"records":{"__proto__":{}}')), ]; + assert.ok(parseManifest({ ...manifest, handoffs: { 'mt5|1': { from: 'a', to: ['b'], createdAt: 1, expiresAt: 2 } } }).handoffs); for (const value of bad) assert.throws(() => parseManifest(value), reason('tampered'), JSON.stringify(value)); }); @@ -77,6 +81,7 @@ test('presence shows where an account is open and expires on its own', () => { presence = withPresence(presence, 'b', ['mt5|456'], 10); assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'b', 20), ['a']); assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'a', 20), []); - assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'b', 400_000), []); + assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'b', 400_000), ['a']); + assert.deepEqual(presentElsewhere(presence, 'mt5|123', 'b', 2_000_000), []); assert.deepEqual(Object.keys(withPresence(presence, 'a', [], 30)), ['b']); }); diff --git a/electron/sync-manifest.ts b/electron/sync-manifest.ts index db42577..126e35d 100644 --- a/electron/sync-manifest.ts +++ b/electron/sync-manifest.ts @@ -11,7 +11,14 @@ export const MAX_MANIFEST_BYTES = 4 * 1024 * 1024; // `fenceAfterMs` without a renewal, before anyone else may take the lease at `lifetimeMs` plus // the margin, so two computers never both have the right to open. export const leaseTiming = { renewEveryMs: 120_000, fenceAfterMs: 420_000, lifetimeMs: 600_000, takeoverMarginMs: 60_000 } as const; -export const PRESENCE_LIFETIME_MS = 300_000; +// Presence costs a commit per renewal, and each commit keeps a new copy of the manifest in the +// repository's history, so it lasts half an hour and renews when less than half is left. A +// computer that logs out removes it at once; one that crashed is covered by "Log in anyway". +export const PRESENCE_LIFETIME_MS = 1_800_000; +export const PRESENCE_RENEW_MS = 900_000; +// A request to move an account stays open long enough for a computer that was asleep to see it +// when it wakes; it then logs out only if its own login is older than the request. +export const HANDOFF_LIFETIME_MS = 12 * 3_600_000; export interface ManifestEntry { // null for a deleted record. @@ -44,6 +51,14 @@ export interface Presence { readonly expiresAt: number; } +// A computer asking the ones that hold an account to log out of it, so it can log in alone. +export interface Handoff { + readonly from: string; + readonly to: readonly string[]; + readonly createdAt: number; + readonly expiresAt: number; +} + export interface Manifest { readonly v: 1; readonly seq: number; @@ -57,6 +72,8 @@ export interface Manifest { readonly leases: Readonly>; readonly records: Readonly>; readonly presence: Readonly>; + // By account id. Missing in manifests written before login handoff; older builds keep it as is. + readonly handoffs?: Readonly>; } export type IntegrityReason = 'tampered' | 'rollback' | 'epoch' | 'tooNew' | 'stale' | 'revoked'; @@ -90,6 +107,8 @@ export function parseManifest(value: unknown): Manifest { if (!isObject(device) || !isDeviceId(device.id) || !isText(device.name) || !isText(device.platform) || !isText(device.publicKey) || !isText(device.appVersion) || !isCount(device.lastSeen) || !isCount(device.firstSeen) || typeof device.isRevoked !== 'boolean') fail('device'); } + // Keys are assigned into plain objects all through the engine, so none may be `__proto__`. + if ([...Object.keys(records as object), ...Object.keys(leases as object)].some((key) => key === '__proto__' || key.endsWith(':__proto__'))) fail('keys'); for (const entry of Object.values(records as Record)) { if (!isObject(entry) || !(entry.hash === null || (typeof entry.hash === 'string' && /^[0-9a-f]{64}$/.test(entry.hash))) || !isHlc(entry.hlc) || !isCount(entry.schema) || !isCount(entry.generation)) fail('record'); @@ -100,6 +119,13 @@ export function parseManifest(value: unknown): Manifest { for (const [deviceId, entry] of Object.entries(presence as Record)) { if (!isDeviceId(deviceId) || !isObject(entry) || !Array.isArray(entry.accounts) || !entry.accounts.every((account) => isText(account, 512)) || !isCount(entry.expiresAt)) fail('presence'); } + const { handoffs } = value; + if (handoffs !== undefined) { + if (!isObject(handoffs)) fail('handoffs'); + for (const [account, entry] of Object.entries(handoffs as Record)) { + if (account === '__proto__' || !isText(account, 512) || !isObject(entry) || !isDeviceId(entry.from) || !Array.isArray(entry.to) || !entry.to.every(isDeviceId) || !isCount(entry.createdAt) || !isCount(entry.expiresAt)) fail('handoff'); + } + } return value as unknown as Manifest; } @@ -164,6 +190,18 @@ export function withPresence(presence: Manifest['presence'], deviceId: string, a return accounts.length ? { ...live, [deviceId]: { accounts: [...accounts].sort(), expiresAt: serverNow + PRESENCE_LIFETIME_MS } } : live; } +// Whether this computer's presence should be written: its accounts changed, or it runs out soon. +export function isPresenceDue(presence: Manifest['presence'], deviceId: string, accounts: readonly string[], serverNow: number): boolean { + const own = presence[deviceId]; + if (!accounts.length) return !!own; + return !own || own.expiresAt - serverNow < PRESENCE_RENEW_MS || own.accounts.join('\n') !== [...accounts].sort().join('\n'); +} + +// Open requests, expired ones dropped. +export function liveHandoffs(handoffs: Manifest['handoffs'], serverNow: number): Record { + return Object.fromEntries(Object.entries(handoffs ?? {}).filter(([, entry]) => entry.expiresAt > serverNow)); +} + export function presentElsewhere(presence: Manifest['presence'], account: string, deviceId: string, serverNow: number): string[] { return Object.entries(presence).filter(([id, entry]) => id !== deviceId && entry.expiresAt > serverNow && entry.accounts.includes(account)).map(([id]) => id).sort(); } diff --git a/electron/sync-merge.test.ts b/electron/sync-merge.test.ts index 44bf093..06232d9 100644 --- a/electron/sync-merge.test.ts +++ b/electron/sync-merge.test.ts @@ -181,3 +181,8 @@ test('random edits on three replicas converge on the newest edit of every key', for (const replica of replicas) assert.deepEqual(replica.plain, expected, `round ${round}`); } }); + +test('an entry key that would set a prototype is refused when a record is read', () => { + const hlc = '1800000000000.0000.a'; + assert.throws(() => parseSyncedRecord(JSON.parse(`{"entries": {"__proto__": {"hlc": "${hlc}", "value": 1}}}`)), /not allowed/); +}); diff --git a/electron/sync-merge.ts b/electron/sync-merge.ts index f254afa..9364704 100644 --- a/electron/sync-merge.ts +++ b/electron/sync-merge.ts @@ -33,6 +33,17 @@ export interface MergeResult { const SINGLE = ''; +// Entries are plain objects keyed by item ids, so lookups must not reach Object.prototype. +const own = (entries: SyncEntries, key: string): SyncEntry | undefined => Object.hasOwn(entries, key) ? entries[key] : undefined; + +const UNSAFE_KEY = '__proto__'; + +// A key that would set an object's prototype when assigned, among a value's entry keys. +export function hasUnsafeKey(shape: SyncShape, plain: Json): boolean { + if (shape === 'set') return Array.isArray(plain) && plain.includes(UNSAFE_KEY); + return (shape === 'fields' || shape === 'map') && plain !== null && typeof plain === 'object' && !Array.isArray(plain) && Object.hasOwn(plain, UNSAFE_KEY); +} + function plainEntries(shape: SyncShape, plain: Json): Record { if (shape === 'fields' || shape === 'map') { if (plain === null || typeof plain !== 'object' || Array.isArray(plain)) throw new Error(`A ${shape} record must be an object.`); @@ -49,7 +60,7 @@ export function fromEntries(shape: SyncShape, entries: SyncEntries): Json { const live = Object.entries(entries).filter(([, entry]) => !entry.isDeleted); if (shape === 'fields' || shape === 'map') return Object.fromEntries(live.map(([key, entry]) => [key, entry.value ?? null])); if (shape === 'set') return live.map(([key]) => key).sort(); - return entries[SINGLE]?.isDeleted ? null : entries[SINGLE]?.value ?? null; + return own(entries, SINGLE)?.isDeleted ? null : own(entries, SINGLE)?.value ?? null; } export const recordPlain = (shape: SyncShape, record: SyncedRecord | undefined): Json | undefined => @@ -65,7 +76,7 @@ export function stampRecord(shape: SyncShape, baseline: SyncedRecord | undefined const entries: Record = {}; let isChanged = !baseline || !!baseline.deletedHlc; for (const [key, value] of Object.entries(current)) { - const old = previous[key]; + const old = own(previous, key); const isKept = !!old && !old.isDeleted && isSameJson(old.value, value); entries[key] = isKept ? old : { hlc: tick(), value }; isChanged ||= !isKept; @@ -83,7 +94,7 @@ const isSameEntry = (left: SyncEntry | undefined, right: SyncEntry | undefined) const isSameRecord = (left: SyncedRecord | undefined, right: SyncedRecord | undefined) => left === right || (!!left && !!right && left.deletedHlc === right.deletedHlc && Object.keys(left.entries).length === Object.keys(right.entries).length && - Object.entries(left.entries).every(([key, entry]) => isSameEntry(entry, right.entries[key]))); + Object.entries(left.entries).every(([key, entry]) => isSameEntry(entry, own(right.entries, key)))); const isSameValue = (left: SyncEntry, right: SyncEntry) => !!left.isDeleted === !!right.isDeleted && (!!left.isDeleted || isSameJson(left.value, right.value)); @@ -91,7 +102,7 @@ const isSameValue = (left: SyncEntry, right: SyncEntry) => !!left.isDeleted === function latestChange(base: SyncedRecord | undefined, side: SyncedRecord): string { if (side.deletedHlc) return side.deletedHlc; const previous = base && !base.deletedHlc ? base.entries : {}; - return Object.entries(side.entries).reduce((latest, [key, entry]) => !isSameEntry(previous[key], entry) && entry.hlc > latest ? entry.hlc : latest, ''); + return Object.entries(side.entries).reduce((latest, [key, entry]) => !isSameEntry(own(previous, key), entry) && entry.hlc > latest ? entry.hlc : latest, ''); } function combineLists(winner: Json, loser: Json, base: Json | undefined): readonly Json[] { @@ -124,10 +135,10 @@ export function mergeRecords(shape: SyncShape, base: SyncedRecord | undefined, l const conflicts: MergeConflict[] = []; const copies: Json[] = []; for (const key of new Set([...Object.keys(previous), ...Object.keys(local.entries), ...Object.keys(remote.entries)])) { - const original = previous[key]; + const original = own(previous, key); // An entry pruned on one side after its deletion reads as the oldest possible deletion. - const mine = local.entries[key] ?? (original ? { hlc: '', isDeleted: true as const } : undefined); - const theirs = remote.entries[key] ?? (original ? { hlc: '', isDeleted: true as const } : undefined); + const mine = own(local.entries, key) ?? (original ? { hlc: '', isDeleted: true as const } : undefined); + const theirs = own(remote.entries, key) ?? (original ? { hlc: '', isDeleted: true as const } : undefined); const isMineChanged = !isSameEntry(original, mine); const isTheirsChanged = !isSameEntry(original, theirs); let merged: SyncEntry | undefined; @@ -151,6 +162,19 @@ export function mergeRecords(shape: SyncShape, base: SyncedRecord | undefined, l return { merged: { entries }, conflicts, copies }; } +// The first merge of a computer joining synced data: it keeps what only it has, and wherever +// both sides hold a value the synced one wins. Text still keeps the local version as a copy. +export function preferRemote(shape: SyncShape, local: SyncedRecord, remote: SyncedRecord, tick: () => string): MergeResult { + if (shape === 'text') return mergeRecords(shape, undefined, local, remote, tick); + if (shape === 'value') return { merged: remote, conflicts: [], copies: [] }; + if (shape === 'list') { + const [mine, theirs] = [own(local.entries, SINGLE), own(remote.entries, SINGLE)]; + if (!mine || mine.isDeleted || !theirs || theirs.isDeleted) return { merged: remote, conflicts: [], copies: [] }; + return { merged: { entries: { [SINGLE]: { hlc: tick(), value: combineLists(theirs.value ?? null, mine.value ?? null, undefined) } } }, conflicts: [], copies: [] }; + } + return { merged: { entries: { ...local.entries, ...remote.entries } }, conflicts: [], copies: [] }; +} + export function pruneTombstones(record: SyncedRecord, before: string): SyncedRecord { const entries = Object.entries(record.entries); const kept = entries.filter(([, entry]) => !entry.isDeleted || entry.hlc >= before); @@ -165,6 +189,7 @@ export function parseSyncedRecord(value: unknown): SyncedRecord { if (!entries || typeof entries !== 'object' || Array.isArray(entries)) throw new Error('Sync record has no entries.'); const parsed: Record = {}; for (const [key, entry] of Object.entries(entries as Record)) { + if (key === UNSAFE_KEY) throw new Error('Sync record entry key is not allowed.'); const { hlc, value: entryValue, isDeleted } = (entry ?? {}) as { hlc?: unknown; value?: Json; isDeleted?: unknown }; if (!isHlc(hlc) || (isDeleted !== undefined && isDeleted !== true) || (isDeleted === true) === (entryValue !== undefined)) throw new Error(`Sync record entry ${key} is invalid.`); parsed[key] = isDeleted ? { hlc, isDeleted: true } : { hlc, value: entryValue }; diff --git a/electron/sync-renderer.test.ts b/electron/sync-renderer.test.ts new file mode 100644 index 0000000..f4ca724 --- /dev/null +++ b/electron/sync-renderer.test.ts @@ -0,0 +1,78 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { syncKindInfo, type SyncSourceReply, type SyncSourceRequest } from '../shared/sync'; +import { SyncSourceUnavailableError } from './sync-engine'; +import { RendererSources } from './sync-renderer'; + +function setup(answer: (request: SyncSourceRequest) => SyncSourceReply | null, timeoutMs = 50) { + const asked: SyncSourceRequest[] = []; + let hasWindow = true; + const host: RendererSources = new RendererSources({ + timeoutMs, + send: (id, request) => { + if (!hasWindow) return false; + asked.push(request); + const reply = answer(request); + if (reply) queueMicrotask(() => host.receive(id, reply)); + return true; + }, + }); + const source = (kind: SyncSourceRequest['kind']) => host.sources().find((candidate) => candidate.kind === kind)!; + return { host, asked, source, closeWindow: () => { hasWindow = false; } }; +} + +test('nothing is asked before the renderer says its stores have loaded', async () => { + const { host, asked, source } = setup(() => ({ result: { values: {}, invalid: [] } })); + await assert.rejects(source('chart').snapshot(), SyncSourceUnavailableError); + host.setReady(true); + assert.deepEqual(await source('chart').snapshot(), {}); + assert.equal(asked.length, 1); +}); + +test('values a snapshot returned are checked without asking again; others are asked about', async () => { + const values = { c1: { symbol: 'EURUSD' }, c2: { symbol: '' } }; + const { host, asked, source } = setup((request) => request.method === 'snapshot' + ? { result: { values, invalid: ['c2'] } } + : request.method === 'validate' && (request.value as { symbol: string }).symbol ? { result: null } : { error: 'Chart c3 is not readable.' }); + host.setReady(true); + const chart = source('chart'); + const snapshot = await chart.snapshot(); + await chart.validate(snapshot.c1, 'c1'); + await assert.rejects(Promise.resolve(chart.validate(snapshot.c2, 'c2')), /failed its checks/); + assert.equal(asked.length, 1); + await chart.validate({ symbol: 'GBPUSD' }, 'c3'); + await assert.rejects(Promise.resolve(chart.validate({ symbol: '' }, 'c3')), (error: Error) => !(error instanceof SyncSourceUnavailableError) && /not readable/.test(error.message)); + assert.deepEqual(asked.slice(1).map((request) => request.method === 'validate' && request.id), ['c3', 'c3']); +}); + +test('a renderer that stops answering, reloads or has no window fails as unavailable', async () => { + const { host, source, closeWindow } = setup(() => null, 20); + host.setReady(true); + await assert.rejects(source('watchlists').snapshot(), SyncSourceUnavailableError); + const pending = source('watchlists').apply({ upserts: {}, deletes: [] }); + host.setReady(false); + await assert.rejects(pending, SyncSourceUnavailableError); + host.setReady(true); + closeWindow(); + await assert.rejects(source('watchlists').snapshot(), SyncSourceUnavailableError); +}); + +test('a malformed snapshot is refused rather than read as empty', async () => { + const { host, source } = setup(() => ({ result: { values: [], invalid: [] } })); + host.setReady(true); + await assert.rejects(source('chart').snapshot(), /malformed/); +}); + +test('a malformed reply rejects instead of hanging, and snapshots must be plain JSON', async () => { + const replies: unknown[] = [null, { result: { values: { c1: { at: new Date() } }, invalid: [] } }, { result: { values: { c1: { price: Number.NaN } }, invalid: [] } }]; + let index = 0; + const host: RendererSources = new RendererSources({ send: (id) => { const reply = replies[index++]; queueMicrotask(() => host.receive(id, reply)); return true; } }); + host.setReady(true); + const chart = host.sources().find((source) => source.kind === 'chart')!; + for (let attempt = 0; attempt < 3; attempt++) await assert.rejects(chart.snapshot(), /malformed/); +}); + +test('no secret kind is ever served by the renderer', () => { + const host = new RendererSources({ send: () => true }); + for (const source of host.sources()) assert.equal(syncKindInfo(source.kind).isSecret, false); +}); diff --git a/electron/sync-renderer.ts b/electron/sync-renderer.ts new file mode 100644 index 0000000..6d1c550 --- /dev/null +++ b/electron/sync-renderer.ts @@ -0,0 +1,104 @@ +import { rendererSyncKinds, syncKindInfo, type Json, type RendererSyncKind, type SyncSnapshot, type SyncSourceReply, type SyncSourceRequest } from '../shared/sync'; +import { SyncSourceUnavailableError, type SyncSource } from './sync-engine'; + +const REQUEST_TIMEOUT_MS = 30_000; +const MAX_DEPTH = 64; + +export interface RendererSourcesDeps { + // False when there is no window to ask. + readonly send: (id: number, request: SyncSourceRequest) => boolean; + readonly timeoutMs?: number; +} + +interface Waiting { + readonly resolve: (result: unknown) => void; + readonly reject: (error: Error) => void; + readonly timer: ReturnType; +} + +const isObject = (value: unknown): value is object => value !== null && typeof value === 'object'; + +// Structured clone carries Maps, Dates, NaN and cycles that JSON cannot; only plain JSON passes. +function isPlainJson(value: unknown, depth = 0): value is Json { + if (value === null || typeof value === 'string' || typeof value === 'boolean') return true; + if (typeof value === 'number') return Number.isFinite(value); + if (depth > MAX_DEPTH || typeof value !== 'object') return false; + if (Array.isArray(value)) return value.every((item) => isPlainJson(item, depth + 1)); + return Object.getPrototypeOf(value) === Object.prototype && Object.values(value).every((item) => isPlainJson(item, depth + 1)); +} + +// The renderer's stores, reached over IPC. The renderer checks its own values when it takes a +// snapshot, so the engine validating those again needs no round trip; only remote values do. +export class RendererSources { + private next = 1; + private isReady = false; + private readonly waiting = new Map(); + private readonly checked = new WeakSet(); + private readonly refused = new WeakSet(); + + constructor(private readonly deps: RendererSourcesDeps) {} + + // Ready once the renderer's stores have loaded; a reload or a closed window fails what was asked. + setReady(isReady: boolean): void { + this.isReady = isReady; + if (isReady) return; + for (const id of [...this.waiting.keys()]) this.fail(id, new SyncSourceUnavailableError('The window closed before it answered.')); + } + + receive(id: number, reply: unknown): void { + const waiting = this.waiting.get(id); + if (!waiting) return; + this.waiting.delete(id); + clearTimeout(waiting.timer); + const answer = reply as Partial | null; + if (!isObject(answer)) waiting.reject(new Error('The app window sent a malformed answer.')); + else if ('error' in answer) waiting.reject(new Error(String(answer.error))); + else waiting.resolve('result' in answer ? answer.result : undefined); + } + + // Secret kinds never go to the renderer, whatever the kind list says. + sources(): SyncSource[] { + return rendererSyncKinds.filter((kind) => !syncKindInfo(kind).isSecret).map((kind) => this.source(kind)); + } + + private source(kind: RendererSyncKind): SyncSource { + return { + kind, + snapshot: async () => { + const snapshot = await this.request({ method: 'snapshot', kind }) as SyncSnapshot; + if (!isObject(snapshot?.values) || Array.isArray(snapshot.values) || !Array.isArray(snapshot.invalid) || !isPlainJson(snapshot.values)) throw new Error(`The ${kind} snapshot is malformed.`); + const invalid = new Set(snapshot.invalid); + for (const [id, value] of Object.entries(snapshot.values)) { + if (isObject(value)) (invalid.has(id) ? this.refused : this.checked).add(value); + } + return snapshot.values; + }, + validate: async (value: Json, id: string) => { + if (isObject(value) && this.checked.has(value)) return; + if (isObject(value) && this.refused.has(value)) throw new Error(`This ${kind} record failed its checks.`); + await this.request({ method: 'validate', kind, id, value }); + }, + apply: async (change) => { + await this.request({ method: 'apply', kind, change }); + }, + }; + } + + private request(request: SyncSourceRequest): Promise { + if (!this.isReady) return Promise.reject(new SyncSourceUnavailableError('The app window is still loading.')); + const id = this.next++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => this.fail(id, new SyncSourceUnavailableError('The app window did not answer.')), this.deps.timeoutMs ?? REQUEST_TIMEOUT_MS); + this.waiting.set(id, { resolve, reject, timer }); + if (!this.deps.send(id, request)) this.fail(id, new SyncSourceUnavailableError('There is no app window.')); + }); + } + + private fail(id: number, error: Error): void { + const waiting = this.waiting.get(id); + if (!waiting) return; + this.waiting.delete(id); + clearTimeout(waiting.timer); + waiting.reject(error); + } +} diff --git a/electron/sync-service.test.ts b/electron/sync-service.test.ts index 7ff945f..5436f5c 100644 --- a/electron/sync-service.test.ts +++ b/electron/sync-service.test.ts @@ -8,9 +8,10 @@ import type { PasswordCipher } from './account-registry'; import { FakeGitHub } from './github-fixtures'; import { GitHubApi, GitHubRemote } from './github-remote'; import { newRootKey } from './sync-crypto'; -import { FakeTimers } from './sync-engine-fixtures'; +import type { SyncSource } from './sync-engine'; +import { FakeTimers, MemorySource } from './sync-engine-fixtures'; import { inspectRepository, updateKeyring, withDeviceWrap } from './sync-keyring'; -import { SyncService } from './sync-service'; +import { SyncService, type SyncServiceDeps } from './sync-service'; const REPO = 42; const cipher: PasswordCipher = { @@ -28,7 +29,7 @@ async function until(check: () => boolean): Promise { await settle(); } -async function computer(github: FakeGitHub, name: string, dir?: string, file?: string) { +async function computer(github: FakeGitHub, name: string, dir?: string, file?: string, sources: readonly SyncSource[] = [], extra: Partial = {}) { const folder = dir ?? await fs.mkdtemp(path.join(os.tmpdir(), 'openterm-sync-service-')); const timers = new FakeTimers(); const views: SyncView[] = []; @@ -37,7 +38,7 @@ async function computer(github: FakeGitHub, name: string, dir?: string, file?: s const service = new SyncService({ dir: folder, fetch: github.fetch, cipher, platform: 'linux', appVersion: '0.12.0', deviceName: name, send: (view) => views.push(view), journal: (message) => journal.push(message), openExternal: async (url) => { opened.push(url); }, chooseFile: async () => file ?? null, - now: () => 1_800_000_000_000 + timers.time, timers, pause: async () => undefined, passphraseCost: { N: 2 ** 14, r: 8, p: 1 }, + now: () => 1_800_000_000_000 + timers.time, timers, pause: async () => undefined, passphraseCost: { N: 2 ** 14, r: 8, p: 1 }, sources: () => sources, ...extra, }); await service.init(); const tick = async (ms = 0) => { @@ -56,8 +57,8 @@ async function signIn(device: Awaited>) { await until(() => device.view().userCode === null); } -async function setUpFirst(github: FakeGitHub, passphrase: string | null = null) { - const first = await computer(github, 'Office PC'); +async function setUpFirst(github: FakeGitHub, passphrase: string | null = null, sources: readonly SyncSource[] = [], extra: Partial = {}) { + const first = await computer(github, 'Office PC', undefined, undefined, sources, extra); await signIn(first); await first.service.handle('sync.chooseRepository', { id: REPO }); const view = await first.service.handle('sync.create', { passphrase }); @@ -353,3 +354,139 @@ test('without GitHub, two computers sync through a sync file', async () => { assert.equal(first.view().status, 'offline'); assert.equal(first.view().attention, null); }); + +test('a computer joining synced layouts asks whether to merge its own or use the synced ones', async () => { + const github = fakeGitHub(); + const office = new MemorySource('chart'); + office.data = { c1: { symbol: 'XAUUSD' } }; + const { recoveryCode } = await setUpFirst(github, null, [office]); + const laptop = new MemorySource('chart'); + laptop.data = { c9: { symbol: 'EURUSD' } }; + const second = await computer(github, 'Laptop', undefined, undefined, [laptop]); + await signIn(second); + await second.service.handle('sync.chooseRepository', { id: REPO }); + await second.service.handle('sync.join', { recoveryCode }); + await second.tick(60_000); + assert.equal(second.view().attention?.reason, 'join'); + assert.deepEqual(laptop.data, { c9: { symbol: 'EURUSD' } }); + await second.service.handle('sync.resolve', { choice: 'useSynced' }); + await second.tick(); + assert.equal(second.view().status, 'idle'); + assert.deepEqual(laptop.data, { c1: { symbol: 'XAUUSD' } }); +}); + +test('a computer joining a repository with no layouts yet syncs without asking', async () => { + const github = fakeGitHub(); + const { recoveryCode } = await setUpFirst(github, null, [new MemorySource('chart')]); + const laptop = new MemorySource('chart'); + laptop.data = { c9: { symbol: 'EURUSD' } }; + const second = await computer(github, 'Laptop', undefined, undefined, [laptop]); + await signIn(second); + await second.service.handle('sync.chooseRepository', { id: REPO }); + await second.service.handle('sync.join', { recoveryCode }); + await second.tick(); + assert.equal(second.view().attention, null); + assert.equal(second.view().status, 'idle'); +}); + +async function joinSecond(github: FakeGitHub, recoveryCode: string, sources: readonly SyncSource[], dir?: string) { + const second = await computer(github, 'Laptop', dir, undefined, sources); + await signIn(second); + await second.service.handle('sync.chooseRepository', { id: REPO }); + await second.service.handle('sync.join', { recoveryCode }); + await second.tick(); + return second; +} + +test('turning on one CID asks which to keep when the synced one differs, then adopts it', async () => { + const github = fakeGitHub(); + const officeCid = new MemorySource('identity'); + officeCid.data = { cid: { cid: 'a'.repeat(32), isUnique: true } }; + const { first, recoveryCode } = await setUpFirst(github, null, [officeCid]); + await first.service.handle('sync.setSharing', { isSecretsSynced: false, isIdentitySynced: true }); + await first.tick(); + assert.equal(first.view().isIdentitySynced, true); + const laptopCid = new MemorySource('identity'); + laptopCid.data = { cid: { cid: 'b'.repeat(32), isUnique: true } }; + const second = await joinSecond(github, recoveryCode, [laptopCid]); + await second.service.handle('sync.setSharing', { isSecretsSynced: false, isIdentitySynced: true }); + assert.deepEqual([second.view().isIdentityChoiceNeeded, second.view().isIdentitySynced], [true, false]); + await second.service.handle('sync.setSharing', { isSecretsSynced: false, isIdentitySynced: true, identityChoice: 'synced' }); + await second.tick(); + assert.deepEqual(laptopCid.data, officeCid.data); + assert.equal(second.view().isIdentityChoiceNeeded, false); +}); + +test('passwords stay on a computer until it opts in, and only computers that opted in receive them', async () => { + const github = fakeGitHub(); + const officeSecrets = new MemorySource('accountSecret'); + officeSecrets.data = { 'wss://broker.example/terminal|1': { secret: 'pw', savedAt: 1 } }; + const { first, recoveryCode } = await setUpFirst(github, null, [officeSecrets]); + const laptopSecrets = new MemorySource('accountSecret'); + const second = await joinSecond(github, recoveryCode, [laptopSecrets]); + await second.service.handle('sync.setSharing', { isSecretsSynced: true, isIdentitySynced: false }); + await second.tick(); + assert.deepEqual(laptopSecrets.data, {}); + await first.service.handle('sync.setSharing', { isSecretsSynced: true, isIdentitySynced: false }); + await first.tick(); + await second.service.handle('sync.syncNow', {}); + await second.tick(); + assert.deepEqual(laptopSecrets.data, officeSecrets.data); +}); + +test('the merge question survives a restart, and Merge keeps the synced settings', async () => { + const github = fakeGitHub(); + const office = new MemorySource('preferences'); + office.data = { workspace: { theme: 'dark' } }; + const { recoveryCode } = await setUpFirst(github, null, [office]); + const laptop = new MemorySource('preferences'); + laptop.data = { workspace: { theme: 'light', replayMode: 'allCharts' } }; + const second = await joinSecond(github, recoveryCode, [laptop]); + assert.equal(second.view().attention?.reason, 'join'); + second.service.shutdown(); + const restarted = await computer(github, 'Laptop', second.dir, undefined, [laptop]); + await restarted.tick(60_000); + assert.equal(restarted.view().attention?.reason, 'join'); + assert.deepEqual(laptop.data, { workspace: { theme: 'light', replayMode: 'allCharts' } }); + await restarted.service.handle('sync.resolve', { choice: 'merge' }); + await restarted.tick(); + assert.deepEqual(laptop.data, { workspace: { theme: 'dark', replayMode: 'allCharts' } }); + assert.equal(restarted.view().attention, null); +}); + +test('an account open on one computer shows there, and moving it here logs that computer out', async () => { + const github = fakeGitHub(); + const account = 'wss://broker.example/terminal|1234'; + let live = [{ id: account, since: 0 }]; + const released: [readonly string[], string][] = []; + const { first, recoveryCode } = await setUpFirst(github, null, [], { + liveAccounts: () => live, + releaseAccounts: async (ids, to) => { released.push([ids, to]); live = []; }, + }); + first.service.presenceChanged(); + await first.tick(5000); + const second = await joinSecond(github, recoveryCode, []); + assert.deepEqual(await second.service.whereOpen(account), ['Office PC']); + assert.equal(await second.service.moveHere(account), false); + await first.service.handle('sync.syncNow', {}); + await first.tick(); + assert.deepEqual(released, [[[account], 'Laptop']]); + assert.deepEqual(await second.service.whereOpen(account), []); + assert.equal(await second.service.moveHere(account), true); +}); + +test('a login made after the move request is a choice to come back, so it stays', async () => { + const github = fakeGitHub(); + const account = 'wss://broker.example/terminal|1234'; + let live = [{ id: account, since: 0 }]; + const released: string[] = []; + const { first, recoveryCode } = await setUpFirst(github, null, [], { liveAccounts: () => live, releaseAccounts: async (ids) => { released.push(...ids); } }); + first.service.presenceChanged(); + await first.tick(5000); + const second = await joinSecond(github, recoveryCode, []); + await second.service.moveHere(account); + live = [{ id: account, since: Number.MAX_SAFE_INTEGER }]; + await first.service.handle('sync.syncNow', {}); + await first.tick(); + assert.deepEqual(released, []); +}); diff --git a/electron/sync-service.ts b/electron/sync-service.ts index dc085e5..76a2b0d 100644 --- a/electron/sync-service.ts +++ b/electron/sync-service.ts @@ -1,7 +1,7 @@ import { randomInt } from 'node:crypto'; import path from 'node:path'; import { - normalizeCodeGroup, passphraseProblem, type SyncAttention, type SyncAttentionReason, type SyncDeviceView, type SyncMassChange, type SyncMethod, type SyncMethods, + isSameJson, normalizeCodeGroup, passphraseProblem, recordKey, rendererSyncKinds, splitRecordKey, syncKindInfo, type SyncSharing, type SyncAttention, type SyncAttentionReason, type SyncDeviceView, type SyncMassChange, type SyncMethod, type SyncMethods, type SyncPage, type SyncPairingRequest, type SyncPhase, type SyncRepositoryChoice, type SyncResolution, type SyncStatus, type SyncView, } from '../shared/sync'; import type { PasswordCipher } from './account-registry'; @@ -9,14 +9,17 @@ import { FileRemote, SyncFileDamagedError, SyncFileUnavailableError } from './fi import { GITHUB_APP_SLUG, GITHUB_WEB, GitHubAuth, GitHubAuthError, type DeviceCode, type Fetch } from './github-auth'; import { GitHubApi, GitHubNetworkError, GitHubRateLimitError, GitHubRemote, GitHubRequestError } from './github-remote'; import { deriveKeys, newDeviceKeyPair, SyncCryptoError } from './sync-crypto'; -import { SyncEngine, type SyncOptions, type SyncRemote, type SyncSource } from './sync-engine'; +import { SyncEngine, SyncSourceUnavailableError, type SyncOptions, type SyncRemote, type SyncSource } from './sync-engine'; import { answerPairing, FORMAT_PATH, inspectRepository, newKeyring, PAIRING_LIFETIME_MS, pairingConfirmation, pairingWords, requestPairing, SyncSetupError, unlockKeyring, updateKeyring, withDeviceWrap, withNewRecovery, type RepositorySetup, } from './sync-keyring'; -import { MANIFEST_PATH, openManifest, SyncIntegrityError } from './sync-manifest'; +import { IDENTITY_ID } from './sync-main-sources'; +import { + HANDOFF_LIFETIME_MS, isPresenceDue, liveHandoffs, MANIFEST_PATH, openManifest, presentElsewhere, SyncIntegrityError, withPresence, type Manifest, +} from './sync-manifest'; import { realTimers, SyncScheduler, type RunResult, type Timers } from './sync-scheduler'; -import { CredentialStore, EncryptedStateStore, type SyncTarget } from './sync-storage'; +import { CredentialStore, EncryptedStateStore, type PendingJoin, type SyncTarget } from './sync-storage'; const REPOSITORY_CHECK_MS = 3_600_000; const PAIRING_POLL_MS = 5000; @@ -24,6 +27,17 @@ const PAIRING_ATTEMPTS = 3; const CODE_GROUPS = 7; // A sync file has no repository id, so it gets a random one that binds its data the same way. const MAX_GROUP_ID = 2 ** 47; +const PRESENCE_DELAY_MS = 2000; +const PEEK_TIMEOUT_MS = 5000; +const HANDOFF_POLL_MS = 3000; +const HANDOFF_POLLS = 20; + +// An account with a live session here, and since when (local time). Background sessions report +// Infinity: they are not handed over until copiers and signals have leases. +export interface LiveAccount { + readonly id: string; + readonly since: number; +} export interface SyncServiceDeps { readonly dir: string; @@ -38,6 +52,9 @@ export interface SyncServiceDeps { readonly revealFile?: (file: string) => void; readonly chooseFile?: (mode: 'new' | 'open') => Promise; readonly sources?: () => readonly SyncSource[]; + readonly liveAccounts?: () => readonly LiveAccount[]; + // Another computer moved these accounts to itself: log out of them here. + readonly releaseAccounts?: (accountIds: readonly string[], to: string) => Promise; readonly now?: () => number; readonly timers?: Timers; readonly pause?: (ms: number, signal?: AbortSignal) => Promise; @@ -48,6 +65,15 @@ const heldMessage = (changes: readonly SyncMassChange[]) => changes.some((change ? 'Much of the data on this computer would be deleted from every computer. Choose whether to delete it everywhere or restore it from sync.' : 'Another computer deleted much of the synced data. Choose whether to delete it here too.'; +const joinMessage = 'This computer has layouts, drawings and lists of its own, and the synced data has some too. Merge adds this computer\'s to the synced ones on every computer; Use synced data replaces this computer\'s with the synced ones.'; + +// A joining computer's first merge: the synced value wins where both hold one. Use synced data +// replaces layouts and lists; saved accounts and their passwords always merge. +const joinOptions: Record, SyncOptions> = { + merge: { isRemotePreferred: true }, + useSynced: { isRemotePreferred: true, isLocalDiscarded: true, isMassChangeAllowed: true, discardedKinds: rendererSyncKinds }, +}; + const groupIdOf = (target: SyncTarget) => target.kind === 'github' ? target.repository.id : target.groupId; const labelOf = (target: SyncTarget) => target.kind === 'github' ? target.repository.fullName : target.path; @@ -72,6 +98,10 @@ export class SyncService { private pairingRequests: SyncPairingRequest[] = []; private pairingConfirmation: string[] = []; private pairingAttempts = 0; + private isIdentityChoiceNeeded = false; + private presenceTimer: unknown = null; + private hasPresence = false; + private readonly answeredHandoffs = new Set(); // A key that arrived for this computer, held until the confirmation words prove it is the real one. private pairedKey: { readonly rootKey: Buffer; readonly epoch: number } | null = null; // The public key behind each request whose words were shown, so approval binds to what was checked. @@ -108,6 +138,7 @@ export class SyncService { this.phase = 'on'; this.startEngine(target); if (target.kind === 'github' && !this.auth.isSignedIn) this.attend('reconnect', 'Reconnect GitHub to keep syncing.'); + else if (this.credentials.get().pendingJoin === 'ask') this.attend('join', joinMessage); else this.scheduler.start(); } else if (target && (target.kind === 'file' || this.auth.isSignedIn)) { [this.choice, this.phase] = [target, 'joining']; @@ -120,6 +151,7 @@ export class SyncService { shutdown(): void { this.scheduler.stop(); + this.timers.clear(this.presenceTimer); this.stopPairingPoll(); this.signIn?.abort(); } @@ -130,6 +162,54 @@ export class SyncService { this.scheduler.localChange(); } + // A session started or ended here; presence is written shortly after, without the stores. + presenceChanged(): void { + if (!this.engine || this.presenceTimer) return; + this.presenceTimer = this.timers.set(() => { + this.presenceTimer = null; + void this.writePresence(); + }, PRESENCE_DELAY_MS); + } + + // Names of the other computers with a live session on the account, from the remote when it + // answers in time, else from the last sync. + async whereOpen(accountId: string): Promise { + const manifest = await this.currentManifest(); + if (!manifest || !this.engine) return []; + const ids = presentElsewhere(manifest.presence, accountId, this.credentials.get().deviceId, this.engine.serverNow()); + return ids.map((id) => manifest.devices.find((device) => device.id === id)?.name || 'another computer'); + } + + // Asks the computers holding the account to log out of it, and waits a minute for them to. + async moveHere(accountId: string): Promise { + const engine = this.engine; + if (!engine) return true; + const { deviceId } = this.credentials.get(); + const manifest = await this.currentManifest(); + const holders = manifest ? presentElsewhere(manifest.presence, accountId, deviceId, engine.serverNow()) : []; + if (!holders.length) return true; + await engine.editManifest((current, now) => ({ + ...current, handoffs: { ...liveHandoffs(current.handoffs, now), [accountId]: { from: deviceId, to: holders, createdAt: now, expiresAt: now + HANDOFF_LIFETIME_MS } }, + })); + const pause = this.deps.pause ?? ((ms: number) => new Promise((resolve) => setTimeout(resolve, ms))); + for (let poll = 0; poll < HANDOFF_POLLS; poll++) { + await pause(HANDOFF_POLL_MS); + const latest = await engine.peekManifest().catch(() => null); + if (latest && !presentElsewhere(latest.presence, accountId, deviceId, engine.serverNow()).length) return true; + } + return false; + } + + // At quit: edits pushed, and this computer no longer listed as holding any account. + async leave(): Promise { + if (this.hasUnpushedChanges) await this.flush(); + if (this.hasPresence) await this.writePresence([]); + } + + get isLeaveNeeded(): boolean { + return this.hasUnpushedChanges || this.hasPresence; + } + get hasUnpushedChanges(): boolean { return !!this.engine && !this.attention && this.changedAt > 0 && this.changedAt >= this.pushedAt; } @@ -139,7 +219,8 @@ export class SyncService { if (!this.engine || this.attention) return; this.scheduler.stop(); await this.engine.recordLocalChanges().catch(() => undefined); - await this.run({}); + // Closing the window on macOS leaves the app running, so syncing carries on afterwards. + if ((await this.run({})).kind !== 'stop') this.scheduler.start(); } setFocused(isFocused: boolean): void { @@ -171,6 +252,7 @@ export class SyncService { else if (method === 'sync.syncNow') this.syncNow(); else if (method === 'sync.resolve') await this.resolve((params as SyncMethods['sync.resolve'][0]).choice); else if (method === 'sync.newRecoveryCode') await this.newRecoveryCode(); + else if (method === 'sync.setSharing') await this.setSharing(params as SyncMethods['sync.setSharing'][0]); else if (method === 'sync.disconnect') await this.disconnect(); else throw new Error(`Unknown sync method ${method}`); this.emit(); @@ -193,6 +275,8 @@ export class SyncService { repositories: this.repositories, pairingWords: this.pairingWords, pairingRequests: this.pairingRequests, canKeepSecrets: this.credentials.canKeepSecrets, isPairingApproved: !!this.pairedKey, pairingConfirmation: this.pairingConfirmation, isRecoveryConfirmed: credentials.isRecoveryConfirmed, + isSecretsSynced: credentials.isSecretsSynced && this.credentials.canKeepSecrets, isIdentitySynced: credentials.isIdentitySynced, + isIdentityChoiceNeeded: this.isIdentityChoiceNeeded, }; } @@ -375,7 +459,27 @@ export class SyncService { await this.credentials.update({ target, rootKey, deviceKeys, epoch, isRecoveryConfirmed: true }); [this.phase, this.pairingWords] = ['on', []]; this.startEngine(target); - this.scheduler.start(); + if (await this.hasSyncedContent(target, rootKey, epoch)) { + await this.credentials.update({ pendingJoin: 'ask' }); + this.attend('join', joinMessage); + } else this.scheduler.start(); + } + + // A computer joining for the first time asks before its own data meets the synced data. When the + // repository cannot be read now it asks anyway, so nothing merges without a choice. + private async hasSyncedContent(target: SyncTarget, rootKey: Buffer, epoch: number): Promise { + if (await this.stateStore?.load()) return false; + try { + const remote = this.remoteFor(target); + const head = await remote.head(); + const envelope = head === null ? null : (await remote.read(head, [MANIFEST_PATH])).get(MANIFEST_PATH); + if (!envelope) return false; + const manifest = openManifest(deriveKeys(rootKey), envelope, { epoch, repoId: groupIdOf(target), highWater: 0 }); + const kinds = new Set(rendererSyncKinds); + return Object.entries(manifest.records).some(([key, entry]) => entry.hash !== null && kinds.has(splitRecordKey(key).kind)); + } catch { + return true; + } } // The new computer posts its public key and waits for a linked computer to wrap the key to it. @@ -513,7 +617,14 @@ export class SyncService { } private async resolve(choice: SyncResolution): Promise { + if (this.attention?.reason === 'join' || this.credentials.get().pendingJoin === 'ask') { + await this.credentials.update({ pendingJoin: choice === 'useSynced' ? 'useSynced' : 'merge' }); + this.attention = null; + if ((await this.run({})).kind !== 'stop') this.scheduler.start(); + return; + } const options: Record = { + merge: {}, allowDeletions: { isMassChangeAllowed: true }, useSynced: { isLocalDiscarded: true, isMassChangeAllowed: true }, acceptHistory: { isHistoryAccepted: true, isLocalDiscarded: true, isMassChangeAllowed: true }, @@ -544,11 +655,96 @@ export class SyncService { this.choice = null; this.stateStore = new EncryptedStateStore(path.join(this.deps.dir, 'sync-state.json'), keys.state, { epoch, repoId }); this.engine = new SyncEngine({ - epoch, repoId, keys, remote: this.remoteFor(target), store: this.stateStore, sources: this.deps.sources?.() ?? [], now: this.now, + epoch, repoId, keys, remote: this.remoteFor(target), store: this.stateStore, sources: this.engineSources(), now: this.now, device: { id: deviceId, name: this.deps.deviceName.slice(0, 256), platform: this.deps.platform, publicKey: deviceKeys?.publicKey ?? '', appVersion: this.deps.appVersion }, }); } + // Kinds this computer opted into. Secret kinds also need the keychain; while the synced CID is + // being adopted, this computer offers none of its own, so the synced one is taken as it is. + private engineSources(): SyncSource[] { + const { isSecretsSynced, isIdentitySynced, isIdentityAdopting } = this.credentials.get(); + return (this.deps.sources?.() ?? []).flatMap((source): SyncSource[] => { + if (source.kind === 'identity') { + if (!isIdentitySynced) return []; + return [isIdentityAdopting ? { kind: source.kind, snapshot: async () => ({}), validate: (value, id) => source.validate(value, id), apply: (change) => source.apply(change) } : source]; + } + return syncKindInfo(source.kind).isSecret && !(isSecretsSynced && this.credentials.canKeepSecrets) ? [] : [source]; + }); + } + + // Turning on one CID when another one is already synced asks which to keep, unless the choice + // came with the request. Each change restarts the engine with the kinds now chosen. + private async setSharing({ isSecretsSynced, isIdentitySynced, identityChoice }: SyncSharing): Promise { + if (!this.engine) throw new Error('Sync is not set up on this computer.'); + if (isSecretsSynced && !this.credentials.canKeepSecrets) throw new Error('This computer has no secure storage for passwords, so they cannot sync here.'); + const current = this.credentials.get(); + let isIdentityAdopting = current.isIdentityAdopting && isIdentitySynced; + if (isIdentitySynced && !current.isIdentitySynced) { + const source = this.deps.sources?.().find((candidate) => candidate.kind === 'identity'); + if (!source) throw new Error('This computer cannot sync its CID.'); + const synced = await this.engine.peek(recordKey('identity', IDENTITY_ID)); + const own = (await source.snapshot())[IDENTITY_ID]; + const isDifferent = synced !== undefined && !isSameJson(synced, own); + if (isDifferent && !identityChoice) { + this.isIdentityChoiceNeeded = true; + return; + } + isIdentityAdopting = isDifferent && identityChoice === 'synced'; + } + this.isIdentityChoiceNeeded = false; + this.scheduler.stop(); + await this.running?.catch(() => undefined); + await this.credentials.update({ isSecretsSynced, isIdentitySynced, isIdentityAdopting }); + this.startEngine(this.target()); + if (!this.attention) this.scheduler.start(); + } + + // The synced CID has arrived once it is a merge base; from then on this computer's CID syncs. + private async finishAdopting(): Promise { + if (!this.credentials.get().isIdentityAdopting) return; + const state = await this.stateStore?.load(); + if (!state?.bases[recordKey('identity', IDENTITY_ID)]) return; + await this.credentials.update({ isIdentityAdopting: false }); + this.startEngine(this.target()); + } + + private async currentManifest(): Promise { + if (!this.engine) return null; + const late = new Promise((resolve) => this.timers.set(() => resolve(null), PEEK_TIMEOUT_MS)); + const fresh = await Promise.race([this.engine.peekManifest().catch(() => null), late]); + return fresh ?? (await this.stateStore?.load())?.manifest ?? null; + } + + private async writePresence(accounts = (this.deps.liveAccounts?.() ?? []).map((account) => account.id)): Promise { + if (!this.engine || this.attention) return; + const { deviceId } = this.credentials.get(); + const isWritten = await this.engine.editManifest((manifest, now) => + isPresenceDue(manifest.presence, deviceId, accounts, now) ? { ...manifest, presence: withPresence(manifest.presence, deviceId, accounts, now) } : null).catch(() => false); + if (isWritten || !accounts.length) this.hasPresence = accounts.length > 0; + } + + // Requests from computers that moved an account to themselves after this one logged in to it. + // A login here made after the request was a choice to come back, so it stays. + private async answerHandoffs(): Promise { + const engine = this.engine; + const manifest = (await this.stateStore?.load())?.manifest; + if (!engine || !manifest || !this.deps.releaseAccounts) return; + const { deviceId } = this.credentials.get(); + const offset = engine.serverNow() - this.now(); + const live = new Map((this.deps.liveAccounts?.() ?? []).map((account) => [account.id, account.since + offset])); + const byComputer = new Map(); + for (const [account, request] of Object.entries(liveHandoffs(manifest.handoffs, engine.serverNow()))) { + const key = `${account}|${request.from}|${request.createdAt}`; + if (!request.to.includes(deviceId) || !(live.get(account)! < request.createdAt) || this.answeredHandoffs.has(key)) continue; + this.answeredHandoffs.add(key); + const name = manifest.devices.find((device) => device.id === request.from)?.name || 'another computer'; + byComputer.set(name, [...byComputer.get(name) ?? [], account]); + } + for (const [name, released] of byComputer) await this.deps.releaseAccounts(released, name); + if (byComputer.size) await this.writePresence(); + } + private async stamp(): Promise { await this.engine?.recordLocalChanges(); } @@ -562,6 +758,8 @@ export class SyncService { private async runOnce(options: SyncOptions): Promise { if (!this.engine) return { kind: 'stop' }; + const join = this.credentials.get().pendingJoin; + if (join === 'ask') return this.attend('join', joinMessage); const startedAt = this.now(); const target = this.target(); this.status = 'syncing'; @@ -569,14 +767,18 @@ export class SyncService { try { const problem = await this.checkRepository(target); if (problem) return this.attend(problem.reason, problem.message); - const outcome = await this.engine.sync(options); + const outcome = await this.engine.sync(join ? { ...options, ...joinOptions[join] } : options); if (outcome.status === 'held') return this.attend('held', heldMessage(outcome.changes), outcome.changes); if (outcome.status === 'busy') { this.status = 'idle'; return { kind: 'retry' }; } + if (join) await this.credentials.update({ pendingJoin: null }); + await this.answerHandoffs(); + await this.writePresence(); await this.refreshDevices(); await this.refreshPairing(target); + await this.finishAdopting(); this.pushedAt = startedAt + 1; [this.status, this.attention, this.error, this.conflicts, this.lastSyncedAt] = ['idle', null, null, outcome.conflicts.length, this.now()]; if (outcome.conflicts.length) this.deps.journal(`${outcome.conflicts.length} edit(s) made on two computers at once were settled in favour of the newer one.`); @@ -619,6 +821,10 @@ export class SyncService { private failed(error: Error): RunResult { if (error instanceof SyncIntegrityError) return this.attend(error.reason, error.message); + if (error instanceof SyncSourceUnavailableError) { + [this.status, this.error] = ['loading', error.message]; + return { kind: 'retry' }; + } if (error instanceof GitHubAuthError && error.reason === 'reconnect') return this.attend('reconnect', error.message); if (error instanceof GitHubRateLimitError) { [this.status, this.error] = ['waiting', error.message]; diff --git a/electron/sync-storage.ts b/electron/sync-storage.ts index 77b89b0..61e6584 100644 --- a/electron/sync-storage.ts +++ b/electron/sync-storage.ts @@ -24,8 +24,20 @@ export interface SyncCredentials { readonly deviceKeys: DeviceKeyPair | null; readonly target: SyncTarget | null; readonly isRecoveryConfirmed: boolean; + // This computer's opt-ins: saved passwords, API keys and webhook secrets; one CID for all. + readonly isSecretsSynced: boolean; + readonly isIdentitySynced: boolean; + // Set when turning on one CID chose the synced one: this computer offers none until it arrives. + readonly isIdentityAdopting: boolean; + // A first join into synced content: 'ask' until the user chooses, then the choice until a + // sync with it succeeds, so a restart in between neither skips the question nor the choice. + readonly pendingJoin: PendingJoin | null; } +export type PendingJoin = 'ask' | 'merge' | 'useSynced'; + +const pendingJoins: readonly PendingJoin[] = ['ask', 'merge', 'useSynced']; + interface StoredCredentials { readonly epoch: number; readonly deviceId: string; @@ -33,6 +45,10 @@ interface StoredCredentials { readonly secrets: string | null; readonly target: SyncTarget | null; readonly isRecoveryConfirmed: boolean; + readonly isSecretsSynced: boolean; + readonly isIdentitySynced: boolean; + readonly isIdentityAdopting: boolean; + readonly pendingJoin: PendingJoin | null; } interface Secrets { @@ -62,6 +78,10 @@ function parseStored(value: unknown): StoredCredentials { secrets: typeof stored.secrets === 'string' ? stored.secrets : null, target: parseTarget(stored.target), isRecoveryConfirmed: stored.isRecoveryConfirmed !== false, + isSecretsSynced: stored.isSecretsSynced === true, + isIdentitySynced: stored.isIdentitySynced === true, + isIdentityAdopting: stored.isIdentityAdopting === true, + pendingJoin: pendingJoins.find((join) => join === stored.pendingJoin) ?? null, }; } @@ -111,7 +131,10 @@ export class CredentialStore { // Forgets everything but this computer's id, so it stays one device to the repository. async clear(): Promise { const { deviceId } = this.get(); - this.current = { epoch: 1, deviceId, login: null, rootKey: null, tokens: null, deviceKeys: null, target: null, isRecoveryConfirmed: true }; + this.current = { + epoch: 1, deviceId, login: null, rootKey: null, tokens: null, deviceKeys: null, target: null, isRecoveryConfirmed: true, + isSecretsSynced: false, isIdentitySynced: false, isIdentityAdopting: false, pendingJoin: null, + }; await this.write(this.current); return this.current; } diff --git a/packages/meridian-charts/src/index.ts b/packages/meridian-charts/src/index.ts index 758f01c..1599f6e 100644 --- a/packages/meridian-charts/src/index.ts +++ b/packages/meridian-charts/src/index.ts @@ -24,6 +24,7 @@ export { type MeridianUiTokens, type ThemePatch, } from './themes'; +export { customColorList, onCustomColorsChange, setCustomColors } from './ui/color-picker'; export { iconSvg } from './ui/icons'; export { DrawingToolbar, type CursorMode, type MagnetMode, type ToolbarActions, type ToolbarState } from './ui/toolbar'; export { ensureStyles } from './ui/styles'; diff --git a/packages/meridian-charts/src/ui/color-picker.ts b/packages/meridian-charts/src/ui/color-picker.ts index 74f5b0b..e35ba75 100644 --- a/packages/meridian-charts/src/ui/color-picker.ts +++ b/packages/meridian-charts/src/ui/color-picker.ts @@ -29,6 +29,21 @@ function saveCustom(colors: readonly string[]): void { // Shared by every picker in the window, as the custom row is in TradingView. let customColors = loadCustom(); +const customListeners = new Set<() => void>(); + +// For a host that keeps the custom row elsewhere too (sync between computers). +export const customColorList = (): readonly string[] => customColors; + +export function setCustomColors(colors: readonly string[]): void { + customColors = colors.filter((color) => /^#[0-9a-f]{6}$/i.test(color)).slice(0, CUSTOM_LIMIT); + saveCustom(customColors); +} + +export function onCustomColorsChange(listener: () => void): () => void { + customListeners.add(listener); + return () => customListeners.delete(listener); +} + // Places a popup under `anchor` inside `host`, flipping above it when there is no room below. export function placePopup(popup: HTMLElement, host: HTMLElement, anchor: HTMLElement): void { const hostBox = host.getBoundingClientRect(); @@ -112,6 +127,7 @@ export function openColorPicker(host: HTMLElement, anchor: HTMLElement, current: const color = input.value.toLowerCase(); customColors = [color, ...customColors.filter((candidate) => candidate !== color)].slice(0, CUSTOM_LIMIT); saveCustom(customColors); + customListeners.forEach((listener) => listener()); renderCustom(); choose(color); }; diff --git a/shared/sync.ts b/shared/sync.ts index 6b03ae3..ec4e380 100644 --- a/shared/sync.ts +++ b/shared/sync.ts @@ -12,40 +12,48 @@ export type Json = null | boolean | number | string | readonly Json[] | { readon // - value: one value; the newer write wins. export type SyncShape = 'fields' | 'map' | 'set' | 'list' | 'text' | 'value'; -// Declaration order is the order kinds sync in and are listed in. Raise `schema` when a kind's body -// changes shape; older builds then leave records of the newer schema alone instead of rewriting them. +// Declaration order is the order kinds sync in and are listed in: a kind applies after the kinds it +// refers to (scripts and themes before the charts using them, charts before the layouts holding +// them). Raise `schema` when a kind's body changes shape; older builds then leave records of the +// newer schema alone instead of rewriting them. `side` is the process whose stores hold the kind. export const syncKinds = [ - { key: 'account', label: 'Accounts', shape: 'fields', schema: 1, isSecret: false }, - { key: 'accountSecret', label: 'Saved passwords and API keys', shape: 'value', schema: 1, isSecret: true }, - { key: 'identity', label: 'CID', shape: 'value', schema: 1, isSecret: true }, - { key: 'webhookSecrets', label: 'Webhook settings', shape: 'fields', schema: 1, isSecret: true }, - { key: 'layout', label: 'Layouts', shape: 'fields', schema: 1, isSecret: false }, - { key: 'layoutOrder', label: 'Layout order', shape: 'list', schema: 1, isSecret: false }, - { key: 'chart', label: 'Charts', shape: 'fields', schema: 1, isSecret: false }, - { key: 'chartDrawings', label: 'Chart drawings', shape: 'map', schema: 1, isSecret: false }, - { key: 'symbolDrawings', label: 'Symbol drawings', shape: 'map', schema: 1, isSecret: false }, - { key: 'chartProfile', label: 'Chart profiles', shape: 'fields', schema: 1, isSecret: false }, - { key: 'chartTemplate', label: 'Chart templates', shape: 'fields', schema: 1, isSecret: false }, - { key: 'drawingTemplates', label: 'Drawing templates', shape: 'map', schema: 1, isSecret: false }, - { key: 'drawingLooks', label: 'Drawing looks', shape: 'map', schema: 1, isSecret: false }, - { key: 'theme', label: 'Themes', shape: 'fields', schema: 1, isSecret: false }, - { key: 'watchlists', label: 'Watchlists', shape: 'map', schema: 1, isSecret: false }, - { key: 'indicatorFavorites', label: 'Indicator favourites', shape: 'set', schema: 1, isSecret: false }, - { key: 'timeframeFavorites', label: 'Timeframe favourites', shape: 'set', schema: 1, isSecret: false }, - { key: 'drawingFavorites', label: 'Drawing favourites', shape: 'set', schema: 1, isSecret: false }, - { key: 'customColors', label: 'Custom colours', shape: 'list', schema: 1, isSecret: false }, - { key: 'priceAlert', label: 'Price alerts', shape: 'fields', schema: 1, isSecret: false }, - { key: 'pineScript', label: 'Pine scripts', shape: 'text', schema: 1, isSecret: false }, - { key: 'preferences', label: 'Preferences', shape: 'fields', schema: 1, isSecret: false }, - { key: 'journalingLink', label: 'Journaling connection', shape: 'fields', schema: 1, isSecret: false }, - { key: 'copierConfig', label: 'Copier settings', shape: 'fields', schema: 1, isSecret: false }, - { key: 'automationRule', label: 'Automation rules', shape: 'fields', schema: 1, isSecret: false }, -] as const satisfies readonly { key: string; label: string; shape: SyncShape; schema: number; isSecret: boolean }[]; + { key: 'account', label: 'Accounts', shape: 'fields', schema: 1, isSecret: false, side: 'main' }, + { key: 'accountSecret', label: 'Saved passwords and API keys', shape: 'value', schema: 1, isSecret: true, side: 'main' }, + { key: 'identity', label: 'CID', shape: 'value', schema: 1, isSecret: true, side: 'main' }, + { key: 'webhookSecrets', label: 'Webhook settings', shape: 'fields', schema: 1, isSecret: true, side: 'main' }, + { key: 'theme', label: 'Themes', shape: 'value', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'pineScript', label: 'Pine scripts', shape: 'text', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'chart', label: 'Charts', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'chartDrawings', label: 'Chart drawings', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'layout', label: 'Layouts', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'layoutCharts', label: 'Layout charts', shape: 'list', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'layoutDrawings', label: 'Layout drawings', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'layoutOrder', label: 'Layout order', shape: 'list', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'symbolDrawings', label: 'Symbol drawings', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'chartProfile', label: 'Chart profiles', shape: 'value', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'chartTemplate', label: 'Chart templates', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'drawingTemplates', label: 'Drawing templates', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'drawingLooks', label: 'Drawing looks', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'watchlists', label: 'Watchlists', shape: 'map', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'indicatorFavorites', label: 'Indicator favourites', shape: 'set', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'timeframeFavorites', label: 'Timeframe favourites', shape: 'set', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'drawingFavorites', label: 'Drawing favourites', shape: 'set', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'customColors', label: 'Custom colours', shape: 'list', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'priceAlert', label: 'Price alerts', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'preferences', label: 'Preferences', shape: 'fields', schema: 1, isSecret: false, side: 'renderer' }, + { key: 'journalingLink', label: 'Journaling connection', shape: 'fields', schema: 1, isSecret: false, side: 'main' }, + { key: 'copierConfig', label: 'Copier settings', shape: 'fields', schema: 1, isSecret: false, side: 'main' }, + { key: 'automationRule', label: 'Automation rules', shape: 'fields', schema: 1, isSecret: false, side: 'main' }, +] as const satisfies readonly { key: string; label: string; shape: SyncShape; schema: number; isSecret: boolean; side: 'main' | 'renderer' }[]; export type SyncKind = (typeof syncKinds)[number]['key']; +export type RendererSyncKind = Extract<(typeof syncKinds)[number], { side: 'renderer' }>['key']; + export const isSyncKind = (value: string): value is SyncKind => syncKinds.some((kind) => kind.key === value); +export const rendererSyncKinds: readonly RendererSyncKind[] = syncKinds.flatMap((kind) => kind.side === 'renderer' ? [kind.key] : []); + export const syncKindInfo = (kind: SyncKind) => syncKinds.find((item) => item.key === kind)!; export const recordKey = (kind: SyncKind, id: string) => `${kind}:${id}`; @@ -90,6 +98,7 @@ export const syncStatuses = [ { key: 'syncing', label: 'Syncing' }, { key: 'offline', label: 'Offline' }, { key: 'waiting', label: 'Waiting for GitHub' }, + { key: 'loading', label: 'Waiting for the app window' }, { key: 'attention', label: 'Needs attention' }, ] as const; @@ -97,7 +106,7 @@ export type SyncStatus = (typeof syncStatuses)[number]['key']; export type SyncAttentionReason = | 'tampered' | 'rollback' | 'epoch' | 'tooNew' | 'stale' | 'revoked' - | 'held' | 'reconnect' | 'repositoryPublic' | 'repositoryMissing' | 'failed'; + | 'held' | 'join' | 'reconnect' | 'repositoryPublic' | 'repositoryMissing' | 'failed'; export interface SyncMassChange { readonly kind: SyncKind; @@ -159,11 +168,25 @@ export interface SyncView { readonly pairingConfirmation: readonly string[]; readonly canKeepSecrets: boolean; readonly isRecoveryConfirmed: boolean; + // Opt-ins of this computer: saved passwords, API keys and webhook secrets; one CID for all. + readonly isSecretsSynced: boolean; + readonly isIdentitySynced: boolean; + // Turning on one CID found a different one already synced: which to keep waits for the user. + readonly isIdentityChoiceNeeded: boolean; } export type SyncPage = 'install' | 'verification' | 'authorizations' | 'repository' | 'newRepository'; -export type SyncResolution = 'allowDeletions' | 'useSynced' | 'acceptHistory'; +export type SyncResolution = 'merge' | 'allowDeletions' | 'useSynced' | 'acceptHistory'; + +// Which CID to keep when this computer's and the synced one differ. +export type SyncIdentityChoice = 'synced' | 'this'; + +export interface SyncSharing { + readonly isSecretsSynced: boolean; + readonly isIdentitySynced: boolean; + readonly identityChoice?: SyncIdentityChoice; +} export interface SyncMethods { 'sync.view': [Record, SyncView]; @@ -184,6 +207,7 @@ export interface SyncMethods { 'sync.syncNow': [Record, SyncView]; 'sync.resolve': [{ choice: SyncResolution }, SyncView]; 'sync.newRecoveryCode': [Record, SyncView]; + 'sync.setSharing': [SyncSharing, SyncView]; 'sync.disconnect': [Record, SyncView]; } @@ -195,9 +219,33 @@ export interface SyncEvents { export type SyncEvent = keyof SyncEvents; +export interface SyncSourceChange { + readonly upserts: Readonly>; + readonly deletes: readonly string[]; +} + +// A snapshot also names the records that fail their own checks, so they are left out of sync +// instead of being read as deleted. +export interface SyncSnapshot { + readonly values: Readonly>; + readonly invalid: readonly string[]; +} + +// What the main process asks of the renderer's stores. +export type SyncSourceRequest = + | { readonly method: 'snapshot'; readonly kind: RendererSyncKind } + | { readonly method: 'validate'; readonly kind: RendererSyncKind; readonly id: string; readonly value: Json } + | { readonly method: 'apply'; readonly kind: RendererSyncKind; readonly change: SyncSourceChange }; + +export type SyncSourceReply = { readonly result: unknown } | { readonly error: string }; + export interface SyncApi { request(method: M, params: SyncMethods[M][0]): Promise; on(event: E, listener: (data: SyncEvents[E]) => void): () => void; + // Answers the main process's requests for the renderer's stores; called once they have loaded. + serve(handler: (request: SyncSourceRequest) => Promise): void; + // Tells the main process a synced store changed, so the edit is stamped and pushed. + changed(): void; } // Anyone holding the repository can guess passphrases offline for ever, so they must be long. diff --git a/src/components/chart/ChartWindow.tsx b/src/components/chart/ChartWindow.tsx index 0be5bf0..34ee40b 100644 --- a/src/components/chart/ChartWindow.tsx +++ b/src/components/chart/ChartWindow.tsx @@ -20,7 +20,7 @@ import { drawingsForSymbol, setSymbolDrawings, symbolDrawings } from '../../lib/ import { themeCssVariables } from '@meridian/charts/themes'; import { chartTheme, themeStore } from '../../lib/theme'; import { importedThemes } from '../../lib/custom-themes'; -import { focusChart, layoutDrawings, setLayoutDrawings, updateChart, workspace, type ChartConfig } from '../../lib/workspace'; +import { focusChart, isSyncedDrawings, layoutDrawings, setLayoutDrawings, updateChart, workspace, type ChartConfig } from '../../lib/workspace'; import { chartMenu, liteChartMenu } from '../../lib/chart-menu'; import { openContextMenu } from '../../lib/context-menu'; import { OneClickPanel } from './OneClickPanel'; @@ -256,11 +256,12 @@ export function ChartWindow({ chart, isActive, isLite = false }: ChartWindowProp }, [instance, chart.indicators]); // A change to the chart's own list came from this chart's menus or dialogs and can be undone there; - // a change to its tab's or symbol's list alone came from another chart or an import. + // a change to its tab's or symbol's list alone came from another chart or an import, and one from + // sync came from another computer. const appliedOwn = useRef(chart.drawings); useEffect(() => { if (!instance) return; - const isOwnChange = appliedOwn.current !== chart.drawings; + const isOwnChange = appliedOwn.current !== chart.drawings && !isSyncedDrawings(chart.drawings); appliedOwn.current = chart.drawings; const drawings = [...chart.drawings, ...tabShared, ...shared]; keepOwn(chart.drawings); diff --git a/src/components/dialogs/RiskTicket.tsx b/src/components/dialogs/RiskTicket.tsx index 8842561..f39f237 100644 --- a/src/components/dialogs/RiskTicket.tsx +++ b/src/components/dialogs/RiskTicket.tsx @@ -7,6 +7,7 @@ import { formatMoney, formatPrice } from '../../lib/format'; import { session } from '../../lib/session'; import { useStore } from '../../lib/store'; import { useRiskPreview } from '../../lib/risk-preview'; +import { riskTicketStorageKeys as storageKeys } from '../../lib/risk-preferences'; import { NumberField, Segmented, SelectField, Switch, TextField } from '../ui/Controls'; import { LeverageField } from './LeverageField'; @@ -26,7 +27,6 @@ type EntryKind = (typeof entryKinds)[number]['key']; // Forex and crypto accounts keep separate settings: a pip stop or a per-lot commission means // nothing on a crypto contract. Exchanges share theirs; the crypto key predates Binance. -const storageKeys: Record = { fx: 'openterm.riskTicket.v1', crypto: 'openterm.riskTicket.bybit.v1' }; interface Preferences { mode: RiskMode; diff --git a/src/components/sync/SyncDialog.tsx b/src/components/sync/SyncDialog.tsx index fff9925..0714ba8 100644 --- a/src/components/sync/SyncDialog.tsx +++ b/src/components/sync/SyncDialog.tsx @@ -3,7 +3,7 @@ import { passphraseProblem, syncKindInfo, syncStatuses, type SyncAttention, type import { requestSync, syncStore } from '../../lib/sync'; import { useStore } from '../../lib/store'; import { Dialog } from '../dialogs/Dialog'; -import { Section } from '../ui/Controls'; +import { Section, Switch } from '../ui/Controls'; function useAction() { const [error, setError] = useState(null); @@ -281,6 +281,7 @@ function RecoveryStep({ view }: SyncStepProps) { const resolutionLabels: Partial> = { held: [{ choice: 'useSynced', label: 'Restore from sync' }, { choice: 'allowDeletions', label: 'Delete everywhere' }], + join: [{ choice: 'merge', label: 'Merge' }, { choice: 'useSynced', label: 'Use synced data' }], stale: [{ choice: 'useSynced', label: 'Use the synced data' }], rollback: [{ choice: 'acceptHistory', label: 'I restored it on purpose' }], }; @@ -305,7 +306,7 @@ function AttentionBanner({ attention }: AttentionBannerProps) {
{attention.reason === 'reconnect' && } {choices.map(({ choice, label }) => )} - {attention.reason !== 'reconnect' && } + {attention.reason !== 'reconnect' && attention.reason !== 'join' && }
@@ -336,6 +337,40 @@ function DeviceList({ devices }: DeviceListProps) { ); } +// Secrets and the CID sync only on request, and only where the system keychain can hold them. +function SharingPanel({ view }: SyncStepProps) { + const { error, isBusy, run } = useAction(); + const share = (change: Partial) => void run('sync.setSharing', { isSecretsSynced: view.isSecretsSynced, isIdentitySynced: view.isIdentitySynced, ...change }); + return ( +
+ share({ isSecretsSynced })} + /> + share({ isIdentitySynced })} + /> + {view.isIdentityChoiceNeeded && ( +
+

Your other computers already share a different CID. Which one should all your computers use? A change applies from each computer's next login.

+
+ + +
+
+ )} + +
+ ); +} + function OnPanel({ view }: SyncStepProps) { const { error, isBusy, run } = useAction(); const [isConfirmingDisconnect, setIsConfirmingDisconnect] = useState(false); @@ -348,7 +383,7 @@ function OnPanel({ view }: SyncStepProps) { {view.attention && } {!view.isRecoveryConfirmed &&

The recovery code for this sync was never confirmed. Make a new one and save it.

} {view.status === 'offline' && view.error &&

{view.error} OpenTerm5 keeps your edits and tries again.

} -

Charts, drawings and other data start syncing in a coming update. For now this connects your computers and checks the encrypted {view.backend === 'file' ? 'sync file' : 'repository'}.

+

Layouts, charts, drawings, watchlists, templates, alerts, Pine scripts, saved accounts and settings stay the same on your computers. Panels, and which layout is in front, stay with each one.

@@ -378,7 +413,7 @@ function SyncStep({ view }: SyncStepProps) { case 'joining': return ; case 'pairing': return ; case 'savingCode': return ; - case 'on': return ; + case 'on': return <>; } } diff --git a/src/components/sync/SyncStatus.tsx b/src/components/sync/SyncStatus.tsx index 7e0dacc..2b9f898 100644 --- a/src/components/sync/SyncStatus.tsx +++ b/src/components/sync/SyncStatus.tsx @@ -3,7 +3,7 @@ import { syncStore } from '../../lib/sync'; import { useStore } from '../../lib/store'; import { openDialog } from '../../lib/workspace'; -const tones = { idle: 'is-good', syncing: '', offline: 'is-warn', waiting: 'is-warn', attention: 'is-bad' } as const; +const tones = { idle: 'is-good', syncing: '', offline: 'is-warn', waiting: 'is-warn', loading: 'is-warn', attention: 'is-bad' } as const; export function SyncStatus() { const view = useStore(syncStore, (state) => state.view); diff --git a/src/lib/pine/pine-scripts.ts b/src/lib/pine/pine-scripts.ts index 6a628f6..060380c 100644 --- a/src/lib/pine/pine-scripts.ts +++ b/src/lib/pine/pine-scripts.ts @@ -231,6 +231,20 @@ export async function importPineScript(source: string, origin: PineOrigin, isQui return script; } +export const pineScriptSource = (id: string): Promise => sourceOf(id); + +// A script from another computer: saved and registered as it is, keeping its id. Charts showing +// it get their new study ids through their own synced records. +export async function putSyncedPineScript(script: PineScript, source: string): Promise { + await saveScript(script, source); + sources.set(script.id, Promise.resolve(source)); + noticed.delete(script.id); + register(script); + pineScripts.set((state) => ({ scripts: [...state.scripts.filter((candidate) => candidate.id !== script.id), script].sort(byRank) })); +} + +export const restorePineScript = (value: unknown): PineScript | null => restore([value])[0] ?? null; + // Scripts registered but not yet in the list. const pending = new Map(); diff --git a/src/lib/risk-preferences.ts b/src/lib/risk-preferences.ts index c4f09f0..ac43335 100644 --- a/src/lib/risk-preferences.ts +++ b/src/lib/risk-preferences.ts @@ -1,10 +1,14 @@ import { defaultRiskPreferences, type RiskPreferences } from '../../shared/automation'; +import type { Market } from '../../shared/protocol'; import { automation } from './automation'; import { log } from './session'; import { Store } from './store'; export const riskPreferences = new Store(defaultRiskPreferences); +// The Risk Ticket's own settings per market, saved by the ticket itself. +export const riskTicketStorageKeys: Record = { fx: 'openterm.riskTicket.v1', crypto: 'openterm.riskTicket.bybit.v1' }; + let saveTimer: ReturnType | undefined; export function loadRiskPreferences(): void { diff --git a/src/lib/sync-model.test.ts b/src/lib/sync-model.test.ts new file mode 100644 index 0000000..9b2f77b --- /dev/null +++ b/src/lib/sync-model.test.ts @@ -0,0 +1,146 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { deriveKeys, newRootKey } from '../../electron/sync-crypto'; +import { SyncEngine, type SyncSource } from '../../electron/sync-engine'; +import { FakeRemote, MemoryStore, REPO_ID } from '../../electron/sync-engine-fixtures'; +import type { Json } from '../../shared/sync'; +import { makeTab, type ChartTab } from './chart-tabs'; +import type { Drawing } from './chart-model'; +import { + applyPreferencesChange, applyWorkspaceChange, checkWorkspacePreferences, checkWorkspaceRecord, orderedItems, orderedMap, ORDER, workspaceKinds, workspaceRecords, type WorkspaceKind, +} from './sync-model'; +import { makeChart, workspace, type ChartConfig, type WorkspaceState } from './workspace'; + +const line = (id: string, price = 1.1): Drawing => ({ + id, type: 'horizontal-line', paneId: 'price', anchors: [{ time: 1_700_000_000_000, price }], style: { lineColor: '#2962ff', lineWidth: 1, lineStyle: 'solid' }, locked: false, visible: true, zIndex: 0, createdAt: 1, +}); + +function chart(id: string, symbol: string, drawings: Drawing[] = []): ChartConfig { + return { ...makeChart(symbol), id, drawings }; +} + +function stateWith(tabs: ChartTab[], charts: ChartConfig[]): WorkspaceState { + return { ...workspace.get(), tabs, charts, activeTabId: tabs[0]?.id ?? '', profiles: [], templates: [] }; +} + +const empty = () => stateWith([], []); + +const allRecords = (state: WorkspaceState) => Object.fromEntries(workspaceKinds.map((kind) => [kind, workspaceRecords(state, kind)])); + +// Applies every kind's records in sync order, as the engine does for a computer that has none yet. +function applyAll(state: WorkspaceState, records: ReturnType): WorkspaceState { + let next = state; + for (const kind of workspaceKinds) next = { ...next, ...applyWorkspaceChange(next, kind, { upserts: records[kind], deletes: [] }) }; + return next; +} + +function sample(): WorkspaceState { + const charts = [chart('c1', 'EURUSD', [line('d1'), line('d2', 1.2)]), { ...chart('c2', 'XAUUSD'), theme: 'meridian-light', linkGroup: 'red' as const }, chart('c3', 'BTCUSDT')]; + const tabs = [ + { ...makeTab('t1', ['c1', 'c2'], '2h', ['crosshair', 'symbol'], 'fx'), drawings: { EURUSD: [line('s1')] } }, + makeTab('t2', ['c3'], 's', ['crosshair'], 'crypto'), + ]; + return stateWith(tabs, charts); +} + +test('records applied to an empty workspace read back the same', () => { + const state = sample(); + const records = allRecords(state); + const applied = applyAll(empty(), records); + assert.deepEqual(allRecords(applied), records); + assert.deepEqual(applied.charts.find((item) => item.id === 'c1')!.drawings.map((drawing) => drawing.id), ['d1', 'd2']); + assert.deepEqual(applied.tabs.find((tab) => tab.id === 't1')!.drawings, { EURUSD: [line('s1')] }); + for (const kind of workspaceKinds) for (const [id, value] of Object.entries(records[kind])) checkWorkspaceRecord(kind, id, value); +}); + +test('items an order does not list follow it, so two racing orders lose nothing', () => { + const map = { ...orderedMap([{ id: 'b' }, { id: 'a' }]), c: { id: 'c' }, [ORDER]: ['b', 'missing', 'b', 'a'] }; + assert.deepEqual(orderedItems(map).map(([id]) => id), ['b', 'a', 'c']); +}); + +test('a removed chart leaves its layout, and a reordered market keeps the other market\'s tabs in place', () => { + const state = sample(); + const removed = { ...state, ...applyWorkspaceChange(state, 'chart', { upserts: {}, deletes: ['c2'] }) }; + assert.deepEqual(removed.tabs.find((tab) => tab.id === 't1')!.chartIds, ['c1']); + const withThird = { ...state, tabs: [...state.tabs, makeTab('t3', ['c1'], 's', [], 'fx')] }; + const reordered = applyWorkspaceChange(withThird, 'layoutOrder', { upserts: { fx: ['t3', 't1'] }, deletes: [] }); + assert.deepEqual(reordered.tabs!.map((tab) => tab.id), ['t3', 't2', 't1']); +}); + +test('broken records are refused before anything applies them', () => { + const bad: [WorkspaceKind, string, Json][] = [ + ['chart', 'c1', { symbol: 'EURUSD', type: 'nope', timeframe: 'H1', indicators: [], isGridShown: true, isVolumeShown: false }], + ['chartDrawings', 'c1', { d1: { id: 'other', type: 'horizontal-line', anchors: [], style: {} } }], + ['layout', 't1', { layout: 'zz', market: 'fx', syncs: [], sizes: {} }], + ['layoutCharts', 't1', ['c1', 2]], + ['layoutOrder', 'stocks', []], + ['chartTemplate', ' padded ', { settings: {} }], + ]; + for (const [kind, id, value] of bad) assert.throws(() => checkWorkspaceRecord(kind, id, value), undefined, `${kind} ${id}`); +}); + +test('workspace preferences check what they know and keep what a newer build added', () => { + assert.throws(() => checkWorkspacePreferences({ drawingSync: 'everywhere' })); + checkWorkspacePreferences({ theme: 'meridian-light', futureSetting: 3 }); + const state = workspace.get(); + const patch = applyPreferencesChange(state, { theme: 'meridian-light', defaultChartTheme: null, activeProfile: 'Missing here', futureSetting: 3 }); + assert.equal(patch.theme, 'meridian-light'); + assert.ok('defaultChartTheme' in patch && patch.defaultChartTheme === undefined); + assert.ok(!('activeProfile' in patch)); +}); + +// Two computers whose engines read and write the workspace model, as the renderer does. +function computer(id: string, remote: FakeRemote, rootKey: Buffer, clock: { now: number }, state: WorkspaceState) { + const holder = { state }; + const sources: SyncSource[] = workspaceKinds.map((kind) => ({ + kind, + snapshot: async () => structuredClone(workspaceRecords(holder.state, kind)), + validate: (value, recordId) => checkWorkspaceRecord(kind, recordId, value), + apply: async (change) => { holder.state = { ...holder.state, ...applyWorkspaceChange(holder.state, kind, change) }; }, + })); + const engine = new SyncEngine({ + epoch: 1, repoId: REPO_ID, keys: deriveKeys(rootKey), remote, store: new MemoryStore(), sources, now: () => clock.now, pause: async () => undefined, + device: { id, name: id, platform: 'linux', publicKey: 'x', appVersion: '1.0.0' }, + }); + return { holder, engine }; +} + +test('layouts, charts and drawings edited on two computers end the same on both', async () => { + const remote = new FakeRemote(); + const rootKey = newRootKey(); + const clock = { now: 1_800_000_000_000 }; + const a = computer('a', remote, rootKey, clock, sample()); + const b = computer('b', remote, rootKey, clock, empty()); + await a.engine.sync(); + await b.engine.sync({ isLocalDiscarded: true }); + assert.deepEqual(allRecords(b.holder.state), allRecords(a.holder.state)); + + // A moves one drawing and clears a chart's theme; B adds a drawing to the same chart and a layout. + clock.now += 1000; + a.holder.state = { ...a.holder.state, charts: a.holder.state.charts.map((item) => item.id === 'c1' ? { ...item, drawings: [line('d1', 1.15), item.drawings[1]] } : item.id === 'c2' ? { ...item, theme: undefined } : item) }; + b.holder.state = { + ...b.holder.state, + charts: [...b.holder.state.charts.map((item) => item.id === 'c1' ? { ...item, drawings: [...item.drawings, line('d3', 1.3)] } : item), chart('c4', 'GBPUSD')], + tabs: [...b.holder.state.tabs, makeTab('t4', ['c4'], 's', ['crosshair'], 'fx')], + }; + await a.engine.sync(); + await b.engine.sync(); + await a.engine.sync(); + assert.deepEqual(allRecords(a.holder.state), allRecords(b.holder.state)); + const c1 = a.holder.state.charts.find((item) => item.id === 'c1')!; + assert.deepEqual(c1.drawings.map((drawing) => [drawing.id, drawing.anchors[0].price]), [['d1', 1.15], ['d2', 1.2], ['d3', 1.3]]); + assert.equal(b.holder.state.charts.find((item) => item.id === 'c2')!.theme, undefined); + assert.deepEqual(a.holder.state.tabs.filter((tab) => tab.market === 'fx').map((tab) => tab.id), ['t1', 't4']); + + // B closes the new layout with its chart; A loses both, and nothing is left behind. + clock.now += 1000; + b.holder.state = { ...b.holder.state, tabs: b.holder.state.tabs.filter((tab) => tab.id !== 't4'), charts: b.holder.state.charts.filter((item) => item.id !== 'c4') }; + await b.engine.sync(); + await a.engine.sync(); + assert.deepEqual(a.holder.state.tabs.map((tab) => tab.id), ['t1', 't2']); + assert.ok(!a.holder.state.charts.some((item) => item.id === 'c4')); + for (const { engine } of [a, b]) { + const settled = await engine.sync(); + assert.equal(settled.status === 'synced' && settled.pulled + settled.pushed, 0); + } +}); diff --git a/src/lib/sync-model.ts b/src/lib/sync-model.ts new file mode 100644 index 0000000..7b491b4 --- /dev/null +++ b/src/lib/sync-model.ts @@ -0,0 +1,296 @@ +import { restoreSettings } from '@meridian/charts/settings'; +import { markets, type Market } from '../../shared/protocol'; +import type { Json, SyncSourceChange } from '../../shared/sync'; +import { drawingSyncs, isLayoutKey, layoutOf, layoutSyncs, restoreSizes, type LayoutKey, type SplitSizes } from './chart-layouts'; +import { migrateDrawing, type Drawing } from './chart-model'; +import { makeTab, type ChartTab } from './chart-tabs'; +import { presetName, restoreChart, restoreProfiles, restoreTemplates, type ChartProfile, type ChartTemplate } from './presets'; +import { replayModes, replaySpeeds } from './replay-model'; +import type { ChartConfig, WorkspaceState } from './workspace'; + +// How the workspace and the other renderer stores turn into sync records and back. A record +// applied and read again must come out the same, or every computer would keep re-sending it. + +export type Records = Readonly>; + +// Singleton kinds (favourites, templates, looks, colours) hold one record under this id. +export const SINGLETON = 'all'; + +// Maps whose items have an order carry it as one more item: the ids in order. Items it does not +// list (added on another computer at the same time) follow in id order, so nothing is lost when +// two orders race and the newer one wins. +export const ORDER = '#order'; + +type JsonObject = { readonly [key: string]: Json }; + +export const isObject = (value: Json | undefined): value is JsonObject => value !== null && typeof value === 'object' && !Array.isArray(value); + +// What a store holds, as plain JSON: undefined fields go, as they would through storage. +export const toJson = (value: unknown): Json => JSON.parse(JSON.stringify(value)) as Json; + +function fail(message: string): never { + throw new Error(message); +} + +export function orderedMap(items: readonly T[], value: (item: T) => unknown = (item) => item): JsonObject { + return toJson({ ...Object.fromEntries(items.map((item) => [item.id, value(item)])), [ORDER]: items.map((item) => item.id) }) as JsonObject; +} + +export function orderedItems(map: JsonObject): [string, Json][] { + const order = Array.isArray(map[ORDER]) ? (map[ORDER] as readonly Json[]).filter((id): id is string => typeof id === 'string' && id !== ORDER && id in map) : []; + const listed = [...new Set(order)]; + const rest = Object.keys(map).filter((id) => id !== ORDER && !listed.includes(id)).sort(); + return [...listed, ...rest].map((id) => [id, map[id]]); +} + +function checkOrder(map: JsonObject): void { + const order = map[ORDER]; + if (order !== undefined && (!Array.isArray(order) || order.some((id) => typeof id !== 'string'))) fail('The order is not a list of ids.'); +} + +// Profile and template names are their record ids, so they must already be valid names. +function presetId(id: string): string { + const isValid = (() => { try { return presetName(id) === id; } catch { return false; } })(); + return isValid ? id : fail(`"${id}" is not a valid name.`); +} + +function restoreDrawingItem(id: string, value: Json): Drawing { + const drawing = migrateDrawing(value); + if (!drawing || drawing.id !== id) fail(`Drawing ${id} is not readable.`); + return drawing; +} + +// A drawings map in drawing order; throws when any drawing does not read. +export function restoreDrawingMap(value: Json): Drawing[] { + if (!isObject(value)) fail('Drawings must be a map.'); + checkOrder(value); + return orderedItems(value).map(([id, item]) => restoreDrawingItem(id, item)); +} + +export const drawingMap = (drawings: readonly Drawing[]) => orderedMap(drawings); + +// Charts: everything but the id and drawings, which sync apart. Optional fields are written as +// null, because a field missing from a record means "unchanged", not "removed". +export function chartRecord(chart: ChartConfig): Json { + const { id, drawings, settings, theme, linkGroup, ...fields } = chart; + return toJson({ ...fields, settings: settings ?? null, theme: theme ?? null, linkGroup: linkGroup ?? null }); +} + +export function restoreChartRecord(id: string, value: Json, drawings: Drawing[]): ChartConfig { + if (!isObject(value)) fail('A chart must be an object.'); + const fields = Object.fromEntries(Object.entries(value).filter(([, field]) => field !== null)); + return restoreChart({ ...fields, id, drawings }) ?? fail(`Chart ${id} is not readable.`); +} + +// Layouts: the tab's own fields. Its charts, their order and its shared drawings sync apart, and +// which chart is active stays with each computer. +export function layoutRecord(tab: ChartTab): Json { + return toJson({ layout: tab.layout, market: tab.market, syncs: tab.syncs, sizes: tab.sizes }); +} + +export function restoreLayoutRecord(value: Json): Pick { + if (!isObject(value) || !isLayoutKey(value.layout)) fail('A layout needs a known layout.'); + const market = markets.find((candidate) => candidate.key === value.market)?.key ?? fail('A layout needs a known market.'); + const syncs = Array.isArray(value.syncs) ? value.syncs : fail('Layout syncs must be a list.'); + if (syncs.some((sync) => !layoutSyncs.some((candidate) => candidate.key === sync))) fail('A layout sync is unknown.'); + if (!isObject(value.sizes)) fail('Layout sizes must be a map.'); + const sizes = Object.fromEntries(Object.entries(value.sizes).filter(([key]) => isLayoutKey(key)).map(([key, shares]) => [key, restoreSizes(shares, layoutOf(key as LayoutKey).tree)])) as Partial>; + return { layout: value.layout as LayoutKey, market, syncs: syncs as ChartTab['syncs'], sizes }; +} + +export function restoreIdList(value: Json): string[] { + if (!Array.isArray(value) || value.some((id) => typeof id !== 'string')) fail('Expected a list of ids.'); + return [...value] as string[]; +} + +// Drawings shared across a layout, by instrument: one map item per drawing, carrying its instrument. +export function layoutDrawingsRecord(tab: ChartTab): Json { + const items = Object.entries(tab.drawings).flatMap(([symbol, drawings]) => drawings.map((drawing) => ({ id: drawing.id, symbol, drawing }))); + return orderedMap(items, ({ symbol, drawing }) => ({ symbol, drawing })); +} + +export function restoreLayoutDrawings(value: Json): Record { + if (!isObject(value)) fail('Layout drawings must be a map.'); + checkOrder(value); + // A Map, so an instrument named like an Object.prototype member is just a name. + const bySymbol = new Map(); + for (const [id, item] of orderedItems(value)) { + if (!isObject(item) || typeof item.symbol !== 'string' || !item.symbol || item.symbol === '__proto__') fail(`Layout drawing ${id} has no instrument.`); + bySymbol.set(item.symbol, [...bySymbol.get(item.symbol) ?? [], restoreDrawingItem(id, item.drawing)]); + } + return Object.fromEntries(bySymbol); +} + +export function restoreProfile(name: string, value: Json): ChartProfile { + if (!isObject(value)) fail('A profile must be an object.'); + const [profile] = restoreProfiles([{ ...value, name }]); + return profile ?? fail(`Profile ${name} is not readable.`); +} + +export const profileRecord = ({ name, ...profile }: ChartProfile): Json => toJson(profile); + +export function restoreTemplate(name: string, value: Json): ChartTemplate { + if (!isObject(value)) fail('A template must be an object.'); + const [template] = restoreTemplates([{ name, settings: value.settings }]); + return template ?? fail(`Template ${name} is not readable.`); +} + +export const templateRecord = (template: ChartTemplate): Json => toJson({ settings: template.settings }); + +const workspacePreferenceChecks: Readonly boolean>> = { + theme: (value) => typeof value === 'string' && !!value, + drawingSync: (value) => drawingSyncs.some((candidate) => candidate.key === value), + isCrosshairSyncedByDefault: (value) => typeof value === 'boolean', + replayMode: (value) => replayModes.some((candidate) => candidate.key === value), + replaySpeed: (value) => replaySpeeds.some((candidate) => candidate.key === value), + defaultChartSettings: (value) => value === null || isObject(value), + defaultChartTheme: (value) => value === null || typeof value === 'string', + activeProfile: (value) => typeof value === 'string' && (() => { try { return presetId(value) === value; } catch { return false; } })(), +}; + +// The workspace settings that follow the user rather than the screen. Panels, docks, the page and +// the one-click consent stay with each computer. +export function workspacePreferences(state: WorkspaceState): Json { + return toJson({ + theme: state.theme, + drawingSync: state.drawingSync, + isCrosshairSyncedByDefault: state.isCrosshairSyncedByDefault, + replayMode: state.replayMode, + replaySpeed: state.replaySpeed, + defaultChartSettings: state.defaultChartSettings ?? null, + defaultChartTheme: state.defaultChartTheme ?? null, + activeProfile: state.activeProfile, + }); +} + +// Fields a newer build added are left alone; the known ones must hold valid values. +export function checkWorkspacePreferences(value: Json): void { + if (!isObject(value)) fail('Preferences must be an object.'); + for (const [key, field] of Object.entries(value)) if (workspacePreferenceChecks[key] && !workspacePreferenceChecks[key](field)) fail(`The ${key} preference is invalid.`); +} + +function applyWorkspacePreferences(state: WorkspaceState, value: JsonObject): Partial { + const patch: Partial = {}; + const known = (key: string) => key in value && !!workspacePreferenceChecks[key]; + if (known('theme')) patch.theme = value.theme as string; + if (known('drawingSync')) patch.drawingSync = value.drawingSync as WorkspaceState['drawingSync']; + if (known('isCrosshairSyncedByDefault')) patch.isCrosshairSyncedByDefault = value.isCrosshairSyncedByDefault as boolean; + if (known('replayMode')) patch.replayMode = value.replayMode as WorkspaceState['replayMode']; + if (known('replaySpeed')) patch.replaySpeed = value.replaySpeed as WorkspaceState['replaySpeed']; + if (known('defaultChartSettings')) patch.defaultChartSettings = value.defaultChartSettings === null ? undefined : restoreSettings(value.defaultChartSettings); + if (known('defaultChartTheme')) patch.defaultChartTheme = value.defaultChartTheme === null ? undefined : value.defaultChartTheme as string; + // The layouts of the other profile arrive as their own records; a profile this computer does not + // have yet keeps the current one. + if (known('activeProfile') && state.profiles.some((profile) => profile.name === value.activeProfile)) patch.activeProfile = value.activeProfile as string; + return patch; +} + +export const workspaceKinds = ['chart', 'chartDrawings', 'layout', 'layoutCharts', 'layoutDrawings', 'layoutOrder', 'chartProfile', 'chartTemplate'] as const; + +export type WorkspaceKind = (typeof workspaceKinds)[number]; + +export function workspaceRecords(state: WorkspaceState, kind: WorkspaceKind): Records { + const byId = (items: readonly T[], record: (item: T) => Json) => Object.fromEntries(items.map((item) => [item.id, record(item)])); + if (kind === 'chart') return byId(state.charts, chartRecord); + if (kind === 'chartDrawings') return byId(state.charts, (chart) => drawingMap(chart.drawings)); + if (kind === 'layout') return byId(state.tabs, layoutRecord); + if (kind === 'layoutCharts') return byId(state.tabs, (tab) => toJson(tab.chartIds)); + if (kind === 'layoutDrawings') return byId(state.tabs, layoutDrawingsRecord); + if (kind === 'layoutOrder') return Object.fromEntries(markets.map(({ key }) => [key, state.tabs.filter((tab) => tab.market === key).map((tab) => tab.id)])); + if (kind === 'chartProfile') return Object.fromEntries(state.profiles.map((profile) => [profile.name, profileRecord(profile)])); + return Object.fromEntries(state.templates.map((template) => [template.name, templateRecord(template)])); +} + +export function checkWorkspaceRecord(kind: WorkspaceKind, id: string, value: Json): void { + if (kind === 'chart') restoreChartRecord(id, value, []); + else if (kind === 'chartDrawings') restoreDrawingMap(value); + else if (kind === 'layout') restoreLayoutRecord(value); + else if (kind === 'layoutCharts' || kind === 'layoutOrder') restoreIdList(value); + else if (kind === 'layoutDrawings') restoreLayoutDrawings(value); + else if (kind === 'chartProfile') restoreProfile(presetId(id), value); + else restoreTemplate(presetId(id), value); + if (kind === 'layoutOrder' && !markets.some((market) => market.key === id)) fail(`Unknown market ${id}.`); +} + +// Tabs of `market` in the order given, in the places that market's tabs held; tabs the order does +// not list keep their order after the listed ones. +function reorderTabs(tabs: readonly ChartTab[], market: Market, order: readonly string[]): ChartTab[] { + const rank = (tab: ChartTab) => { const index = order.indexOf(tab.id); return index < 0 ? Infinity : index; }; + const sorted = tabs.filter((tab) => tab.market === market).sort((left, right) => rank(left) - rank(right)); + return tabs.map((tab) => tab.market === market ? sorted.shift()! : tab); +} + +// The state after a change from sync, for one kind. `markSynced` tags drawing lists that came from +// another computer, so the chart does not offer them for undo. +export function applyWorkspaceChange(state: WorkspaceState, kind: WorkspaceKind, change: SyncSourceChange, markSynced: (drawings: Drawing[]) => Drawing[] = (drawings) => drawings): Partial { + const deleted = new Set(change.deletes); + const upserts = Object.entries(change.upserts); + const patchTabs = (patch: (tab: ChartTab) => ChartTab) => state.tabs.map(patch); + if (kind === 'chart') { + const charts = state.charts.filter((chart) => !deleted.has(chart.id)); + for (const [id, value] of upserts) { + const index = charts.findIndex((chart) => chart.id === id); + const chart = restoreChartRecord(id, value, index < 0 ? [] : charts[index].drawings); + if (index < 0) charts.push(chart); + else charts[index] = chart; + } + const tabs = deleted.size ? patchTabs((tab) => tab.chartIds.some((id) => deleted.has(id)) ? { ...tab, chartIds: tab.chartIds.filter((id) => !deleted.has(id)) } : tab) : state.tabs; + return { charts, tabs }; + } + if (kind === 'chartDrawings') { + const values = new Map(upserts); + return { + charts: state.charts.map((chart) => { + if (values.has(chart.id)) return { ...chart, drawings: markSynced(restoreDrawingMap(values.get(chart.id)!)) }; + return deleted.has(chart.id) && chart.drawings.length ? { ...chart, drawings: markSynced([]) } : chart; + }), + }; + } + if (kind === 'layout') { + const tabs = state.tabs.filter((tab) => !deleted.has(tab.id)); + for (const [id, value] of upserts) { + const fields = restoreLayoutRecord(value); + const index = tabs.findIndex((tab) => tab.id === id); + if (index < 0) tabs.push({ ...makeTab(id, [], fields.layout, fields.syncs, fields.market), sizes: fields.sizes }); + else tabs[index] = { ...tabs[index], ...fields }; + } + return { tabs }; + } + if (kind === 'layoutCharts') { + const values = new Map(upserts); + return { tabs: patchTabs((tab) => values.has(tab.id) ? { ...tab, chartIds: restoreIdList(values.get(tab.id)!) } : tab) }; + } + if (kind === 'layoutDrawings') { + const values = new Map(upserts); + return { tabs: patchTabs((tab) => values.has(tab.id) ? { ...tab, drawings: restoreLayoutDrawings(values.get(tab.id)!) } : deleted.has(tab.id) ? { ...tab, drawings: {} } : tab) }; + } + if (kind === 'layoutOrder') { + let tabs = [...state.tabs]; + for (const [market, value] of upserts) tabs = reorderTabs(tabs, market as Market, restoreIdList(value)); + return { tabs }; + } + if (kind === 'chartProfile') { + // The profile in use is the live workspace, so it is never removed from under it. + const profiles = state.profiles.filter((profile) => !deleted.has(profile.name) || profile.name === state.activeProfile); + for (const [name, value] of upserts) { + const profile = restoreProfile(name, value); + const index = profiles.findIndex((candidate) => candidate.name === name); + if (index < 0) profiles.push(profile); + else profiles[index] = profile; + } + return { profiles }; + } + const templates = state.templates.filter((template) => !deleted.has(template.name)); + for (const [name, value] of upserts) { + const template = restoreTemplate(name, value); + const index = templates.findIndex((candidate) => candidate.name === name); + if (index < 0) templates.push(template); + else templates[index] = template; + } + return { templates }; +} + +export function applyPreferencesChange(state: WorkspaceState, value: Json): Partial { + checkWorkspacePreferences(value); + return applyWorkspacePreferences(state, value as JsonObject); +} diff --git a/src/lib/sync-sources.ts b/src/lib/sync-sources.ts new file mode 100644 index 0000000..0f7298f --- /dev/null +++ b/src/lib/sync-sources.ts @@ -0,0 +1,290 @@ +import { customColorList, drawingToolInfo, onCustomColorsChange, saveDrawingFavorites, setCustomColors, type DrawingTemplate } from '@meridian/charts'; +import { parseRiskPreferences } from '../../shared/automation'; +import { markets, type Market } from '../../shared/protocol'; +import type { Json, RendererSyncKind, SyncSourceChange } from '../../shared/sync'; +import { automation } from './automation'; +import { importedThemes, removeImportedTheme, saveImportedTheme } from './custom-themes'; +import { drawingTemplates, restoreDrawingLooks, restoreDrawingTemplates } from './drawing-templates'; +import { drawingTools } from './drawing-tools'; +import { indicatorFavorites } from './indicator-favorites'; +import { isChartTimeframe, type ChartTimeframe } from './mt5-timeframes'; +import { pineSourceProblem } from './pine/pine-script'; +import { pineScripts, pineScriptSource, pineScriptsReady, putSyncedPineScript, removePineScripts, restorePineScript } from './pine/pine-scripts'; +import { priceAlerts, updateAlerts } from './price-alerts'; +import { restoreAlerts } from './price-alerts-model'; +import { riskPreferences, riskTicketStorageKeys } from './risk-preferences'; +import type { Store } from './store'; +import { symbolDrawings } from './symbol-drawings'; +import { + applyPreferencesChange, applyWorkspaceChange, checkWorkspacePreferences, checkWorkspaceRecord, drawingMap, isObject, orderedItems, orderedMap, ORDER, restoreDrawingMap, + SINGLETON, toJson, workspacePreferences, workspaceRecords, type Records, type WorkspaceKind, +} from './sync-model'; +import { timeframeFavorites } from './timeframe-favorites'; +import { assistantPreferences, restoreAssistantPreferences } from './trade-assistant'; +import { restoreThemePatches } from './tradingview-model'; +import { restoreWatchlists, type WatchlistsState } from './watchlist-model'; +import { savedWatchlists, setSavedWatchlists, watchlists } from './watchlists'; +import { markSyncedDrawings, workspace } from './workspace'; + +// The renderer's side of sync: each kind's records read from and written to its store. The main +// process holds the keys and the engine, and asks for these over IPC. + +export interface RendererSource { + readonly snapshot: () => Records | Promise; + // Throws when the record must not be applied under that id. + readonly validate: (value: Json, id: string) => void; + readonly apply: (change: SyncSourceChange) => void | Promise; +} + +function fail(message: string): never { + throw new Error(message); +} + +const workspaceSource = (kind: WorkspaceKind): RendererSource => ({ + snapshot: () => workspaceRecords(workspace.get(), kind), + validate: (value, id) => checkWorkspaceRecord(kind, id, value), + apply: (change) => workspace.set((state) => applyWorkspaceChange(state, kind, change, markSyncedDrawings)), +}); + +// The singleton's new value, [] when it was deleted, or null when the change leaves it alone. +function singletonValue(change: SyncSourceChange): Json | null { + if (SINGLETON in change.upserts) return change.upserts[SINGLETON]; + return change.deletes.includes(SINGLETON) ? [] : null; +} + +function checkSingleton(id: string): void { + if (id !== SINGLETON) fail(`Unknown record ${id}.`); +} + +// Favourites keep the order they were added in here; ones from elsewhere go at the end. +function setSource(read: () => readonly string[], write: (items: string[]) => void, isMember: (item: string) => boolean): RendererSource { + return { + snapshot: () => ({ [SINGLETON]: [...read()] }), + validate: (value, id) => { + checkSingleton(id); + if (!Array.isArray(value) || value.some((item) => typeof item !== 'string' || !isMember(item))) fail('Favourites must be known names.'); + }, + apply: (change) => { + const value = singletonValue(change); + if (value === null) return; + const next = value as string[]; + const current = read(); + write([...current.filter((item) => next.includes(item)), ...next.filter((item) => !current.includes(item))]); + }, + }; +} + +// Drawing templates are one map item per tool and name. +function templatesFromMap(value: Json): Record { + if (!isObject(value)) fail('Drawing templates must be a map.'); + const byType: Record = {}; + for (const [key, item] of Object.entries(value)) { + const split = key.indexOf('/'); + const [type, name] = [key.slice(0, split), key.slice(split + 1)]; + const restored = split > 0 ? restoreDrawingTemplates({ [type]: [item] })[type]?.[0] : undefined; + if (!restored || restored.name !== name) fail(`Drawing template ${key} is not readable.`); + (byType[type] ??= []).push(restored); + } + for (const list of Object.values(byType)) list.sort((left, right) => left.name.localeCompare(right.name)); + return byType; +} + +function looksFromMap(value: Json): Record { + if (!isObject(value)) fail('Drawing looks must be a map.'); + const looks = restoreDrawingLooks(value); + if (Object.keys(looks).length !== Object.keys(value).length) fail('A drawing look is not readable.'); + return looks; +} + +function watchlistsFromMap(market: string, value: Json, activeId: string): WatchlistsState { + if (!markets.some((candidate) => candidate.key === market)) fail(`Unknown market ${market}.`); + if (!isObject(value) || (value[ORDER] !== undefined && !Array.isArray(value[ORDER]))) fail('Watchlists must be a map.'); + const lists = orderedItems(value).map(([id, list]) => ({ ...(isObject(list) ? list : {}), id })); + const restored = restoreWatchlists({ activeId, lists }); + const isWhole = restored?.lists.length === lists.length && restored.lists.every((list, index) => list.entries.length === (lists[index] as { entries?: unknown[] }).entries?.length); + return isWhole ? restored! : fail('A watchlist is not readable.'); +} + +const conflictPattern = /~conflict~[^~]*~(\d+)$/; + +function checkPineScript(value: Json, id: string): void { + if (!isObject(value) || typeof value.source !== 'string' || pineSourceProblem(value.source) || !restorePineScript({ ...value, id })) fail(`Pine script ${id} is not readable.`); +} + +const riskTicketIds = markets.map(({ key }) => `riskTicket:${key}`); + +function readRiskTicket(market: Market): Json | undefined { + try { + const value = JSON.parse(localStorage.getItem(riskTicketStorageKeys[market]) ?? 'null') as Json; + return isObject(value) ? toJson(value) : undefined; + } catch { + return undefined; + } +} + +export const syncSources: Readonly> = { + theme: { + snapshot: () => Object.fromEntries(restoreThemePatches(importedThemes.get().themes).map((theme) => [theme.id, toJson(theme)])), + validate: (value, id) => { + if (restoreThemePatches([value])[0]?.id !== id) fail(`Theme ${id} is not readable.`); + }, + apply: (change) => { + change.deletes.forEach(removeImportedTheme); + for (const value of Object.values(change.upserts)) saveImportedTheme(restoreThemePatches([value])[0]); + }, + }, + pineScript: { + // A script whose source is missing is still listed, so it reads as invalid rather than deleted. + snapshot: async () => { + await pineScriptsReady; + const entries = await Promise.all(pineScripts.get().scripts.map(async ({ id, ...script }) => { + const source = await pineScriptSource(id).catch(() => null); + return [id, toJson(source === null ? script : { ...script, source })] as const; + })); + return Object.fromEntries(entries); + }, + validate: checkPineScript, + // Both computers edited a script: the copy kept from this one is named so it can be told apart. + apply: async (change) => { + if (change.deletes.length) removePineScripts(change.deletes); + for (const [id, value] of Object.entries(change.upserts)) { + checkPineScript(value, id); + const { source, ...fields } = value as { source: string }; + const script = restorePineScript({ ...fields, id })!; + const conflict = conflictPattern.exec(id); + const name = conflict && !script.name.includes('(conflict') ? `${script.name} (conflict, ${new Date(Number(conflict[1])).toISOString().slice(0, 10)})` : script.name; + await putSyncedPineScript({ ...script, name }, source); + } + }, + }, + chart: workspaceSource('chart'), + chartDrawings: workspaceSource('chartDrawings'), + layout: workspaceSource('layout'), + layoutCharts: workspaceSource('layoutCharts'), + layoutDrawings: workspaceSource('layoutDrawings'), + layoutOrder: workspaceSource('layoutOrder'), + symbolDrawings: { + snapshot: () => Object.fromEntries(Object.entries(symbolDrawings.get().bySymbol).map(([key, drawings]) => [key, drawingMap(drawings)])), + validate: (value, id) => { + if (!id) fail('Symbol drawings need an instrument.'); + restoreDrawingMap(value); + }, + apply: (change) => symbolDrawings.set((state) => { + const bySymbol = Object.fromEntries(Object.entries(state.bySymbol).filter(([key]) => !change.deletes.includes(key))); + for (const [key, value] of Object.entries(change.upserts)) { + const drawings = restoreDrawingMap(value); + if (drawings.length) bySymbol[key] = drawings; + else delete bySymbol[key]; + } + return { bySymbol }; + }), + }, + chartProfile: workspaceSource('chartProfile'), + chartTemplate: workspaceSource('chartTemplate'), + drawingTemplates: { + snapshot: () => ({ [SINGLETON]: toJson(Object.fromEntries(Object.entries(drawingTemplates.get().byType).flatMap(([type, list]) => list.map((template) => [`${type}/${template.name}`, template])))) }), + validate: (value, id) => { + checkSingleton(id); + templatesFromMap(value); + }, + apply: (change) => { + const value = singletonValue(change); + if (value !== null) drawingTemplates.set({ byType: Array.isArray(value) ? {} : templatesFromMap(value) }); + }, + }, + drawingLooks: { + snapshot: () => ({ [SINGLETON]: toJson(drawingTemplates.get().lastByType) }), + validate: (value, id) => { + checkSingleton(id); + looksFromMap(value); + }, + apply: (change) => { + const value = singletonValue(change); + if (value !== null) drawingTemplates.set({ lastByType: Array.isArray(value) ? {} : looksFromMap(value) }); + }, + }, + watchlists: { + snapshot: () => Object.fromEntries(markets.flatMap(({ key }) => { + const state = savedWatchlists(key); + return state ? [[key, orderedMap(state.lists, ({ name, entries }) => ({ name, entries }))]] : []; + })), + validate: (value, id) => void watchlistsFromMap(id, value, ''), + // Which list is open stays with each computer. A market's lists are never removed as a whole. + apply: (change) => { + for (const [market, value] of Object.entries(change.upserts)) { + setSavedWatchlists(market as Market, watchlistsFromMap(market, value, savedWatchlists(market as Market)?.activeId ?? '')); + } + }, + }, + indicatorFavorites: setSource(() => indicatorFavorites.get().types, (types) => indicatorFavorites.set({ types }), (type) => type.length > 0), + timeframeFavorites: setSource(() => timeframeFavorites.get().labels, (labels) => timeframeFavorites.set({ labels: labels as ChartTimeframe[] }), isChartTimeframe), + drawingFavorites: setSource(() => drawingTools.get().favorites, (favorites) => { + saveDrawingFavorites(favorites); + drawingTools.set({ favorites }); + }, (type) => drawingToolInfo.has(type)), + customColors: { + snapshot: () => ({ [SINGLETON]: [...customColorList()] }), + validate: (value, id) => { + checkSingleton(id); + if (!Array.isArray(value) || value.length > 10 || value.some((color) => typeof color !== 'string' || !/^#[0-9a-f]{6}$/i.test(color))) fail('Custom colours must be up to ten #rrggbb colours.'); + }, + apply: (change) => { + const value = singletonValue(change); + if (value !== null) setCustomColors(value as string[]); + }, + }, + priceAlert: { + snapshot: () => Object.fromEntries(priceAlerts.get().alerts.map(({ id, ...alert }) => [id, toJson(alert)])), + validate: (value, id) => { + if (!isObject(value) || restoreAlerts([{ ...value, id }]).length !== 1) fail(`Alert ${id} is not readable.`); + }, + apply: (change) => { + const alerts = priceAlerts.get().alerts.filter((alert) => !change.deletes.includes(alert.id)); + for (const [id, value] of Object.entries(change.upserts)) { + const [alert] = restoreAlerts([{ ...(value as object), id }]); + const index = alerts.findIndex((candidate) => candidate.id === id); + if (index < 0) alerts.push(alert); + else alerts[index] = alert; + } + updateAlerts(alerts); + }, + }, + // Preferences are never removed as a whole; a deletion leaves this computer's as they are. + preferences: { + snapshot: async () => { + const records: Record = { + workspace: workspacePreferences(workspace.get()), + tradeAssistant: toJson(assistantPreferences.get()), + risk: toJson(await automation.request('preferences.risk', {})), + }; + for (const { key } of markets) { + const ticket = readRiskTicket(key); + if (ticket !== undefined) records[`riskTicket:${key}`] = ticket; + } + return records; + }, + validate: (value, id) => { + if (!isObject(value)) fail('Preferences must be an object.'); + if (id === 'workspace') checkWorkspacePreferences(value); + else if (id !== 'tradeAssistant' && id !== 'risk' && !riskTicketIds.includes(id)) fail(`Unknown preferences ${id}.`); + }, + apply: async (change) => { + for (const [id, value] of Object.entries(change.upserts)) { + if (id === 'workspace') workspace.set((state) => applyPreferencesChange(state, value)); + else if (id === 'tradeAssistant') assistantPreferences.set(restoreAssistantPreferences({ ...assistantPreferences.get(), ...(value as object) })); + else if (id === 'risk') riskPreferences.set(await automation.request('preferences.setRisk', parseRiskPreferences({ ...riskPreferences.get(), ...(value as object) }))); + else { + const market = markets.find(({ key }) => id === `riskTicket:${key}`)?.key; + if (market) localStorage.setItem(riskTicketStorageKeys[market], JSON.stringify({ ...(readRiskTicket(market) as object | undefined), ...(value as object) })); + } + } + }, + }, +}; + +// Every store sync reads, for telling the main process an edit happened. The Risk Ticket saves its +// settings itself, so they are found at the next sync instead. +export const syncedStores: readonly Pick, 'subscribe'>[] = [ + workspace, symbolDrawings, drawingTemplates, importedThemes, watchlists, indicatorFavorites, timeframeFavorites, drawingTools, priceAlerts, pineScripts, assistantPreferences, riskPreferences, + { subscribe: onCustomColorsChange }, +]; diff --git a/src/lib/sync.ts b/src/lib/sync.ts index 5e0f2be..41ee8bb 100644 --- a/src/lib/sync.ts +++ b/src/lib/sync.ts @@ -1,5 +1,9 @@ -import type { SyncApi, SyncMethod, SyncMethods, SyncView } from '../../shared/sync'; +import type { SyncApi, SyncMethod, SyncMethods, SyncSnapshot, SyncSourceRequest, SyncView } from '../../shared/sync'; import { Store } from './store'; +import { syncedStores, syncSources } from './sync-sources'; +import { toJson } from './sync-model'; + +const CHANGE_NOTICE_MS = 250; declare global { interface Window { @@ -11,9 +15,38 @@ export const sync = window.sync; export const syncStore = new Store<{ view: SyncView | null }>({ view: null }); +// Store changes made by applying synced records are not news to the main process. +let isApplying = false; + +async function serve(request: SyncSourceRequest): Promise { + const source = syncSources[request.kind]; + if (request.method === 'validate') return source.validate(request.value, request.id) ?? null; + if (request.method === 'snapshot') { + const values = toJson(await source.snapshot()) as SyncSnapshot['values']; + const invalid = Object.entries(values).flatMap(([id, value]) => { + try { source.validate(value, id); return []; } catch { return [id]; } + }); + return { values, invalid } satisfies SyncSnapshot; + } + isApplying = true; + try { await source.apply(request.change); } + finally { isApplying = false; } + return null; +} + export function startSync(): void { sync.on('view', (view) => syncStore.set({ view })); void sync.request('sync.view', {}).then((view) => syncStore.set({ view })).catch(() => undefined); + let noticeTimer: ReturnType | undefined; + const notice = () => { + if (isApplying || noticeTimer || syncStore.get().view?.phase !== 'on') return; + noticeTimer = setTimeout(() => { + noticeTimer = undefined; + sync.changed(); + }, CHANGE_NOTICE_MS); + }; + syncedStores.forEach((store) => store.subscribe(notice)); + sync.serve(serve); } // Errors are returned for the dialog to show next to what was tried. diff --git a/src/lib/watchlists.ts b/src/lib/watchlists.ts index 2041e25..f255f89 100644 --- a/src/lib/watchlists.ts +++ b/src/lib/watchlists.ts @@ -33,6 +33,23 @@ let market: Market = 'fx'; export const watchlists = new Store(load(market)); +// A market's lists as saved, for sync: the market in use from the store, another from storage, and +// null for a market whose lists were never saved here (its default list has a new id each load). +export function savedWatchlists(of: Market): WatchlistsState | null { + if (of === market) return watchlists.get(); + try { + const saved = localStorage.getItem(storageKeys[of]); + return saved ? restoreWatchlists(JSON.parse(saved)) : null; + } catch { + return null; + } +} + +export function setSavedWatchlists(of: Market, state: WatchlistsState): void { + if (of === market) watchlists.set(state); + else localStorage.setItem(storageKeys[of], JSON.stringify(state)); +} + // The platform of the account in use; null while logged out or while the account changes. export function activePlatform(): Platform | null { const { accounts, profileId } = session.get(); diff --git a/src/lib/workspace.ts b/src/lib/workspace.ts index ed55ce8..440105f 100644 --- a/src/lib/workspace.ts +++ b/src/lib/workspace.ts @@ -280,6 +280,16 @@ export function activeChart(state: WorkspaceState): ChartConfig | undefined { const noDrawings: readonly Drawing[] = []; +// Drawing lists that arrived from another computer: the chart shows them without offering an undo. +const syncedDrawingLists = new WeakSet(); + +export function markSyncedDrawings(drawings: Drawing[]): Drawing[] { + syncedDrawingLists.add(drawings); + return drawings; +} + +export const isSyncedDrawings = (drawings: readonly Drawing[]) => syncedDrawingLists.has(drawings); + // Drawings shared across the chart's tab for the chart's symbol. export function layoutDrawings(state: WorkspaceState, chart: ChartConfig): readonly Drawing[] { return tabOfChart(state, chart.id)?.drawings[instrumentKey(chart.symbol)] ?? noDrawings;