From d12a7f977b69d7f5fa6153c8598ba123d7c9f5cc Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 14 Aug 2026 10:39:23 -0700 Subject: [PATCH 1/6] feat(voice): ordered provider failover chain Voice has exactly one path to speech. If ElevenLabs is unreachable, rate limited, or the account lapses, every spoken notification is simply lost, and an install with no ElevenLabs account cannot speak at all. [voice].providers, when present, is an ordered chain. The first provider that returns audio speaks; any failure (non-2xx, timeout, connection refused) falls through to the next; an exhausted chain logs and stays silent rather than throwing into the notify path. Absent or empty, nothing changes: the legacy ElevenLabs branch is untouched and still gated on the API key, which is what every existing install runs. - VoiceServer/providers.ts: new. Config normalization, both provider clients, and the chain runner. Zero external deps and no import of ../lib, so it is testable before 'bun install' has ever run. - voice.ts: pronunciation preprocessing split out of generateSpeech so a chain applies it once rather than once per attempt; playAudio takes the container format, because the Linux players pick a demuxer off the file extension and a wav body in a .mp3 file plays as noise. - The openai-compatible client sends stream: false. Kokoro-FastAPI defaults it to true, and a streamed reply commits to 200 before generation finishes, so a model that dies mid-sentence would arrive as truncated audio that the chain reads as success. Verified against schemas.py in v0.7.2. - ElevenLabs keeps no request timeout on the legacy path, where it never had one. Inside a chain every hop is bounded, or the fallback never gets a turn. Tests at PULSE/test/ per the doctrine's subsystem-package clause, driving both the chain library and the real /notify contract against stub servers on ephemeral ports. 44 pass, no network. --- LifeOS/install/LIFEOS/PULSE/PULSE.toml | 45 ++ .../LIFEOS/PULSE/VoiceServer/providers.ts | 353 ++++++++++++++ .../install/LIFEOS/PULSE/VoiceServer/voice.ts | 140 ++++-- LifeOS/install/LIFEOS/PULSE/package.json | 1 + .../PULSE/test/VoiceServer/providers.test.ts | 448 ++++++++++++++++++ .../PULSE/test/VoiceServer/voice.test.ts | 238 ++++++++++ 6 files changed, 1181 insertions(+), 44 deletions(-) create mode 100644 LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts create mode 100644 LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts create mode 100644 LifeOS/install/LIFEOS/PULSE/test/VoiceServer/voice.test.ts diff --git a/LifeOS/install/LIFEOS/PULSE/PULSE.toml b/LifeOS/install/LIFEOS/PULSE/PULSE.toml index e2df9ee2e6..9bf47ada8b 100644 --- a/LifeOS/install/LIFEOS/PULSE/PULSE.toml +++ b/LifeOS/install/LIFEOS/PULSE/PULSE.toml @@ -64,6 +64,51 @@ da = false [voice] enabled = true +# ── Voice provider chain (optional) ── +# +# Left unset, voice behaves exactly as it always has: ElevenLabs, keyed by +# ELEVENLABS_API_KEY, voices resolved from settings.json daidentity.voices. +# +# Define `[[voice.providers]]` blocks to get an ordered failover chain instead. +# The first provider that returns audio speaks; any failure (non-2xx, timeout, +# connection refused) falls through to the next. If every provider fails, the +# desktop notification still goes out and /notify answers 502 — the daemon +# never dies because a TTS box is down. +# +# A chain brings its own credentials, so an install with no ElevenLabs key at +# all is valid as long as the chain does not contain an `elevenlabs` entry. +# +# Local-first, hosted fallback. The local half here is Kokoro-FastAPI +# (`ghcr.io/remsky/kokoro-fastapi-cpu:v0.7.2`, default port 8880) — pin the tag, +# `latest` moves weekly: +# +# [[voice.providers]] +# type = "openai-compatible" +# base_url = "http://127.0.0.1:8880" # server root, or its /v1 base +# voice = "am_michael" # provider's own voice ID +# model = "kokoro" +# # response_format = "mp3" # mp3 (default), wav, opus, flac, aac, pcm +# # timeout_ms = 15000 +# # api_key = "${LOCAL_TTS_KEY}" # sent as a bearer token when set +# +# [[voice.providers]] +# type = "elevenlabs" +# # voice = "..." # overrides the resolved voice ID here only +# # model = "eleven_turbo_v2_5" +# # api_key = "${ELEVENLABS_API_KEY}" # defaults to the module-level key +# +# `openai-compatible` targets POST {base_url}/v1/audio/speech with +# {model, voice, input, response_format} and expects raw audio bytes back — +# the OpenAI speech contract, which Kokoro-FastAPI and LocalAI both implement. +# `stream: false` is sent on every request: Kokoro-FastAPI streams by default, +# and a streamed reply returns 200 before generation finishes, so a failure +# mid-sentence would arrive as truncated audio instead of falling through. +# +# Kokoro voice IDs are `{lang}{gender}_{name}`. On the male English side its +# own quality grades put `am_fenrir`, `am_michael` and `am_puck` at the top and +# `am_adam` at the bottom; blends are expressed in the same `voice` string, +# weighted — `voice = "am_fenrir(2)+am_michael(1)"`. + [imessage] enabled = false diff --git a/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts b/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts new file mode 100644 index 0000000000..bb7085cae4 --- /dev/null +++ b/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts @@ -0,0 +1,353 @@ +/** + * LifeOS Pulse — Voice Provider Chain + * + * An ordered list of TTS providers. The first one that produces audio wins; + * every failure (non-2xx, timeout, connection refused) falls through to the + * next. When the list is exhausted the caller stays silent — synthesis never + * throws out of this module, because /notify must not die when a TTS box is + * down. + * + * Two provider types ship today: + * elevenlabs — the hosted default, unchanged from the pre-chain path + * openai-compatible — any server speaking POST {base_url}/v1/audio/speech + * (Kokoro-FastAPI, LocalAI, OpenAI itself) + * + * ZERO external dependencies, by design: this file must be importable before + * `bun install` has ever run, so the tests can exercise it on a fresh clone + * (Testing doctrine rule 1). That is also why the logger is injected rather + * than imported from ../lib — lib.ts pulls in smol-toml. + */ + +// ── Config Types ── + +export interface ElevenLabsProviderConfig { + type: "elevenlabs" + /** Overrides the module-level key. Usually omitted. */ + api_key?: string + /** Overrides the resolved voice ID for this link in the chain only. */ + voice?: string + /** ElevenLabs model_id. Defaults to the pre-chain value. */ + model?: string + timeout_ms?: number +} + +export interface OpenAiCompatibleProviderConfig { + type: "openai-compatible" + /** Server root (`http://127.0.0.1:8880`) or its /v1 base — both accepted. */ + base_url: string + voice?: string + model?: string + /** mp3 (default), wav, opus, flac, aac, pcm — whatever the server supports. */ + response_format?: string + /** Sent as `Authorization: Bearer …` when present. Local servers ignore it. */ + api_key?: string + timeout_ms?: number +} + +export type VoiceProviderConfig = ElevenLabsProviderConfig | OpenAiCompatibleProviderConfig + +export interface SynthesisResult { + audio: ArrayBuffer + /** Container format of `audio`, used to pick the temp-file extension. */ + format: string + /** Human-readable label of the provider that answered, for logging. */ + provider: string +} + +export interface ElevenLabsVoiceSettings { + stability: number + similarity_boost: number + style?: number + speed?: number + use_speaker_boost?: boolean +} + +/** Everything the chain needs that is resolved per-notification, not per-config. */ +export interface SynthesisContext { + /** Pronunciation/homograph preprocessing is already applied by the caller. */ + text: string + elevenLabsVoiceId: string + elevenLabsSettings: ElevenLabsVoiceSettings + elevenLabsApiKey?: string +} + +export type Logger = (level: "info" | "warn" | "error", message: string, meta?: Record) => void + +// ── Constants ── + +export const DEFAULT_ELEVENLABS_MODEL = "eleven_turbo_v2_5" + +/** + * Kokoro-FastAPI ignores `model` but the OpenAI schema requires it, so a + * non-empty default keeps strict servers happy without forcing config. + */ +export const DEFAULT_OPENAI_MODEL = "kokoro" + +/** Kokoro-FastAPI's own default voice. */ +export const DEFAULT_OPENAI_VOICE = "af_heart" + +/** mp3 keeps the existing playback path (afplay/ffplay/mpg123) working as-is. */ +export const DEFAULT_RESPONSE_FORMAT = "mp3" + +export const DEFAULT_TIMEOUT_MS = 15_000 + +const FORMAT_EXTENSIONS: Record = { + mp3: "mp3", + wav: "wav", + opus: "opus", + flac: "flac", + aac: "aac", + pcm: "pcm", +} + +/** Temp-file extension for a container format. Unknown formats play as mp3. */ +export function extensionForFormat(format: string): string { + return FORMAT_EXTENSIONS[format.toLowerCase()] ?? "mp3" +} + +// ── Config Normalization ── + +function asString(value: unknown): string | undefined { + return typeof value === "string" && value.length > 0 ? value : undefined +} + +function asNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined +} + +/** + * Turn the raw `[voice].providers` TOML value into a validated chain. + * + * Returns an empty array for anything that is not a non-empty array — that is + * the signal for "no chain configured", which keeps the legacy ElevenLabs path + * in charge. Individual malformed entries are dropped with a warning rather + * than failing the whole chain: one typo should not mute the system. + */ +export function normalizeProviders(raw: unknown, log?: Logger): VoiceProviderConfig[] { + if (!Array.isArray(raw) || raw.length === 0) return [] + + const chain: VoiceProviderConfig[] = [] + + raw.forEach((entry, index) => { + if (!entry || typeof entry !== "object") { + log?.("warn", `Voice: provider[${index}] is not a table — skipped`) + return + } + + const record = entry as Record + const type = asString(record.type) + + if (type === "elevenlabs") { + chain.push({ + type: "elevenlabs", + api_key: asString(record.api_key), + voice: asString(record.voice), + model: asString(record.model), + timeout_ms: asNumber(record.timeout_ms), + }) + return + } + + if (type === "openai-compatible") { + const baseUrl = asString(record.base_url) + if (!baseUrl) { + log?.("warn", `Voice: provider[${index}] type "openai-compatible" has no base_url — skipped`) + return + } + chain.push({ + type: "openai-compatible", + base_url: baseUrl, + voice: asString(record.voice), + model: asString(record.model), + response_format: asString(record.response_format), + api_key: asString(record.api_key), + timeout_ms: asNumber(record.timeout_ms), + }) + return + } + + log?.("warn", `Voice: provider[${index}] has unknown type ${JSON.stringify(record.type)} — skipped`) + }) + + return chain +} + +/** Label used in logs and health output. Never includes credentials. */ +export function providerLabel(provider: VoiceProviderConfig): string { + return provider.type === "openai-compatible" ? `openai-compatible(${provider.base_url})` : "elevenlabs" +} + +// ── Endpoint Resolution ── + +/** + * Build the speech URL from a configured base. + * + * Both spellings are accepted because both are natural: the server root + * (`http://127.0.0.1:8880`, what Kokoro-FastAPI's README prints) and the + * OpenAI-style base that already carries /v1 (what OPENAI_BASE_URL looks + * like). Appending blindly would produce /v1/v1/audio/speech for the second. + */ +export function speechEndpoint(baseUrl: string): string { + const trimmed = baseUrl.replace(/\/+$/, "") + return trimmed.endsWith("/v1") ? `${trimmed}/audio/speech` : `${trimmed}/v1/audio/speech` +} + +// ── Provider Clients ── + +/** + * Raw ElevenLabs call. Pronunciation preprocessing happens upstream so the + * chain applies it exactly once regardless of how many providers it tries. + */ +export async function elevenLabsSynthesize(opts: { + text: string + voiceId: string + settings: ElevenLabsVoiceSettings + apiKey?: string + modelId?: string + timeoutMs?: number +}): Promise { + if (!opts.apiKey) throw new Error("ElevenLabs API key not configured") + + const response = await fetch(`https://api.elevenlabs.io/v1/text-to-speech/${opts.voiceId}`, { + method: "POST", + headers: { + Accept: "audio/mpeg", + "Content-Type": "application/json", + "xi-api-key": opts.apiKey, + }, + body: JSON.stringify({ + text: opts.text, + model_id: opts.modelId ?? DEFAULT_ELEVENLABS_MODEL, + voice_settings: opts.settings, + }), + // No timeoutMs means no signal at all, not a default one. The pre-chain + // path never bounded this call, and silently capping it would change + // behaviour for every install that has no chain configured. + signal: opts.timeoutMs ? AbortSignal.timeout(opts.timeoutMs) : undefined, + }) + + if (!response.ok) { + const errorText = await response.text() + throw new Error(`ElevenLabs API error: ${response.status} - ${errorText}`) + } + + return await response.arrayBuffer() +} + +/** + * OpenAI-compatible speech call — the Kokoro-FastAPI contract. + * + * POST {base_url}/v1/audio/speech with {model, voice, input, response_format} + * returning raw audio bytes. `input` is the text field; that naming is the + * OpenAI schema, not a typo. + * + * `stream: false` is sent deliberately. Kokoro-FastAPI defaults `stream` to + * TRUE — unlike OpenAI, which has no such field — and a streamed reply commits + * to HTTP 200 before generation finishes. A model that dies halfway then looks + * like success carrying a truncated body, and the chain would play the stub + * instead of failing over. Asking for the complete body puts failures back in + * the status code, which is the only thing the chain can act on. Verified + * against api/src/structures/schemas.py (Kokoro-FastAPI v0.7.2). + */ +export async function openAiCompatibleSynthesize(opts: { + text: string + baseUrl: string + voice?: string + model?: string + responseFormat?: string + /** Playback rate, carried over from the resolved voice settings when set. */ + speed?: number + apiKey?: string + timeoutMs?: number +}): Promise { + const headers: Record = { "Content-Type": "application/json" } + if (opts.apiKey) headers.Authorization = `Bearer ${opts.apiKey}` + + const response = await fetch(speechEndpoint(opts.baseUrl), { + method: "POST", + headers, + body: JSON.stringify({ + model: opts.model ?? DEFAULT_OPENAI_MODEL, + voice: opts.voice ?? DEFAULT_OPENAI_VOICE, + input: opts.text, + response_format: opts.responseFormat ?? DEFAULT_RESPONSE_FORMAT, + stream: false, + ...(Number.isFinite(opts.speed) ? { speed: opts.speed } : {}), + }), + signal: opts.timeoutMs ? AbortSignal.timeout(opts.timeoutMs) : undefined, + }) + + if (!response.ok) { + const errorText = await response.text().catch(() => "") + throw new Error(`TTS server error: ${response.status} - ${errorText}`) + } + + const audio = await response.arrayBuffer() + if (audio.byteLength === 0) throw new Error("TTS server returned an empty body") + + return audio +} + +// ── Chain Runner ── + +/** + * Walk the chain until something speaks. + * + * Returns null when every provider failed — the caller logs and stays silent. + * This function does not throw: a chain that cannot synthesize is a degraded + * notification, not a failed request. + */ +export async function synthesizeViaChain( + providers: VoiceProviderConfig[], + ctx: SynthesisContext, + log?: Logger, +): Promise { + for (let i = 0; i < providers.length; i++) { + const provider = providers[i] + const label = providerLabel(provider) + + try { + if (provider.type === "elevenlabs") { + const audio = await elevenLabsSynthesize({ + text: ctx.text, + voiceId: provider.voice ?? ctx.elevenLabsVoiceId, + settings: ctx.elevenLabsSettings, + apiKey: provider.api_key ?? ctx.elevenLabsApiKey, + modelId: provider.model, + // Inside a chain every hop is bounded — an unbounded first provider + // would mean the fallback never gets its turn. + timeoutMs: provider.timeout_ms ?? DEFAULT_TIMEOUT_MS, + }) + log?.("info", `Voice: provider[${i}] ${label} synthesized ${audio.byteLength} bytes`) + return { audio, format: "mp3", provider: label } + } + + const audio = await openAiCompatibleSynthesize({ + text: ctx.text, + baseUrl: provider.base_url, + voice: provider.voice, + model: provider.model, + responseFormat: provider.response_format, + // Speed is the one voice setting both provider families understand, so + // the notification sounds the same whichever link answers. + speed: ctx.elevenLabsSettings.speed, + apiKey: provider.api_key, + timeoutMs: provider.timeout_ms ?? DEFAULT_TIMEOUT_MS, + }) + const format = provider.response_format ?? DEFAULT_RESPONSE_FORMAT + log?.("info", `Voice: provider[${i}] ${label} synthesized ${audio.byteLength} bytes`) + return { audio, format, provider: label } + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error) + const isLast = i === providers.length - 1 + log?.("warn", `Voice: provider[${i}] ${label} failed — ${isLast ? "chain exhausted" : "falling through"}`, { + error: message, + }) + } + } + + log?.("error", "Voice: every provider in the chain failed — staying silent", { + providers: providers.map(providerLabel), + }) + return null +} diff --git a/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts b/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts index 0f0d2d3df3..29e314e4a1 100644 --- a/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts +++ b/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts @@ -19,6 +19,17 @@ import { existsSync, readFileSync, rmSync } from "fs" import { log } from "../lib" import { disambiguateHomographs } from "../lib/homographs" import { homedir } from "node:os"; +import { + DEFAULT_ELEVENLABS_MODEL, + elevenLabsSynthesize, + extensionForFormat, + normalizeProviders, + providerLabel, + synthesizeViaChain, + type ElevenLabsVoiceSettings, + type SynthesisResult, + type VoiceProviderConfig, +} from "./providers" // ── Public Config Interface ── @@ -27,18 +38,17 @@ export interface VoiceConfig { elevenlabs_api_key?: string default_voice_id?: string pronunciations_path?: string + /** + * Ordered provider chain from PULSE.toml `[voice].providers`. Raw because it + * arrives straight off the TOML parser; normalizeProviders validates it. + * Absent or empty means "no chain" — the legacy ElevenLabs path stays in + * charge, byte-for-byte. + */ + providers?: unknown } // ── Internal Types ── -interface ElevenLabsVoiceSettings { - stability: number - similarity_boost: number - style?: number - speed?: number - use_speaker_boost?: boolean -} - interface VoiceEntry { voiceId: string voiceName?: string @@ -74,6 +84,7 @@ let pronunciationRules: CompiledRule[] = [] let voiceConfig: LoadedVoiceConfig = { defaultVoiceId: "", voices: {}, voicesByVoiceId: {}, desktopNotifications: true } let defaultVoiceId = "" let initialized = false +let providerChain: VoiceProviderConfig[] = [] // ── Constants ── @@ -319,41 +330,52 @@ function escapeForAppleScript(input: string): string { // ── TTS Generation ── -async function generateSpeech( - text: string, - voiceId: string, - voiceSettings: ElevenLabsVoiceSettings, -): Promise { - const apiKey = moduleConfig.elevenlabs_api_key - if (!apiKey) throw new Error("ElevenLabs API key not configured") - +/** + * Pronunciation + homograph preprocessing. Split out of generateSpeech so the + * provider chain applies it exactly once, no matter how many providers it has + * to try — re-running the rules per attempt could double-substitute. + */ +function preprocessForSpeech(text: string): string { const pronouncedText = applyPronunciations(disambiguateHomographs(text)) if (pronouncedText !== text) { log("info", `Voice pronunciation: "${text}" -> "${pronouncedText}"`) } + return pronouncedText +} - const url = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}` - - const response = await fetch(url, { - method: "POST", - headers: { - Accept: "audio/mpeg", - "Content-Type": "application/json", - "xi-api-key": apiKey, - }, - body: JSON.stringify({ - text: pronouncedText, - model_id: "eleven_turbo_v2_5", - voice_settings: voiceSettings, - }), +async function generateSpeech( + text: string, + voiceId: string, + voiceSettings: ElevenLabsVoiceSettings, +): Promise { + return await elevenLabsSynthesize({ + text: preprocessForSpeech(text), + voiceId, + settings: voiceSettings, + apiKey: moduleConfig.elevenlabs_api_key, + modelId: DEFAULT_ELEVENLABS_MODEL, }) +} - if (!response.ok) { - const errorText = await response.text() - throw new Error(`ElevenLabs API error: ${response.status} - ${errorText}`) - } - - return await response.arrayBuffer() +/** + * Chain equivalent of generateSpeech. Returns null when every provider failed, + * which the caller treats as "stay silent" rather than as an exception. + */ +async function generateSpeechViaChain( + text: string, + voiceId: string, + voiceSettings: ElevenLabsVoiceSettings, +): Promise { + return await synthesizeViaChain( + providerChain, + { + text: preprocessForSpeech(text), + elevenLabsVoiceId: voiceId, + elevenLabsSettings: voiceSettings, + elevenLabsApiKey: moduleConfig.elevenlabs_api_key, + }, + log, + ) } // ── Audio Playback ── @@ -435,7 +457,11 @@ function enqueuePlayback(task: () => Promise): Promise { return next } -async function playAudio(audioBuffer: ArrayBuffer, volume: number = FALLBACK_VOLUME): Promise { +async function playAudio( + audioBuffer: ArrayBuffer, + volume: number = FALLBACK_VOLUME, + format = "mp3", +): Promise { const player = resolveAudioPlayer() if (!player) { const tried = process.platform === "darwin" ? "afplay" : "ffplay/mpg123/paplay/aplay" @@ -443,7 +469,9 @@ async function playAudio(audioBuffer: ArrayBuffer, volume: number = FALLBACK_VOL return } - const tempFile = `/tmp/voice-${Date.now()}.mp3` + // The extension matters: the Linux players sniff it to pick a demuxer, so a + // wav body in a .mp3 file plays as noise. + const tempFile = `/tmp/voice-${Date.now()}.${extensionForFormat(format)}` await Bun.write(tempFile, audioBuffer) return new Promise((resolve, reject) => { @@ -528,7 +556,12 @@ async function sendNotification( let voicePlayed = false let voiceError: string | undefined - if (voiceEnabled && moduleConfig.elevenlabs_api_key) { + // A configured chain owns synthesis and brings its own credentials, so it is + // not gated on an ElevenLabs key — an install running only a local Kokoro box + // has none. With no chain, the gate is exactly what it always was. + const chainConfigured = providerChain.length > 0 + + if (voiceEnabled && (chainConfigured || moduleConfig.elevenlabs_api_key)) { try { const voice = voiceId || defaultVoiceId @@ -586,9 +619,22 @@ async function sendNotification( volume: resolvedVolume, }) - const audioBuffer = await generateSpeech(safeMessage, voice, resolvedSettings) - await enqueuePlayback(() => playAudio(audioBuffer, resolvedVolume)) - voicePlayed = true + if (chainConfigured) { + const result = await generateSpeechViaChain(safeMessage, voice, resolvedSettings) + if (result) { + await enqueuePlayback(() => playAudio(result.audio, resolvedVolume, result.format)) + voicePlayed = true + } else { + // Every provider failed. synthesizeViaChain already logged each one, + // so this only records the outcome for the /notify response — the + // notification itself still goes out. + voiceError = "all voice providers failed" + } + } else { + const audioBuffer = await generateSpeech(safeMessage, voice, resolvedSettings) + await enqueuePlayback(() => playAudio(audioBuffer, resolvedVolume)) + voicePlayed = true + } } catch (error: unknown) { const msg = error instanceof Error ? error.message : String(error) log("error", "Voice: failed to generate/play speech", { error: msg }) @@ -632,7 +678,9 @@ export function startVoice(config: VoiceConfig): void { return } - if (!config.elevenlabs_api_key) { + providerChain = normalizeProviders(config.providers, log) + + if (!config.elevenlabs_api_key && providerChain.length === 0) { log("warn", "Voice module: ELEVENLABS_API_KEY not set in config or env") } @@ -654,6 +702,7 @@ export function startVoice(config: VoiceConfig): void { pronunciationRules: pronunciationRules.length, configuredVoices: Object.keys(voiceConfig.voices), apiKeyConfigured: !!config.elevenlabs_api_key, + ...(providerChain.length > 0 ? { providerChain: providerChain.map(providerLabel) } : {}), }) } @@ -664,7 +713,10 @@ export function voiceHealth(): Record { return { initialized, enabled: moduleConfig.enabled, - voice_system: "ElevenLabs", + // Unchanged when no chain is configured, so existing health consumers see + // exactly the string they saw before. + voice_system: providerChain.length > 0 ? "chain" : "ElevenLabs", + ...(providerChain.length > 0 ? { providers: providerChain.map(providerLabel) } : {}), default_voice_id: defaultVoiceId, api_key_configured: !!moduleConfig.elevenlabs_api_key, pronunciation_rules: pronunciationRules.length, diff --git a/LifeOS/install/LIFEOS/PULSE/package.json b/LifeOS/install/LIFEOS/PULSE/package.json index 154cd6c157..6f469e382a 100644 --- a/LifeOS/install/LIFEOS/PULSE/package.json +++ b/LifeOS/install/LIFEOS/PULSE/package.json @@ -5,6 +5,7 @@ "type": "module", "scripts": { "start": "bun run pulse.ts", + "test": "bun test", "install-service": "bash manage.sh install", "status": "bash manage.sh status" }, diff --git a/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts new file mode 100644 index 0000000000..9ec598f2ff --- /dev/null +++ b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts @@ -0,0 +1,448 @@ +/** + * Voice provider chain — regression suite. + * + * Everything runs against a stub Bun.serve on an ephemeral port. No network, + * no ElevenLabs credentials, no fixed ports. + */ + +import { afterEach, describe, expect, test } from "bun:test" +import { + DEFAULT_OPENAI_MODEL, + DEFAULT_OPENAI_VOICE, + DEFAULT_RESPONSE_FORMAT, + elevenLabsSynthesize, + extensionForFormat, + normalizeProviders, + openAiCompatibleSynthesize, + providerLabel, + speechEndpoint, + synthesizeViaChain, + type SynthesisContext, + type VoiceProviderConfig, +} from "../../VoiceServer/providers" + +// ── Stub Server Harness ── + +interface Stub { + url: string + /** Bodies of every /v1/audio/speech request this stub received, in order. */ + requests: Array> + headers: Array +} + +const running: Array<{ stop: (force?: boolean) => void }> = [] + +function startStub(handler: (body: Record, req: Request) => Response | Promise): Stub { + const requests: Array> = [] + const headers: Array = [] + + const server = Bun.serve({ + port: 0, + async fetch(req) { + const url = new URL(req.url) + if (url.pathname !== "/v1/audio/speech" || req.method !== "POST") { + return new Response("not found", { status: 404 }) + } + const body = (await req.json()) as Record + requests.push(body) + headers.push(req.headers) + return await handler(body, req) + }, + }) + + running.push(server) + return { url: `http://127.0.0.1:${server.port}`, requests, headers } +} + +/** A port with nothing listening on it — for the connection-refused cases. */ +function deadUrl(): string { + const server = Bun.serve({ port: 0, fetch: () => new Response("ok") }) + const port = server.port + server.stop(true) + return `http://127.0.0.1:${port}` +} + +function audioResponse(bytes = new Uint8Array([0x49, 0x44, 0x33, 0x04])): Response { + return new Response(bytes, { status: 200, headers: { "Content-Type": "audio/mpeg" } }) +} + +function ctx(overrides: Partial = {}): SynthesisContext { + return { + text: "the estate is clean", + elevenLabsVoiceId: "21m00Tcm4TlvDq8ikWAM", + elevenLabsSettings: { stability: 0.5, similarity_boost: 0.75 }, + ...overrides, + } +} + +afterEach(() => { + while (running.length) running.pop()?.stop(true) +}) + +// ── Config Parsing ── + +describe("normalizeProviders", () => { + test("absent config yields an empty chain — the legacy-path signal", () => { + expect(normalizeProviders(undefined)).toEqual([]) + expect(normalizeProviders(null)).toEqual([]) + }) + + test("an empty array is also no chain", () => { + expect(normalizeProviders([])).toEqual([]) + }) + + test("a non-array value is rejected rather than coerced", () => { + expect(normalizeProviders("elevenlabs")).toEqual([]) + expect(normalizeProviders({ type: "elevenlabs" })).toEqual([]) + }) + + test("parses an ordered mixed chain and preserves order", () => { + const chain = normalizeProviders([ + { type: "openai-compatible", base_url: "http://127.0.0.1:8880", voice: "am_michael", model: "kokoro" }, + { type: "elevenlabs" }, + ]) + + expect(chain).toHaveLength(2) + expect(chain[0]).toMatchObject({ + type: "openai-compatible", + base_url: "http://127.0.0.1:8880", + voice: "am_michael", + model: "kokoro", + }) + expect(chain[1]).toMatchObject({ type: "elevenlabs" }) + }) + + test("carries the optional openai-compatible fields through", () => { + const chain = normalizeProviders([ + { + type: "openai-compatible", + base_url: "http://localhost:8880", + response_format: "wav", + api_key: "sk-local", + timeout_ms: 3000, + }, + ]) + + expect(chain[0]).toMatchObject({ response_format: "wav", api_key: "sk-local", timeout_ms: 3000 }) + }) + + test("drops an openai-compatible entry with no base_url, keeping the rest", () => { + const warnings: string[] = [] + const chain = normalizeProviders( + [{ type: "openai-compatible", voice: "am_michael" }, { type: "elevenlabs" }], + (level, message) => { + if (level === "warn") warnings.push(message) + }, + ) + + expect(chain).toHaveLength(1) + expect(chain[0].type).toBe("elevenlabs") + expect(warnings.join(" ")).toContain("base_url") + }) + + test("drops unknown types and non-tables without failing the chain", () => { + const chain = normalizeProviders([ + { type: "piper" }, + "elevenlabs", + null, + 42, + { type: "elevenlabs" }, + ]) + + expect(chain).toHaveLength(1) + expect(chain[0].type).toBe("elevenlabs") + }) + + test("ignores non-positive and non-numeric timeouts", () => { + const chain = normalizeProviders([ + { type: "openai-compatible", base_url: "http://x", timeout_ms: 0 }, + { type: "openai-compatible", base_url: "http://y", timeout_ms: "soon" }, + ]) + + expect(chain[0].timeout_ms).toBeUndefined() + expect(chain[1].timeout_ms).toBeUndefined() + }) + + test("labels never leak credentials", () => { + const chain = normalizeProviders([ + { type: "openai-compatible", base_url: "http://127.0.0.1:8880", api_key: "sk-secret" }, + { type: "elevenlabs", api_key: "xi-secret" }, + ]) + + const labels = chain.map(providerLabel).join(" ") + expect(labels).not.toContain("sk-secret") + expect(labels).not.toContain("xi-secret") + }) +}) + +// ── Endpoint Resolution ── + +describe("speechEndpoint", () => { + test("appends /v1/audio/speech to a server root", () => { + expect(speechEndpoint("http://127.0.0.1:8880")).toBe("http://127.0.0.1:8880/v1/audio/speech") + }) + + test("tolerates trailing slashes", () => { + expect(speechEndpoint("http://127.0.0.1:8880/")).toBe("http://127.0.0.1:8880/v1/audio/speech") + }) + + test("does not double the /v1 when the base already carries it", () => { + expect(speechEndpoint("http://127.0.0.1:8880/v1")).toBe("http://127.0.0.1:8880/v1/audio/speech") + expect(speechEndpoint("http://127.0.0.1:8880/v1/")).toBe("http://127.0.0.1:8880/v1/audio/speech") + }) +}) + +describe("extensionForFormat", () => { + test("maps the known container formats", () => { + expect(extensionForFormat("mp3")).toBe("mp3") + expect(extensionForFormat("wav")).toBe("wav") + expect(extensionForFormat("opus")).toBe("opus") + expect(extensionForFormat("WAV")).toBe("wav") + }) + + test("falls back to mp3 for anything unrecognised", () => { + expect(extensionForFormat("ogg-but-not-really")).toBe("mp3") + }) +}) + +// ── OpenAI-Compatible Client Contract ── + +describe("openAiCompatibleSynthesize", () => { + test("sends {model, voice, input, response_format} and returns the bytes", async () => { + const stub = startStub(() => audioResponse(new Uint8Array([1, 2, 3, 4, 5]))) + + const audio = await openAiCompatibleSynthesize({ + text: "good morning", + baseUrl: stub.url, + voice: "am_michael", + model: "kokoro", + }) + + expect(new Uint8Array(audio)).toEqual(new Uint8Array([1, 2, 3, 4, 5])) + expect(stub.requests).toHaveLength(1) + expect(stub.requests[0]).toEqual({ + model: "kokoro", + voice: "am_michael", + input: "good morning", + response_format: "mp3", + stream: false, + }) + }) + + test("applies the documented defaults when voice/model/format are unset", async () => { + const stub = startStub(() => audioResponse()) + + await openAiCompatibleSynthesize({ text: "hello", baseUrl: stub.url }) + + expect(stub.requests[0]).toEqual({ + model: DEFAULT_OPENAI_MODEL, + voice: DEFAULT_OPENAI_VOICE, + input: "hello", + response_format: DEFAULT_RESPONSE_FORMAT, + stream: false, + }) + }) + + test("always asks for a complete body — a streamed 200 can hide a dead generation", async () => { + const stub = startStub(() => audioResponse()) + + await openAiCompatibleSynthesize({ text: "hello", baseUrl: stub.url }) + + // Kokoro-FastAPI defaults stream to true; omitting the field would opt us + // into chunked replies that report success before generation finishes. + expect(stub.requests[0].stream).toBe(false) + }) + + test("sends speed only when one was resolved", async () => { + const withSpeed = startStub(() => audioResponse()) + await openAiCompatibleSynthesize({ text: "hi", baseUrl: withSpeed.url, speed: 1.15 }) + expect(withSpeed.requests[0].speed).toBe(1.15) + + const without = startStub(() => audioResponse()) + await openAiCompatibleSynthesize({ text: "hi", baseUrl: without.url }) + expect(without.requests[0]).not.toHaveProperty("speed") + }) + + test("sends a bearer token only when an api_key is configured", async () => { + const withKey = startStub(() => audioResponse()) + await openAiCompatibleSynthesize({ text: "hi", baseUrl: withKey.url, apiKey: "sk-local" }) + expect(withKey.headers[0].get("authorization")).toBe("Bearer sk-local") + + const without = startStub(() => audioResponse()) + await openAiCompatibleSynthesize({ text: "hi", baseUrl: without.url }) + expect(without.headers[0].get("authorization")).toBeNull() + }) + + test("throws on a non-2xx response", async () => { + const stub = startStub(() => new Response("model not loaded", { status: 503 })) + + await expect(openAiCompatibleSynthesize({ text: "hi", baseUrl: stub.url })).rejects.toThrow(/503/) + }) + + test("treats an empty 200 body as a failure", async () => { + const stub = startStub(() => new Response(new Uint8Array([]), { status: 200 })) + + await expect(openAiCompatibleSynthesize({ text: "hi", baseUrl: stub.url })).rejects.toThrow(/empty body/) + }) +}) + +// ── ElevenLabs Client ── + +describe("elevenLabsSynthesize", () => { + test("refuses to call out without an API key", async () => { + await expect( + elevenLabsSynthesize({ + text: "hi", + voiceId: "abc", + settings: { stability: 0.5, similarity_boost: 0.75 }, + }), + ).rejects.toThrow(/API key not configured/) + }) +}) + +// ── Chain Behaviour ── + +describe("synthesizeViaChain", () => { + test("the first healthy provider speaks and later ones are never called", async () => { + const first = startStub(() => audioResponse(new Uint8Array([9, 9, 9]))) + const second = startStub(() => audioResponse(new Uint8Array([7, 7, 7]))) + + const chain = normalizeProviders([ + { type: "openai-compatible", base_url: first.url }, + { type: "openai-compatible", base_url: second.url }, + ]) + + const result = await synthesizeViaChain(chain, ctx()) + + expect(result).not.toBeNull() + expect(new Uint8Array(result!.audio)).toEqual(new Uint8Array([9, 9, 9])) + expect(first.requests).toHaveLength(1) + expect(second.requests).toHaveLength(0) + }) + + test("falls through to the next provider on a non-2xx", async () => { + const broken = startStub(() => new Response("boom", { status: 500 })) + const healthy = startStub(() => audioResponse(new Uint8Array([4, 2]))) + + const chain = normalizeProviders([ + { type: "openai-compatible", base_url: broken.url }, + { type: "openai-compatible", base_url: healthy.url }, + ]) + + const result = await synthesizeViaChain(chain, ctx()) + + expect(new Uint8Array(result!.audio)).toEqual(new Uint8Array([4, 2])) + expect(broken.requests).toHaveLength(1) + expect(healthy.requests).toHaveLength(1) + }) + + test("falls through on connection refused", async () => { + const healthy = startStub(() => audioResponse()) + + const chain = normalizeProviders([ + { type: "openai-compatible", base_url: deadUrl() }, + { type: "openai-compatible", base_url: healthy.url }, + ]) + + const result = await synthesizeViaChain(chain, ctx()) + + expect(result).not.toBeNull() + expect(healthy.requests).toHaveLength(1) + }) + + test("falls through on timeout", async () => { + const hanging = startStub(async () => { + await new Promise((resolve) => setTimeout(resolve, 5_000)) + return audioResponse() + }) + const healthy = startStub(() => audioResponse(new Uint8Array([1, 1]))) + + const chain = normalizeProviders([ + { type: "openai-compatible", base_url: hanging.url, timeout_ms: 50 }, + { type: "openai-compatible", base_url: healthy.url }, + ]) + + const result = await synthesizeViaChain(chain, ctx()) + + expect(new Uint8Array(result!.audio)).toEqual(new Uint8Array([1, 1])) + }) + + test("falls through across provider types — a keyless elevenlabs link is skipped", async () => { + const healthy = startStub(() => audioResponse(new Uint8Array([5]))) + + const chain = normalizeProviders([ + { type: "elevenlabs" }, + { type: "openai-compatible", base_url: healthy.url }, + ]) + + const result = await synthesizeViaChain(chain, ctx({ elevenLabsApiKey: undefined })) + + expect(result).not.toBeNull() + expect(result!.provider).toContain("openai-compatible") + expect(healthy.requests).toHaveLength(1) + }) + + test("returns null when every provider fails, and does not throw", async () => { + const a = startStub(() => new Response("no", { status: 500 })) + const b = startStub(() => new Response("also no", { status: 502 })) + + const chain = normalizeProviders([ + { type: "openai-compatible", base_url: a.url }, + { type: "openai-compatible", base_url: b.url }, + { type: "openai-compatible", base_url: deadUrl() }, + ]) + + const result = await synthesizeViaChain(chain, ctx()) + + expect(result).toBeNull() + expect(a.requests).toHaveLength(1) + expect(b.requests).toHaveLength(1) + }) + + test("logs an error naming every provider when the chain is exhausted", async () => { + const errors: string[] = [] + const chain = normalizeProviders([{ type: "openai-compatible", base_url: deadUrl() }]) + + await synthesizeViaChain(chain, ctx(), (level, message) => { + if (level === "error") errors.push(message) + }) + + expect(errors.join(" ")).toContain("staying silent") + }) + + test("an empty chain synthesizes nothing", async () => { + expect(await synthesizeViaChain([], ctx())).toBeNull() + }) + + test("reports the response format so playback picks the right extension", async () => { + const stub = startStub(() => audioResponse()) + + const chain: VoiceProviderConfig[] = normalizeProviders([ + { type: "openai-compatible", base_url: stub.url, response_format: "wav" }, + ]) + + const result = await synthesizeViaChain(chain, ctx()) + + expect(result!.format).toBe("wav") + expect(extensionForFormat(result!.format)).toBe("wav") + expect(stub.requests[0].response_format).toBe("wav") + }) + + test("passes the already-preprocessed text straight through as `input`", async () => { + const stub = startStub(() => audioResponse()) + const chain = normalizeProviders([{ type: "openai-compatible", base_url: stub.url }]) + + await synthesizeViaChain(chain, ctx({ text: "DAN-yuhl MIL-ur" })) + + expect(stub.requests[0].input).toBe("DAN-yuhl MIL-ur") + }) + + test("carries the resolved speed onto the openai-compatible provider", async () => { + const stub = startStub(() => audioResponse()) + const chain = normalizeProviders([{ type: "openai-compatible", base_url: stub.url }]) + + await synthesizeViaChain(chain, ctx({ elevenLabsSettings: { stability: 0.5, similarity_boost: 0.75, speed: 0.9 } })) + + expect(stub.requests[0].speed).toBe(0.9) + }) +}) diff --git a/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/voice.test.ts b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/voice.test.ts new file mode 100644 index 0000000000..03e1d552d1 --- /dev/null +++ b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/voice.test.ts @@ -0,0 +1,238 @@ +/** + * Voice module — provider chain wiring and backwards compatibility. + * + * Drives the real /notify contract through handleVoiceRequest so the branch in + * sendNotification is exercised, not just the chain library underneath it. + * + * Two environment neutralisations are required for determinism: + * + * HOME → a temp dir, so the suite never reads the operator's real + * settings.json or PRONUNCIATIONS.json. + * PATH → empty, so Bun.which finds no audio player. playAudio then logs and + * returns instead of spawning ffplay on stub bytes. resolveAudioPlayer + * caches its answer on first use, so this must precede any /notify. + * + * Both are applied AFTER the dynamic import, not before. Bun locates its global + * module cache through HOME, and this package has no node_modules — clobbering + * HOME first makes the import fail to resolve smol-toml (via ../lib). Neither + * value is read by voice.ts at import time; both are read inside startVoice and + * playAudio, which run later. + * + * Rate limiting is real here: voice.ts allows 10 POSTs per minute per client + * IP and every request in this file reports as "localhost". Keep the number of + * /notify calls in this file well under that ceiling or the suite goes yellow + * with 429s that have nothing to do with the code under test. + */ + +import { afterEach, beforeAll, describe, expect, test } from "bun:test" +import { mkdtempSync } from "fs" +import { tmpdir } from "os" +import { join } from "path" +import { parse } from "smol-toml" +import { normalizeProviders } from "../../VoiceServer/providers" + +const sandboxHome = mkdtempSync(join(tmpdir(), "lifeos-voice-test-")) + +let startVoice: typeof import("../../VoiceServer/voice").startVoice +let voiceHealth: typeof import("../../VoiceServer/voice").voiceHealth +let handleVoiceRequest: typeof import("../../VoiceServer/voice").handleVoiceRequest + +beforeAll(async () => { + const mod = await import("../../VoiceServer/voice") + startVoice = mod.startVoice + voiceHealth = mod.voiceHealth + handleVoiceRequest = mod.handleVoiceRequest + + process.env.HOME = sandboxHome + process.env.PATH = "" + delete process.env.ELEVENLABS_API_KEY +}) + +// ── Stub Server ── + +interface Stub { + url: string + requests: Array> +} + +const running: Array<{ stop: (force?: boolean) => void }> = [] + +function startStub(handler: () => Response): Stub { + const requests: Array> = [] + const server = Bun.serve({ + port: 0, + async fetch(req) { + const url = new URL(req.url) + if (url.pathname !== "/v1/audio/speech") return new Response("not found", { status: 404 }) + requests.push((await req.json()) as Record) + return handler() + }, + }) + running.push(server) + return { url: `http://127.0.0.1:${server.port}`, requests } +} + +function audioResponse(): Response { + return new Response(new Uint8Array([0x49, 0x44, 0x33, 0x04, 0x00]), { + status: 200, + headers: { "Content-Type": "audio/mpeg" }, + }) +} + +function notify(message = "the estate is clean"): Promise { + return handleVoiceRequest( + new Request("http://localhost:31337/notify", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ title: "LifeOS", message }), + }), + ) +} + +afterEach(() => { + while (running.length) running.pop()?.stop(true) +}) + +// ── TOML Config Parsing ── + +describe("PULSE.toml [voice] parsing", () => { + test("the shipped template declares no chain, so stock installs stay on ElevenLabs", async () => { + const shipped = parse(await Bun.file(new URL("../../PULSE.toml", import.meta.url)).text()) as { + voice?: Record + } + + expect(shipped.voice).toEqual({ enabled: true }) + expect(normalizeProviders(shipped.voice?.providers)).toEqual([]) + }) + + test("[[voice.providers]] blocks parse into an ordered chain", () => { + const config = parse(` +[voice] +enabled = true + +[[voice.providers]] +type = "openai-compatible" +base_url = "http://127.0.0.1:8880" +voice = "am_michael" +model = "kokoro" +response_format = "mp3" +timeout_ms = 8000 + +[[voice.providers]] +type = "elevenlabs" +`) as { voice: Record } + + const chain = normalizeProviders(config.voice.providers) + + expect(chain).toHaveLength(2) + expect(chain[0]).toMatchObject({ + type: "openai-compatible", + base_url: "http://127.0.0.1:8880", + voice: "am_michael", + model: "kokoro", + response_format: "mp3", + timeout_ms: 8000, + }) + expect(chain[1].type).toBe("elevenlabs") + }) +}) + +// ── Backwards Compatibility ── + +describe("no providers configured", () => { + test("health reports the pre-chain shape", () => { + startVoice({ enabled: true }) + + const health = voiceHealth() + expect(health.voice_system).toBe("ElevenLabs") + expect(health.providers).toBeUndefined() + expect(health.initialized).toBe(true) + }) + + test("an empty providers array is treated as no chain at all", () => { + startVoice({ enabled: true, providers: [] }) + expect(voiceHealth().voice_system).toBe("ElevenLabs") + }) + + test("a malformed providers value falls back to the legacy path", () => { + startVoice({ enabled: true, providers: [{ type: "openai-compatible" }, { type: "piper" }] }) + expect(voiceHealth().voice_system).toBe("ElevenLabs") + }) + + test("with no key and no chain, /notify still succeeds and speaks to nobody", async () => { + startVoice({ enabled: true }) + + const response = await notify() + + expect(response?.status).toBe(200) + expect(await response!.json()).toMatchObject({ status: "success" }) + }) + + test("the ElevenLabs default voice fallback is unchanged", () => { + startVoice({ enabled: true }) + expect(voiceHealth().default_voice_id).toBe("21m00Tcm4TlvDq8ikWAM") + }) +}) + +// ── Chain Configured ── + +describe("provider chain configured", () => { + test("health names the chain without leaking credentials", () => { + startVoice({ + enabled: true, + providers: [ + { type: "openai-compatible", base_url: "http://127.0.0.1:8880", api_key: "sk-secret" }, + { type: "elevenlabs" }, + ], + }) + + const health = voiceHealth() + expect(health.voice_system).toBe("chain") + expect(health.providers).toEqual(["openai-compatible(http://127.0.0.1:8880)", "elevenlabs"]) + expect(JSON.stringify(health)).not.toContain("sk-secret") + }) + + test("speaks through the chain with no ElevenLabs key present", async () => { + const stub = startStub(audioResponse) + startVoice({ + enabled: true, + providers: [{ type: "openai-compatible", base_url: stub.url, voice: "am_michael" }], + }) + + const response = await notify("good morning") + + expect(response?.status).toBe(200) + expect(stub.requests).toHaveLength(1) + expect(stub.requests[0]).toMatchObject({ voice: "am_michael", input: "good morning" }) + }) + + test("falls through a dead provider to a healthy one", async () => { + const broken = startStub(() => new Response("model not loaded", { status: 503 })) + const healthy = startStub(audioResponse) + + startVoice({ + enabled: true, + providers: [ + { type: "openai-compatible", base_url: broken.url }, + { type: "openai-compatible", base_url: healthy.url }, + ], + }) + + const response = await notify() + + expect(response?.status).toBe(200) + expect(broken.requests).toHaveLength(1) + expect(healthy.requests).toHaveLength(1) + }) + + test("all providers failing degrades the notification instead of throwing", async () => { + const broken = startStub(() => new Response("down", { status: 500 })) + startVoice({ enabled: true, providers: [{ type: "openai-compatible", base_url: broken.url }] }) + + const response = await notify() + + // 502 is the pre-chain contract for "notification sent, TTS did not". + expect(response?.status).toBe(502) + expect(await response!.json()).toMatchObject({ status: "error", notification_sent: true }) + }) +}) From ecbc3bcf7435e09bb23e0ac21a2ca9d4493fa979 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 14 Aug 2026 10:45:13 -0700 Subject: [PATCH 2/6] feat(voice): per-rung liveness probe, verified request contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Refines the chain against the Kokoro-FastAPI source rather than against the OpenAI shape it resembles. - Liveness per rung, ahead of generation. openai-compatible gets a cheap GET of the server root's /health on a 2s budget, where ANY HTTP response counts as reachable — non-Kokoro servers have no /health, and a 404 still proves the socket answered. Connection refused and timeout are the hard failures. Without this a dead first provider burns its whole generation budget on every notification instead of being stepped over. ElevenLabs is not probed over the network: its liveness is whether a key is configured, which is the same test the pre-chain path already made, so no new failure mode. /health at the app root and /v1/models on the openai router were both confirmed in api/src/main.py before picking the default. - Generation timeout drops 15s -> 10s and health gets its own 2s, both per-provider configurable. Kokoro on CPU runs ~1.3-2x realtime with ~3.5s to first audio on older x86_64, so a one-line notification genuinely takes seconds; 10s covers that without letting a wedged box hold the chain. - speed is always sent, defaulting to 1.0, rather than omitted when unset. - The container format now comes from the response Content-Type when it is recognised, falling back to the requested response_format. A server that ignores an unsupported format and sends mp3 anyway would otherwise have its bytes written to a .flac temp file and played as noise. - A JSON body on a 200 is now a failure. These servers return audio bytes or nothing, so JSON is an error payload wearing a success code, and the chain should fail over rather than play it. - voice is never charset-validated. Kokoro carries blends in that same field ("am_fenrir(2)+am_michael(1)"), so any validation would reject valid config. Backwards compatibility is unchanged: no providers array configured is still the untouched ElevenLabs path, still gated on the API key, and PULSE.toml still parses to exactly {enabled = true}. 61 pass, no network. --- LifeOS/install/LIFEOS/PULSE/PULSE.toml | 19 +- .../LIFEOS/PULSE/VoiceServer/providers.ts | 188 ++++++++++-- .../PULSE/test/VoiceServer/providers.test.ts | 269 +++++++++++++++--- 3 files changed, 411 insertions(+), 65 deletions(-) diff --git a/LifeOS/install/LIFEOS/PULSE/PULSE.toml b/LifeOS/install/LIFEOS/PULSE/PULSE.toml index 9bf47ada8b..e5ceec0208 100644 --- a/LifeOS/install/LIFEOS/PULSE/PULSE.toml +++ b/LifeOS/install/LIFEOS/PULSE/PULSE.toml @@ -88,7 +88,9 @@ enabled = true # voice = "am_michael" # provider's own voice ID # model = "kokoro" # # response_format = "mp3" # mp3 (default), wav, opus, flac, aac, pcm -# # timeout_ms = 15000 +# # timeout_ms = 10000 # generation budget +# # health_path = "/health" # liveness path, relative to the root +# # health_timeout_ms = 2000 # liveness budget # # api_key = "${LOCAL_TTS_KEY}" # sent as a bearer token when set # # [[voice.providers]] @@ -97,12 +99,23 @@ enabled = true # # model = "eleven_turbo_v2_5" # # api_key = "${ELEVENLABS_API_KEY}" # defaults to the module-level key # +# Each rung is probed for liveness before it is asked to speak — a cheap GET +# of the server root's /health (2s budget), where ANY HTTP response counts as +# reachable. A 404 is fine: non-Kokoro OpenAI-shaped servers have no /health, +# and the question is only whether anything is listening. Connection refused +# and timeout are the hard failures, and skipping those rungs cheaply is the +# point — otherwise a dead first provider burns its full 10s generation budget +# on every single notification. ElevenLabs is not probed over the network; its +# liveness is whether a key is configured, the same test the pre-chain path made. +# # `openai-compatible` targets POST {base_url}/v1/audio/speech with -# {model, voice, input, response_format} and expects raw audio bytes back — -# the OpenAI speech contract, which Kokoro-FastAPI and LocalAI both implement. +# {model, input, voice, response_format, speed, stream} and expects RAW AUDIO +# BYTES back, never JSON — the OpenAI speech contract, which Kokoro-FastAPI and +# LocalAI both implement. A JSON body on a 200 is treated as a failure. # `stream: false` is sent on every request: Kokoro-FastAPI streams by default, # and a streamed reply returns 200 before generation finishes, so a failure # mid-sentence would arrive as truncated audio instead of falling through. +# The 10s generation default reflects Kokoro on CPU at ~1.3-2x realtime. # # Kokoro voice IDs are `{lang}{gender}_{name}`. On the male English side its # own quality grades put `am_fenrir`, `am_michael` and `am_puck` at the top and diff --git a/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts b/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts index bb7085cae4..94ace78371 100644 --- a/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts +++ b/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts @@ -1,11 +1,11 @@ /** * LifeOS Pulse — Voice Provider Chain * - * An ordered list of TTS providers. The first one that produces audio wins; - * every failure (non-2xx, timeout, connection refused) falls through to the - * next. When the list is exhausted the caller stays silent — synthesis never - * throws out of this module, because /notify must not die when a TTS box is - * down. + * An ordered list of TTS providers. Each rung is checked for liveness, then + * asked to speak; every failure (unreachable, non-2xx, timeout, connection + * refused) falls through to the next. When the list is exhausted the caller + * stays silent — synthesis never throws out of this module, because /notify + * must not die when a TTS box is down. * * Two provider types ship today: * elevenlabs — the hosted default, unchanged from the pre-chain path @@ -35,6 +35,7 @@ export interface OpenAiCompatibleProviderConfig { type: "openai-compatible" /** Server root (`http://127.0.0.1:8880`) or its /v1 base — both accepted. */ base_url: string + /** Base voice ID or a blend expression. Passed through verbatim. */ voice?: string model?: string /** mp3 (default), wav, opus, flac, aac, pcm — whatever the server supports. */ @@ -42,6 +43,9 @@ export interface OpenAiCompatibleProviderConfig { /** Sent as `Authorization: Bearer …` when present. Local servers ignore it. */ api_key?: string timeout_ms?: number + /** Liveness path, relative to the server root. Defaults to /health. */ + health_path?: string + health_timeout_ms?: number } export type VoiceProviderConfig = ElevenLabsProviderConfig | OpenAiCompatibleProviderConfig @@ -78,8 +82,8 @@ export type Logger = (level: "info" | "warn" | "error", message: string, meta?: export const DEFAULT_ELEVENLABS_MODEL = "eleven_turbo_v2_5" /** - * Kokoro-FastAPI ignores `model` but the OpenAI schema requires it, so a - * non-empty default keeps strict servers happy without forcing config. + * Kokoro-FastAPI accepts tts-1, tts-1-hd and kokoro, and defaults to kokoro. + * Sending it explicitly keeps stricter OpenAI-shaped servers happy. */ export const DEFAULT_OPENAI_MODEL = "kokoro" @@ -89,7 +93,21 @@ export const DEFAULT_OPENAI_VOICE = "af_heart" /** mp3 keeps the existing playback path (afplay/ffplay/mpg123) working as-is. */ export const DEFAULT_RESPONSE_FORMAT = "mp3" -export const DEFAULT_TIMEOUT_MS = 15_000 +export const DEFAULT_SPEED = 1.0 + +/** + * Generation budget per rung. Kokoro on CPU runs ~1.3–2x realtime on a modest + * x86_64 box with ~3.5s to first audio on older silicon, so a one-line + * notification legitimately takes several seconds. 10s leaves room for that + * without letting a wedged box hold the whole chain. + */ +export const DEFAULT_TIMEOUT_MS = 10_000 + +/** Liveness budget. A reachable server answers this immediately or it is down. */ +export const DEFAULT_HEALTH_TIMEOUT_MS = 2_000 + +/** Verified against api/src/main.py — `@app.get("/health")`, at the app root. */ +export const DEFAULT_HEALTH_PATH = "/health" const FORMAT_EXTENSIONS: Record = { mp3: "mp3", @@ -100,11 +118,40 @@ const FORMAT_EXTENSIONS: Record = { pcm: "pcm", } +/** + * Response Content-Type → container format. Mirrors the server-side map in + * Kokoro-FastAPI's openai_compatible.py, plus the common wav spellings. + */ +const CONTENT_TYPE_FORMATS: Record = { + "audio/mpeg": "mp3", + "audio/mp3": "mp3", + "audio/opus": "opus", + "audio/ogg": "opus", + "audio/aac": "aac", + "audio/flac": "flac", + "audio/x-flac": "flac", + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/wave": "wav", + "audio/pcm": "pcm", +} + /** Temp-file extension for a container format. Unknown formats play as mp3. */ export function extensionForFormat(format: string): string { return FORMAT_EXTENSIONS[format.toLowerCase()] ?? "mp3" } +/** + * Container format implied by a response Content-Type, or null when the header + * is missing or unrecognised — in which case the configured response_format + * stands, since that is what was asked for. + */ +export function formatFromContentType(contentType: string | null): string | null { + if (!contentType) return null + const bare = contentType.split(";")[0].trim().toLowerCase() + return CONTENT_TYPE_FORMATS[bare] ?? null +} + // ── Config Normalization ── function asString(value: unknown): string | undefined { @@ -122,6 +169,10 @@ function asNumber(value: unknown): number | undefined { * the signal for "no chain configured", which keeps the legacy ElevenLabs path * in charge. Individual malformed entries are dropped with a warning rather * than failing the whole chain: one typo should not mute the system. + * + * `voice` is never inspected beyond "is it a non-empty string". Kokoro carries + * blend expressions in that same field — `am_fenrir(2)+am_michael(1)` — so any + * charset validation here would reject valid configuration. */ export function normalizeProviders(raw: unknown, log?: Logger): VoiceProviderConfig[] { if (!Array.isArray(raw) || raw.length === 0) return [] @@ -162,6 +213,8 @@ export function normalizeProviders(raw: unknown, log?: Logger): VoiceProviderCon response_format: asString(record.response_format), api_key: asString(record.api_key), timeout_ms: asNumber(record.timeout_ms), + health_path: asString(record.health_path), + health_timeout_ms: asNumber(record.health_timeout_ms), }) return } @@ -180,16 +233,73 @@ export function providerLabel(provider: VoiceProviderConfig): string { // ── Endpoint Resolution ── /** - * Build the speech URL from a configured base. + * Server root for a configured base. * - * Both spellings are accepted because both are natural: the server root + * Both spellings are accepted because both are natural: the bare root * (`http://127.0.0.1:8880`, what Kokoro-FastAPI's README prints) and the - * OpenAI-style base that already carries /v1 (what OPENAI_BASE_URL looks - * like). Appending blindly would produce /v1/v1/audio/speech for the second. + * OpenAI-style base that already carries /v1 (what OPENAI_BASE_URL looks like). + * No port is ever assumed — base_url is always explicit in config. */ -export function speechEndpoint(baseUrl: string): string { +export function serverRoot(baseUrl: string): string { const trimmed = baseUrl.replace(/\/+$/, "") - return trimmed.endsWith("/v1") ? `${trimmed}/audio/speech` : `${trimmed}/v1/audio/speech` + return trimmed.endsWith("/v1") ? trimmed.slice(0, -"/v1".length) : trimmed +} + +/** Speech endpoint. Appending blindly would double the /v1 on an OpenAI base. */ +export function speechEndpoint(baseUrl: string): string { + return `${serverRoot(baseUrl)}/v1/audio/speech` +} + +/** + * Liveness endpoint. Defaults to the server root's /health, which + * Kokoro-FastAPI defines at the app level (outside the /v1 router). + */ +export function healthEndpoint(baseUrl: string, healthPath?: string): string { + const path = healthPath ?? DEFAULT_HEALTH_PATH + return `${serverRoot(baseUrl)}${path.startsWith("/") ? path : `/${path}`}` +} + +// ── Liveness ── + +/** + * Is this rung worth spending a generation timeout on? + * + * ANY HTTP response counts as reachable, including a 404. Only OpenAI-shaped + * servers that are not Kokoro lack /health, and a 404 from one still proves + * the socket answered — which is the actual question. Hard failure is reserved + * for connection refused and timeout, exactly the cases where the expensive + * POST would burn its full budget before failing anyway. + * + * ElevenLabs is not probed over the network. The liveness the pre-chain code + * implies is "is a key configured", so that is what is reused — no new + * request, no new failure mode, and no spend on a rung that cannot authorise. + */ +export async function isProviderReachable( + provider: VoiceProviderConfig, + ctx: Pick, + log?: Logger, +): Promise { + if (provider.type === "elevenlabs") { + return !!(provider.api_key ?? ctx.elevenLabsApiKey) + } + + const url = healthEndpoint(provider.base_url, provider.health_path) + + try { + const response = await fetch(url, { + method: "GET", + signal: AbortSignal.timeout(provider.health_timeout_ms ?? DEFAULT_HEALTH_TIMEOUT_MS), + }) + // Drain the body so the socket is released promptly. + await response.arrayBuffer().catch(() => undefined) + return true + } catch (error: unknown) { + log?.("warn", `Voice: ${providerLabel(provider)} health probe failed`, { + url, + error: error instanceof Error ? error.message : String(error), + }) + return false + } } // ── Provider Clients ── @@ -237,9 +347,10 @@ export async function elevenLabsSynthesize(opts: { /** * OpenAI-compatible speech call — the Kokoro-FastAPI contract. * - * POST {base_url}/v1/audio/speech with {model, voice, input, response_format} - * returning raw audio bytes. `input` is the text field; that naming is the - * OpenAI schema, not a typo. + * POST {base_url}/v1/audio/speech with + * {model, input, voice, response_format, speed, stream} returning RAW AUDIO + * BYTES, never JSON. `input` is the text field; that naming is the OpenAI + * schema, not a typo. * * `stream: false` is sent deliberately. Kokoro-FastAPI defaults `stream` to * TRUE — unlike OpenAI, which has no such field — and a streamed reply commits @@ -255,24 +366,26 @@ export async function openAiCompatibleSynthesize(opts: { voice?: string model?: string responseFormat?: string - /** Playback rate, carried over from the resolved voice settings when set. */ + /** Playback rate. Defaults to 1.0 rather than being omitted. */ speed?: number apiKey?: string timeoutMs?: number -}): Promise { +}): Promise<{ audio: ArrayBuffer; format: string }> { const headers: Record = { "Content-Type": "application/json" } if (opts.apiKey) headers.Authorization = `Bearer ${opts.apiKey}` + const requestedFormat = opts.responseFormat ?? DEFAULT_RESPONSE_FORMAT + const response = await fetch(speechEndpoint(opts.baseUrl), { method: "POST", headers, body: JSON.stringify({ model: opts.model ?? DEFAULT_OPENAI_MODEL, - voice: opts.voice ?? DEFAULT_OPENAI_VOICE, input: opts.text, - response_format: opts.responseFormat ?? DEFAULT_RESPONSE_FORMAT, + voice: opts.voice ?? DEFAULT_OPENAI_VOICE, + response_format: requestedFormat, + speed: Number.isFinite(opts.speed) ? opts.speed : DEFAULT_SPEED, stream: false, - ...(Number.isFinite(opts.speed) ? { speed: opts.speed } : {}), }), signal: opts.timeoutMs ? AbortSignal.timeout(opts.timeoutMs) : undefined, }) @@ -282,10 +395,21 @@ export async function openAiCompatibleSynthesize(opts: { throw new Error(`TTS server error: ${response.status} - ${errorText}`) } + // A JSON body on a 200 is an error payload wearing a success code — these + // servers return audio bytes or nothing. Treating it as audio would write a + // stub file and call it speech. + const contentType = response.headers.get("content-type") + if (contentType?.toLowerCase().includes("application/json")) { + const detail = await response.text().catch(() => "") + throw new Error(`TTS server returned JSON, not audio: ${detail.slice(0, 200)}`) + } + const audio = await response.arrayBuffer() if (audio.byteLength === 0) throw new Error("TTS server returned an empty body") - return audio + // Trust what the server actually sent over what was asked for; they differ + // when a server silently ignores an unsupported response_format. + return { audio, format: formatFromContentType(contentType) ?? requestedFormat } } // ── Chain Runner ── @@ -305,6 +429,13 @@ export async function synthesizeViaChain( for (let i = 0; i < providers.length; i++) { const provider = providers[i] const label = providerLabel(provider) + const isLast = i === providers.length - 1 + const outcome = isLast ? "chain exhausted" : "falling through" + + if (!(await isProviderReachable(provider, ctx, log))) { + log?.("warn", `Voice: provider[${i}] ${label} is not reachable — ${outcome}`) + continue + } try { if (provider.type === "elevenlabs") { @@ -322,7 +453,7 @@ export async function synthesizeViaChain( return { audio, format: "mp3", provider: label } } - const audio = await openAiCompatibleSynthesize({ + const { audio, format } = await openAiCompatibleSynthesize({ text: ctx.text, baseUrl: provider.base_url, voice: provider.voice, @@ -334,14 +465,11 @@ export async function synthesizeViaChain( apiKey: provider.api_key, timeoutMs: provider.timeout_ms ?? DEFAULT_TIMEOUT_MS, }) - const format = provider.response_format ?? DEFAULT_RESPONSE_FORMAT - log?.("info", `Voice: provider[${i}] ${label} synthesized ${audio.byteLength} bytes`) + log?.("info", `Voice: provider[${i}] ${label} synthesized ${audio.byteLength} bytes as ${format}`) return { audio, format, provider: label } } catch (error: unknown) { - const message = error instanceof Error ? error.message : String(error) - const isLast = i === providers.length - 1 - log?.("warn", `Voice: provider[${i}] ${label} failed — ${isLast ? "chain exhausted" : "falling through"}`, { - error: message, + log?.("warn", `Voice: provider[${i}] ${label} failed — ${outcome}`, { + error: error instanceof Error ? error.message : String(error), }) } } diff --git a/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts index 9ec598f2ff..741d8e1001 100644 --- a/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts +++ b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts @@ -3,6 +3,11 @@ * * Everything runs against a stub Bun.serve on an ephemeral port. No network, * no ElevenLabs credentials, no fixed ports. + * + * The stubs answer 404 on every path except /v1/audio/speech, which is + * deliberate: the liveness probe treats ANY HTTP response as reachable, so a + * stub that does not implement /health still exercises the real code path the + * way a non-Kokoro OpenAI-shaped server would. */ import { afterEach, describe, expect, test } from "bun:test" @@ -10,15 +15,19 @@ import { DEFAULT_OPENAI_MODEL, DEFAULT_OPENAI_VOICE, DEFAULT_RESPONSE_FORMAT, + DEFAULT_SPEED, elevenLabsSynthesize, extensionForFormat, + formatFromContentType, + healthEndpoint, + isProviderReachable, normalizeProviders, openAiCompatibleSynthesize, providerLabel, + serverRoot, speechEndpoint, synthesizeViaChain, type SynthesisContext, - type VoiceProviderConfig, } from "../../VoiceServer/providers" // ── Stub Server Harness ── @@ -28,6 +37,8 @@ interface Stub { /** Bodies of every /v1/audio/speech request this stub received, in order. */ requests: Array> headers: Array + /** Paths of every non-speech GET, so probes can be asserted. */ + probes: string[] } const running: Array<{ stop: (force?: boolean) => void }> = [] @@ -35,12 +46,14 @@ const running: Array<{ stop: (force?: boolean) => void }> = [] function startStub(handler: (body: Record, req: Request) => Response | Promise): Stub { const requests: Array> = [] const headers: Array = [] + const probes: string[] = [] const server = Bun.serve({ port: 0, async fetch(req) { const url = new URL(req.url) if (url.pathname !== "/v1/audio/speech" || req.method !== "POST") { + probes.push(url.pathname) return new Response("not found", { status: 404 }) } const body = (await req.json()) as Record @@ -51,7 +64,20 @@ function startStub(handler: (body: Record, req: Request) => Res }) running.push(server) - return { url: `http://127.0.0.1:${server.port}`, requests, headers } + return { url: `http://127.0.0.1:${server.port}`, requests, headers, probes } +} + +/** A server that accepts the connection and then never answers anything. */ +function startHangingStub(): { url: string } { + const server = Bun.serve({ + port: 0, + async fetch() { + await new Promise((resolve) => setTimeout(resolve, 30_000)) + return new Response("too late") + }, + }) + running.push(server) + return { url: `http://127.0.0.1:${server.port}` } } /** A port with nothing listening on it — for the connection-refused cases. */ @@ -62,8 +88,8 @@ function deadUrl(): string { return `http://127.0.0.1:${port}` } -function audioResponse(bytes = new Uint8Array([0x49, 0x44, 0x33, 0x04])): Response { - return new Response(bytes, { status: 200, headers: { "Content-Type": "audio/mpeg" } }) +function audioResponse(bytes = new Uint8Array([0x49, 0x44, 0x33, 0x04]), contentType = "audio/mpeg"): Response { + return new Response(bytes, { status: 200, headers: { "Content-Type": contentType } }) } function ctx(overrides: Partial = {}): SynthesisContext { @@ -120,10 +146,29 @@ describe("normalizeProviders", () => { response_format: "wav", api_key: "sk-local", timeout_ms: 3000, + health_path: "/v1/models", + health_timeout_ms: 500, }, ]) - expect(chain[0]).toMatchObject({ response_format: "wav", api_key: "sk-local", timeout_ms: 3000 }) + expect(chain[0]).toMatchObject({ + response_format: "wav", + api_key: "sk-local", + timeout_ms: 3000, + health_path: "/v1/models", + health_timeout_ms: 500, + }) + }) + + test("passes voice blend expressions through untouched", () => { + // Kokoro carries blends in the same `voice` field; any charset validation + // here would reject valid config. + const blends = ["am_fenrir(2)+am_michael(1)", "af_bella+af_sky", "am_puck(0.3)-am_adam(0.1)"] + + for (const blend of blends) { + const chain = normalizeProviders([{ type: "openai-compatible", base_url: "http://x", voice: blend }]) + expect(chain[0].voice).toBe(blend) + } }) test("drops an openai-compatible entry with no base_url, keeping the rest", () => { @@ -177,23 +222,38 @@ describe("normalizeProviders", () => { // ── Endpoint Resolution ── -describe("speechEndpoint", () => { - test("appends /v1/audio/speech to a server root", () => { - expect(speechEndpoint("http://127.0.0.1:8880")).toBe("http://127.0.0.1:8880/v1/audio/speech") +describe("endpoint resolution", () => { + test("derives the server root from either spelling", () => { + expect(serverRoot("http://127.0.0.1:8880")).toBe("http://127.0.0.1:8880") + expect(serverRoot("http://127.0.0.1:8880/")).toBe("http://127.0.0.1:8880") + expect(serverRoot("http://127.0.0.1:8880/v1")).toBe("http://127.0.0.1:8880") + expect(serverRoot("http://127.0.0.1:8880/v1/")).toBe("http://127.0.0.1:8880") }) - test("tolerates trailing slashes", () => { + test("appends /v1/audio/speech exactly once", () => { + expect(speechEndpoint("http://127.0.0.1:8880")).toBe("http://127.0.0.1:8880/v1/audio/speech") expect(speechEndpoint("http://127.0.0.1:8880/")).toBe("http://127.0.0.1:8880/v1/audio/speech") + expect(speechEndpoint("http://127.0.0.1:8880/v1")).toBe("http://127.0.0.1:8880/v1/audio/speech") }) - test("does not double the /v1 when the base already carries it", () => { - expect(speechEndpoint("http://127.0.0.1:8880/v1")).toBe("http://127.0.0.1:8880/v1/audio/speech") - expect(speechEndpoint("http://127.0.0.1:8880/v1/")).toBe("http://127.0.0.1:8880/v1/audio/speech") + test("health defaults to the app-level /health, outside the /v1 router", () => { + expect(healthEndpoint("http://127.0.0.1:8880")).toBe("http://127.0.0.1:8880/health") + expect(healthEndpoint("http://127.0.0.1:8880/v1")).toBe("http://127.0.0.1:8880/health") + }) + + test("an override path is honoured, with or without a leading slash", () => { + expect(healthEndpoint("http://x:8880", "/v1/models")).toBe("http://x:8880/v1/models") + expect(healthEndpoint("http://x:8880", "v1/models")).toBe("http://x:8880/v1/models") + }) + + test("no port is ever assumed", () => { + expect(speechEndpoint("https://tts.example.com")).toBe("https://tts.example.com/v1/audio/speech") + expect(healthEndpoint("https://tts.example.com")).toBe("https://tts.example.com/health") }) }) -describe("extensionForFormat", () => { - test("maps the known container formats", () => { +describe("format mapping", () => { + test("maps the known container formats to extensions", () => { expect(extensionForFormat("mp3")).toBe("mp3") expect(extensionForFormat("wav")).toBe("wav") expect(extensionForFormat("opus")).toBe("opus") @@ -203,42 +263,120 @@ describe("extensionForFormat", () => { test("falls back to mp3 for anything unrecognised", () => { expect(extensionForFormat("ogg-but-not-really")).toBe("mp3") }) + + test("reads the container back off a Content-Type", () => { + expect(formatFromContentType("audio/mpeg")).toBe("mp3") + expect(formatFromContentType("audio/wav")).toBe("wav") + expect(formatFromContentType("audio/x-wav")).toBe("wav") + expect(formatFromContentType("audio/flac; charset=binary")).toBe("flac") + expect(formatFromContentType("audio/pcm")).toBe("pcm") + }) + + test("returns null when the header is missing or unknown", () => { + expect(formatFromContentType(null)).toBeNull() + expect(formatFromContentType("application/octet-stream")).toBeNull() + }) +}) + +// ── Liveness ── + +describe("isProviderReachable", () => { + test("a live server is reachable, and the probe hits /health", async () => { + const stub = startStub(() => audioResponse()) + const [provider] = normalizeProviders([{ type: "openai-compatible", base_url: stub.url }]) + + expect(await isProviderReachable(provider, {})).toBe(true) + expect(stub.probes).toEqual(["/health"]) + }) + + test("a 404 still counts as reachable — the socket answered", async () => { + // Non-Kokoro OpenAI-shaped servers have no /health. The question the probe + // asks is "is anything listening", not "is this Kokoro". + const stub = startStub(() => audioResponse()) + const [provider] = normalizeProviders([{ type: "openai-compatible", base_url: stub.url }]) + + expect(await isProviderReachable(provider, {})).toBe(true) + expect(stub.probes[0]).toBe("/health") + }) + + test("connection refused is not reachable", async () => { + const [provider] = normalizeProviders([{ type: "openai-compatible", base_url: deadUrl() }]) + + expect(await isProviderReachable(provider, {})).toBe(false) + }) + + test("a hanging server is not reachable once the health budget expires", async () => { + const stub = startHangingStub() + const [provider] = normalizeProviders([ + { type: "openai-compatible", base_url: stub.url, health_timeout_ms: 40 }, + ]) + + const started = Date.now() + expect(await isProviderReachable(provider, {})).toBe(false) + expect(Date.now() - started).toBeLessThan(1_000) + }) + + test("an override health path is the one actually requested", async () => { + const stub = startStub(() => audioResponse()) + const [provider] = normalizeProviders([ + { type: "openai-compatible", base_url: stub.url, health_path: "/v1/models" }, + ]) + + expect(await isProviderReachable(provider, {})).toBe(true) + expect(stub.probes).toEqual(["/v1/models"]) + }) + + test("elevenlabs liveness is the configured key, with no network call", async () => { + const [provider] = normalizeProviders([{ type: "elevenlabs" }]) + + expect(await isProviderReachable(provider, { elevenLabsApiKey: "xi-key" })).toBe(true) + expect(await isProviderReachable(provider, {})).toBe(false) + }) + + test("a per-provider elevenlabs key satisfies liveness on its own", async () => { + const [provider] = normalizeProviders([{ type: "elevenlabs", api_key: "xi-own" }]) + + expect(await isProviderReachable(provider, {})).toBe(true) + }) }) // ── OpenAI-Compatible Client Contract ── describe("openAiCompatibleSynthesize", () => { - test("sends {model, voice, input, response_format} and returns the bytes", async () => { + test("sends the full documented body and returns the bytes", async () => { const stub = startStub(() => audioResponse(new Uint8Array([1, 2, 3, 4, 5]))) - const audio = await openAiCompatibleSynthesize({ + const { audio } = await openAiCompatibleSynthesize({ text: "good morning", baseUrl: stub.url, voice: "am_michael", model: "kokoro", + speed: 1.0, }) expect(new Uint8Array(audio)).toEqual(new Uint8Array([1, 2, 3, 4, 5])) expect(stub.requests).toHaveLength(1) expect(stub.requests[0]).toEqual({ model: "kokoro", - voice: "am_michael", input: "good morning", + voice: "am_michael", response_format: "mp3", + speed: 1.0, stream: false, }) }) - test("applies the documented defaults when voice/model/format are unset", async () => { + test("applies the documented defaults when voice/model/format/speed are unset", async () => { const stub = startStub(() => audioResponse()) await openAiCompatibleSynthesize({ text: "hello", baseUrl: stub.url }) expect(stub.requests[0]).toEqual({ model: DEFAULT_OPENAI_MODEL, - voice: DEFAULT_OPENAI_VOICE, input: "hello", + voice: DEFAULT_OPENAI_VOICE, response_format: DEFAULT_RESPONSE_FORMAT, + speed: DEFAULT_SPEED, stream: false, }) }) @@ -248,19 +386,19 @@ describe("openAiCompatibleSynthesize", () => { await openAiCompatibleSynthesize({ text: "hello", baseUrl: stub.url }) - // Kokoro-FastAPI defaults stream to true; omitting the field would opt us - // into chunked replies that report success before generation finishes. expect(stub.requests[0].stream).toBe(false) }) - test("sends speed only when one was resolved", async () => { - const withSpeed = startStub(() => audioResponse()) - await openAiCompatibleSynthesize({ text: "hi", baseUrl: withSpeed.url, speed: 1.15 }) - expect(withSpeed.requests[0].speed).toBe(1.15) + test("sends a blend expression verbatim", async () => { + const stub = startStub(() => audioResponse()) - const without = startStub(() => audioResponse()) - await openAiCompatibleSynthesize({ text: "hi", baseUrl: without.url }) - expect(without.requests[0]).not.toHaveProperty("speed") + await openAiCompatibleSynthesize({ + text: "hi", + baseUrl: stub.url, + voice: "am_fenrir(2)+am_michael(1)", + }) + + expect(stub.requests[0].voice).toBe("am_fenrir(2)+am_michael(1)") }) test("sends a bearer token only when an api_key is configured", async () => { @@ -273,12 +411,64 @@ describe("openAiCompatibleSynthesize", () => { expect(without.headers[0].get("authorization")).toBeNull() }) + test("reports the container the server actually sent", async () => { + const stub = startStub(() => audioResponse(new Uint8Array([1]), "audio/wav")) + + const { format } = await openAiCompatibleSynthesize({ + text: "hi", + baseUrl: stub.url, + responseFormat: "wav", + }) + + expect(format).toBe("wav") + }) + + test("trusts the response Content-Type over the requested format", async () => { + // A server that ignores an unsupported response_format and sends mp3 + // anyway must not have its bytes written to a .flac file. + const stub = startStub(() => audioResponse(new Uint8Array([1]), "audio/mpeg")) + + const { format } = await openAiCompatibleSynthesize({ + text: "hi", + baseUrl: stub.url, + responseFormat: "flac", + }) + + expect(format).toBe("mp3") + }) + + test("falls back to the requested format when the server sends no usable type", async () => { + const stub = startStub(() => audioResponse(new Uint8Array([1]), "application/octet-stream")) + + const { format } = await openAiCompatibleSynthesize({ + text: "hi", + baseUrl: stub.url, + responseFormat: "opus", + }) + + expect(format).toBe("opus") + }) + test("throws on a non-2xx response", async () => { const stub = startStub(() => new Response("model not loaded", { status: 503 })) await expect(openAiCompatibleSynthesize({ text: "hi", baseUrl: stub.url })).rejects.toThrow(/503/) }) + test("rejects a JSON body on a 200 — that is an error wearing a success code", async () => { + const stub = startStub( + () => + new Response(JSON.stringify({ detail: "voice not found" }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }), + ) + + await expect(openAiCompatibleSynthesize({ text: "hi", baseUrl: stub.url })).rejects.toThrow( + /JSON, not audio/, + ) + }) + test("treats an empty 200 body as a failure", async () => { const stub = startStub(() => new Response(new Uint8Array([]), { status: 200 })) @@ -318,6 +508,8 @@ describe("synthesizeViaChain", () => { expect(new Uint8Array(result!.audio)).toEqual(new Uint8Array([9, 9, 9])) expect(first.requests).toHaveLength(1) expect(second.requests).toHaveLength(0) + // The unused rung is never even probed. + expect(second.probes).toHaveLength(0) }) test("falls through to the next provider on a non-2xx", async () => { @@ -336,18 +528,22 @@ describe("synthesizeViaChain", () => { expect(healthy.requests).toHaveLength(1) }) - test("falls through on connection refused", async () => { + test("skips an unreachable provider without spending the generation budget", async () => { const healthy = startStub(() => audioResponse()) const chain = normalizeProviders([ + // 10s generation timeout would be paid on every notification without the + // cheap probe in front of it. { type: "openai-compatible", base_url: deadUrl() }, { type: "openai-compatible", base_url: healthy.url }, ]) + const started = Date.now() const result = await synthesizeViaChain(chain, ctx()) expect(result).not.toBeNull() expect(healthy.requests).toHaveLength(1) + expect(Date.now() - started).toBeLessThan(2_000) }) test("falls through on timeout", async () => { @@ -367,7 +563,7 @@ describe("synthesizeViaChain", () => { expect(new Uint8Array(result!.audio)).toEqual(new Uint8Array([1, 1])) }) - test("falls through across provider types — a keyless elevenlabs link is skipped", async () => { + test("skips a keyless elevenlabs rung and speaks on the next one", async () => { const healthy = startStub(() => audioResponse(new Uint8Array([5]))) const chain = normalizeProviders([ @@ -415,9 +611,9 @@ describe("synthesizeViaChain", () => { }) test("reports the response format so playback picks the right extension", async () => { - const stub = startStub(() => audioResponse()) + const stub = startStub(() => audioResponse(new Uint8Array([1]), "audio/wav")) - const chain: VoiceProviderConfig[] = normalizeProviders([ + const chain = normalizeProviders([ { type: "openai-compatible", base_url: stub.url, response_format: "wav" }, ]) @@ -445,4 +641,13 @@ describe("synthesizeViaChain", () => { expect(stub.requests[0].speed).toBe(0.9) }) + + test("defaults speed to 1.0 when the resolved settings carry none", async () => { + const stub = startStub(() => audioResponse()) + const chain = normalizeProviders([{ type: "openai-compatible", base_url: stub.url }]) + + await synthesizeViaChain(chain, ctx()) + + expect(stub.requests[0].speed).toBe(DEFAULT_SPEED) + }) }) From 138344fc8bc69ce032473263297e50c26e844520 Mon Sep 17 00:00:00 2001 From: Daniel Date: Fri, 14 Aug 2026 14:39:16 -0700 Subject: [PATCH 3/6] docs(voice): pcm playback caveat in the format map and PULSE.toml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review nits from the chain review: pcm synthesizes fine but raw headerless samples can't be demuxed from a temp-file extension, so stock players fail. The health string change (voice_system: "chain" when a chain is configured) is deliberate and gated — noted here so the PR body calls it out for dashboard consumers. --- LifeOS/install/LIFEOS/PULSE/PULSE.toml | 3 ++- LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/LifeOS/install/LIFEOS/PULSE/PULSE.toml b/LifeOS/install/LIFEOS/PULSE/PULSE.toml index e5ceec0208..770b4d3d14 100644 --- a/LifeOS/install/LIFEOS/PULSE/PULSE.toml +++ b/LifeOS/install/LIFEOS/PULSE/PULSE.toml @@ -87,7 +87,8 @@ enabled = true # base_url = "http://127.0.0.1:8880" # server root, or its /v1 base # voice = "am_michael" # provider's own voice ID # model = "kokoro" -# # response_format = "mp3" # mp3 (default), wav, opus, flac, aac, pcm +# # response_format = "mp3" # mp3 (default), wav, opus, flac, aac +# # # (pcm accepted but unplayable by stock players — see providers.ts) # # timeout_ms = 10000 # generation budget # # health_path = "/health" # liveness path, relative to the root # # health_timeout_ms = 2000 # liveness budget diff --git a/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts b/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts index 94ace78371..c408a0bc2f 100644 --- a/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts +++ b/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts @@ -115,6 +115,11 @@ const FORMAT_EXTENSIONS: Record = { opus: "opus", flac: "flac", aac: "aac", + // pcm is accepted from the server but is raw 16-bit samples with no header + // (24kHz, known out of band) — file players can't demux it from extension + // alone, so configuring response_format = "pcm" will synthesize and then + // fail playback on most setups. Prefer mp3/wav unless the playback path is + // custom-built for raw PCM. pcm: "pcm", } From 9c8c1b9309fc2332d7eb529703b0740945e3263f Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 15 Aug 2026 11:45:45 -0700 Subject: [PATCH 4/6] test(voice): honour $HOME override over homedir() so the suite's sandbox holds The homedir() refactor reads the OS home directly under Bun, defeating the test suite's HOME-redirection sandbox and letting the suite read the operator's real settings.json. $HOME wins when set; homedir() remains the fallback. --- LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts b/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts index 29e314e4a1..4feeb46591 100644 --- a/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts +++ b/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts @@ -31,6 +31,11 @@ import { type VoiceProviderConfig, } from "./providers" +// $HOME wins over homedir() so a runtime override (the test suite's sandbox +// HOME, a containerized run) is honoured; homedir() is the fallback when the +// variable is absent. +const homeDir = () => process.env.HOME || homedir() + // ── Public Config Interface ── export interface VoiceConfig { @@ -178,7 +183,7 @@ function escapeRegex(str: string): string { } function loadPronunciations(customPath?: string): void { - const paiDir = join(homedir(), ".claude", "LIFEOS") + const paiDir = join(homeDir(), ".claude", "LIFEOS") const userPronPath = customPath ?? join(paiDir, "USER", "PRINCIPAL", "PRONUNCIATIONS.json") try { @@ -218,7 +223,7 @@ function applyPronunciations(text: string): string { // ── Voice Config from settings.json ── function loadVoiceConfigFromSettings(): LoadedVoiceConfig { - const settingsPath = join(homedir(), ".claude", "settings.json") + const settingsPath = join(homeDir(), ".claude", "settings.json") try { if (!existsSync(settingsPath)) { @@ -820,7 +825,7 @@ export async function handleVoiceRequest(req: Request): Promise // /notify/personality honest with whatever the user last selected. let voiceId: string | null = null try { - const settingsFile = join(homedir(), ".claude", "settings.json") + const settingsFile = join(homeDir(), ".claude", "settings.json") const settings = JSON.parse(readFileSync(settingsFile, "utf-8")) const main = settings?.daidentity?.voices?.main const vid = (main?.voiceId || main?.VOICE_ID || main?.voice_id) as string | undefined From 2f7ecc305a6574b837576b46368cb4e3a96e6005 Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 15 Aug 2026 12:06:30 -0700 Subject: [PATCH 5/6] fix(voice): make Kokoro's stream:false opt-out for strict OpenAI endpoints Cross-vendor audit flagged that the unconditional stream:false field can be rejected as unknown by a strict OpenAI-compatible server, failing every request for that provider. New send_stream_flag config (default true, the Kokoro-safe behaviour that keeps failures in the status code) omits the field entirely when set false. Documented in PULSE.toml; regression test asserts both the default-sends and the opt-out-omits paths. --- LifeOS/install/LIFEOS/PULSE/PULSE.toml | 3 ++ .../LIFEOS/PULSE/VoiceServer/providers.ts | 38 +++++++++++++++---- .../PULSE/test/VoiceServer/providers.test.ts | 8 ++++ 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/LifeOS/install/LIFEOS/PULSE/PULSE.toml b/LifeOS/install/LIFEOS/PULSE/PULSE.toml index 770b4d3d14..f575d92ff0 100644 --- a/LifeOS/install/LIFEOS/PULSE/PULSE.toml +++ b/LifeOS/install/LIFEOS/PULSE/PULSE.toml @@ -93,6 +93,9 @@ enabled = true # # health_path = "/health" # liveness path, relative to the root # # health_timeout_ms = 2000 # liveness budget # # api_key = "${LOCAL_TTS_KEY}" # sent as a bearer token when set +# # send_stream_flag = false # omit Kokoro's stream:false for a STRICT +# # # OpenAI endpoint that rejects the field +# # # (default true — keep it for Kokoro) # # [[voice.providers]] # type = "elevenlabs" diff --git a/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts b/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts index c408a0bc2f..4277457627 100644 --- a/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts +++ b/LifeOS/install/LIFEOS/PULSE/VoiceServer/providers.ts @@ -46,6 +46,14 @@ export interface OpenAiCompatibleProviderConfig { /** Liveness path, relative to the server root. Defaults to /health. */ health_path?: string health_timeout_ms?: number + /** + * Whether to send Kokoro's `stream: false` request field. Defaults to true — + * Kokoro-FastAPI streams by default and a streamed reply hides mid-generation + * failures behind a 200. Set false for a STRICT OpenAI-compatible endpoint + * (OpenAI itself) that rejects the unknown `stream` field and would otherwise + * fail every request. + */ + send_stream_flag?: boolean } export type VoiceProviderConfig = ElevenLabsProviderConfig | OpenAiCompatibleProviderConfig @@ -167,6 +175,10 @@ function asNumber(value: unknown): number | undefined { return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : undefined } +function asBoolean(value: unknown): boolean | undefined { + return typeof value === "boolean" ? value : undefined +} + /** * Turn the raw `[voice].providers` TOML value into a validated chain. * @@ -220,6 +232,7 @@ export function normalizeProviders(raw: unknown, log?: Logger): VoiceProviderCon timeout_ms: asNumber(record.timeout_ms), health_path: asString(record.health_path), health_timeout_ms: asNumber(record.health_timeout_ms), + send_stream_flag: asBoolean(record.send_stream_flag), }) return } @@ -375,23 +388,31 @@ export async function openAiCompatibleSynthesize(opts: { speed?: number apiKey?: string timeoutMs?: number + /** Send Kokoro's `stream: false`. Defaults to true; false omits the field + * entirely for a strict OpenAI endpoint that would reject it. */ + sendStreamFlag?: boolean }): Promise<{ audio: ArrayBuffer; format: string }> { const headers: Record = { "Content-Type": "application/json" } if (opts.apiKey) headers.Authorization = `Bearer ${opts.apiKey}` const requestedFormat = opts.responseFormat ?? DEFAULT_RESPONSE_FORMAT + const body: Record = { + model: opts.model ?? DEFAULT_OPENAI_MODEL, + input: opts.text, + voice: opts.voice ?? DEFAULT_OPENAI_VOICE, + response_format: requestedFormat, + speed: Number.isFinite(opts.speed) ? opts.speed : DEFAULT_SPEED, + } + // Kokoro needs stream:false to keep failures in the status code (see above); + // a strict OpenAI target sets send_stream_flag:false so the unknown field is + // omitted rather than 400'd. Default (undefined) keeps the Kokoro-safe field. + if (opts.sendStreamFlag !== false) body.stream = false + const response = await fetch(speechEndpoint(opts.baseUrl), { method: "POST", headers, - body: JSON.stringify({ - model: opts.model ?? DEFAULT_OPENAI_MODEL, - input: opts.text, - voice: opts.voice ?? DEFAULT_OPENAI_VOICE, - response_format: requestedFormat, - speed: Number.isFinite(opts.speed) ? opts.speed : DEFAULT_SPEED, - stream: false, - }), + body: JSON.stringify(body), signal: opts.timeoutMs ? AbortSignal.timeout(opts.timeoutMs) : undefined, }) @@ -469,6 +490,7 @@ export async function synthesizeViaChain( speed: ctx.elevenLabsSettings.speed, apiKey: provider.api_key, timeoutMs: provider.timeout_ms ?? DEFAULT_TIMEOUT_MS, + sendStreamFlag: provider.send_stream_flag, }) log?.("info", `Voice: provider[${i}] ${label} synthesized ${audio.byteLength} bytes as ${format}`) return { audio, format, provider: label } diff --git a/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts index 741d8e1001..2afe9135a2 100644 --- a/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts +++ b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/providers.test.ts @@ -389,6 +389,14 @@ describe("openAiCompatibleSynthesize", () => { expect(stub.requests[0].stream).toBe(false) }) + test("omits the stream field entirely when sendStreamFlag is false (strict OpenAI)", async () => { + const stub = startStub(() => audioResponse()) + + await openAiCompatibleSynthesize({ text: "hello", baseUrl: stub.url, sendStreamFlag: false }) + + expect("stream" in stub.requests[0]).toBe(false) + }) + test("sends a blend expression verbatim", async () => { const stub = startStub(() => audioResponse()) From a11c69556ac403803d8032d53525662551338b9c Mon Sep 17 00:00:00 2001 From: Daniel Date: Sat, 15 Aug 2026 12:25:35 -0700 Subject: [PATCH 6/6] fix(voice): format-aware audio player selection on Linux MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cross-vendor re-audit: once the chain can return wav/flac/opus/aac, the cached single global player breaks non-mp3 playback — mpg123 (MPEG-only) would be chosen for a wav body even with aplay installed. Player resolution now filters by format: ffplay (all formats) stays first, mpg123→mp3, paplay→wav/flac, aplay→wav. selectPlayer() is a pure, unit-tested seam; the PATH probe is cached once and selection runs per playback. The no-player warning now names the format, the present players, and recommends installing ffplay for full coverage. ffplay cannot be bundled, only preferred. --- .../install/LIFEOS/PULSE/VoiceServer/voice.ts | 130 +++++++++++------- .../PULSE/test/VoiceServer/voice.test.ts | 23 ++++ 2 files changed, 104 insertions(+), 49 deletions(-) diff --git a/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts b/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts index 4feeb46591..4a968a8f24 100644 --- a/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts +++ b/LifeOS/install/LIFEOS/PULSE/VoiceServer/voice.ts @@ -386,14 +386,35 @@ async function generateSpeechViaChain( // ── Audio Playback ── // Platform-aware audio player resolution. macOS ships afplay; Linux has no -// single standard, so try the common CLI players in order. ffplay/mpg123 handle -// the MP3 that ElevenLabs returns; paplay/aplay are last-resort fallbacks. -// Resolved once and cached. #1412 — hardcoding afplay ENOENT'd on Linux. +// single standard, so try the common CLI players in order. #1412 — hardcoding +// afplay ENOENT'd on Linux. +// +// Now the provider chain can return wav/flac/opus/aac, not just ElevenLabs mp3, +// player choice must account for the FORMAT: mpg123 plays only MPEG, aplay only +// WAV, so a cached mpg123 would fail a wav body even with aplay installed. +// ffplay handles every format and stays first; `supports` gates the narrower +// fallbacks. ffplay (the ffmpeg CLI) is the recommended install for full format +// coverage — we can only prefer it, not bundle it. interface AudioPlayer { path: string buildArgs: (file: string, volume: number) => string[] } +interface PlayerCandidate { + cmd: string + /** Formats this player can decode; `true` = anything we emit. */ + supports: true | ReadonlySet + buildArgs: (file: string, volume: number) => string[] +} + +/** First available player that can decode `format`. Pure — takes the resolved + * candidate list so it is unit-testable without probing PATH. */ +export function selectPlayer(available: PlayerCandidate[], format: string): PlayerCandidate | null { + return ( + available.find((p) => p.supports === true || p.supports.has(format)) ?? null + ) +} + // (public PR #1548, @m8ryx) `volume` is a multiplier where 1.0 is normal, // matching afplay's -v. Each Linux player expresses volume on its own integer // scale, so map onto that range and clamp. A non-finite or negative value falls @@ -403,52 +424,58 @@ function scaleVolume(volume: number, max: number): number { return Math.min(max, Math.round(volume * max)) } -let resolvedPlayer: AudioPlayer | null | undefined = undefined - -function resolveAudioPlayer(): AudioPlayer | null { - if (resolvedPlayer !== undefined) return resolvedPlayer - - const candidates: Array<{ cmd: string; buildArgs: (file: string, volume: number) => string[] }> = - process.platform === "darwin" - ? [{ cmd: "afplay", buildArgs: (file, volume) => ["-v", volume.toString(), file] }] - : [ - { - cmd: "ffplay", - // ffplay -h: "-volume volume set startup volume 0=min 100=max" - buildArgs: (file, volume) => [ - "-nodisp", - "-autoexit", - "-loglevel", - "quiet", - "-volume", - String(scaleVolume(volume, 100)), - file, - ], - }, - // mpg123 scales with -f, but its range was not verified here; left as-is - // rather than guessing a factor. - { cmd: "mpg123", buildArgs: (file) => ["-q", file] }, - { - cmd: "paplay", - // paplay --help: "--volume=VOLUME Specify the initial (linear) volume - // in range 0...65536" - buildArgs: (file, volume) => [`--volume=${scaleVolume(volume, 65536)}`, file], - }, - // aplay has no volume-set option (only --disable-softvol), so volume - // cannot be honoured on this fallback. - { cmd: "aplay", buildArgs: (file) => ["-q", file] }, - ] - - for (const c of candidates) { +// Per-player format capability, conservative to documented decoders: +// ffplay/afplay — everything we emit mpg123 — MPEG only +// paplay — libsndfile (wav, flac) aplay — WAV/PCM only +const PLAYER_CANDIDATES: PlayerCandidate[] = + process.platform === "darwin" + ? [{ cmd: "afplay", supports: true, buildArgs: (file, volume) => ["-v", volume.toString(), file] }] + : [ + { + cmd: "ffplay", + supports: true, + // ffplay -h: "-volume volume set startup volume 0=min 100=max" + buildArgs: (file, volume) => [ + "-nodisp", + "-autoexit", + "-loglevel", + "quiet", + "-volume", + String(scaleVolume(volume, 100)), + file, + ], + }, + // mpg123 scales with -f, but its range was not verified here; left as-is + // rather than guessing a factor. + { cmd: "mpg123", supports: new Set(["mp3"]), buildArgs: (file) => ["-q", file] }, + { + cmd: "paplay", + supports: new Set(["wav", "flac"]), + // paplay --help: "--volume=VOLUME Specify the initial (linear) volume + // in range 0...65536" + buildArgs: (file, volume) => [`--volume=${scaleVolume(volume, 65536)}`, file], + }, + // aplay has no volume-set option (only --disable-softvol), so volume + // cannot be honoured on this fallback. + { cmd: "aplay", supports: new Set(["wav"]), buildArgs: (file) => ["-q", file] }, + ] + +// Which candidate binaries are actually on PATH — the expensive probe, cached +// once. Selection by format is cheap and runs per playback. +let availableCandidates: PlayerCandidate[] | undefined = undefined + +function availablePlayers(): PlayerCandidate[] { + if (availableCandidates !== undefined) return availableCandidates + availableCandidates = PLAYER_CANDIDATES.flatMap((c) => { const path = Bun.which(c.cmd) - if (path) { - resolvedPlayer = { path, buildArgs: c.buildArgs } - return resolvedPlayer - } - } + return path ? [{ ...c, cmd: path }] : [] // resolve to the absolute path for spawn + }) + return availableCandidates +} - resolvedPlayer = null - return resolvedPlayer +function resolveAudioPlayer(format: string): AudioPlayer | null { + const chosen = selectPlayer(availablePlayers(), format) + return chosen ? { path: chosen.cmd, buildArgs: chosen.buildArgs } : null } // Serialize playback so concurrent /notify calls don't overlap on the speaker. @@ -467,10 +494,15 @@ async function playAudio( volume: number = FALLBACK_VOLUME, format = "mp3", ): Promise { - const player = resolveAudioPlayer() + const player = resolveAudioPlayer(format) if (!player) { + const present = availablePlayers().map((p) => p.cmd).join(", ") || "none" const tried = process.platform === "darwin" ? "afplay" : "ffplay/mpg123/paplay/aplay" - log("warn", `Voice: no audio player found (tried ${tried}) on ${process.platform} — skipping playback`) + log( + "warn", + `Voice: no installed player decodes ${format} on ${process.platform} — skipping playback. ` + + `Tried ${tried}; present: ${present}. Install ffplay (ffmpeg) for full format support.`, + ) return } diff --git a/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/voice.test.ts b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/voice.test.ts index 03e1d552d1..1c7d7edd4a 100644 --- a/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/voice.test.ts +++ b/LifeOS/install/LIFEOS/PULSE/test/VoiceServer/voice.test.ts @@ -36,12 +36,14 @@ const sandboxHome = mkdtempSync(join(tmpdir(), "lifeos-voice-test-")) let startVoice: typeof import("../../VoiceServer/voice").startVoice let voiceHealth: typeof import("../../VoiceServer/voice").voiceHealth let handleVoiceRequest: typeof import("../../VoiceServer/voice").handleVoiceRequest +let selectPlayer: typeof import("../../VoiceServer/voice").selectPlayer beforeAll(async () => { const mod = await import("../../VoiceServer/voice") startVoice = mod.startVoice voiceHealth = mod.voiceHealth handleVoiceRequest = mod.handleVoiceRequest + selectPlayer = mod.selectPlayer process.env.HOME = sandboxHome process.env.PATH = "" @@ -236,3 +238,24 @@ describe("provider chain configured", () => { expect(await response!.json()).toMatchObject({ status: "error", notification_sent: true }) }) }) + +describe("selectPlayer — format-aware fallback", () => { + const noop = () => [] + const ffplay = { cmd: "ffplay", supports: true as const, buildArgs: noop } + const mpg123 = { cmd: "mpg123", supports: new Set(["mp3"]), buildArgs: noop } + const aplay = { cmd: "aplay", supports: new Set(["wav"]), buildArgs: noop } + + test("ffplay wins for any format when present", () => { + expect(selectPlayer([ffplay, mpg123, aplay], "opus")?.cmd).toBe("ffplay") + expect(selectPlayer([ffplay, mpg123, aplay], "mp3")?.cmd).toBe("ffplay") + }) + + test("a wav body skips the MPEG-only mpg123 for aplay", () => { + // The exact audit case: no ffplay, mpg123 first but incompatible. + expect(selectPlayer([mpg123, aplay], "wav")?.cmd).toBe("aplay") + }) + + test("returns null when no available player decodes the format", () => { + expect(selectPlayer([mpg123], "flac")).toBeNull() + }) +})