From 29ba6ac993f8e16e305b306ab340cee101a7df59 Mon Sep 17 00:00:00 2001 From: vi Date: Thu, 16 Jul 2026 01:49:54 +0300 Subject: [PATCH 1/2] fix: correct reliability routing hint to heartbeat key-health share The Reliability column tooltip described stale traffic-vs-errors semantics. Server-side reliability is computed as heartbeat key-health share (proportionHealthyKeys), independent of real traffic. Rewrite the hint to match, drop the dead reliabilityWithHeartbeat property, and update the test assertion. --- client/src/lib/routing-explanations.test.ts | 9 +-------- client/src/lib/routing-explanations.ts | 2 -- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/client/src/lib/routing-explanations.test.ts b/client/src/lib/routing-explanations.test.ts index 9ff51545f..b60f6a792 100644 --- a/client/src/lib/routing-explanations.test.ts +++ b/client/src/lib/routing-explanations.test.ts @@ -28,18 +28,11 @@ describe("ROUTING_EXPLANATIONS", () => { it("documents routing score axes", () => { expect(ROUTING_EXPLANATIONS.score).toContain("higher score routes first"); - expect(ROUTING_EXPLANATIONS.reliability).toContain("successful requests"); + expect(ROUTING_EXPLANATIONS.reliability).toContain("healthy"); expect(ROUTING_EXPLANATIONS.speed).toContain("tokens per second"); expect(ROUTING_EXPLANATIONS.intelligence).toContain("benchmark score"); }); - it("documents heartbeat-toggled reliability as key-health share", () => { - expect(ROUTING_EXPLANATIONS.reliabilityWithHeartbeat).toContain( - "heartbeat", - ); - expect(ROUTING_EXPLANATIONS.reliabilityWithHeartbeat).toContain("keys"); - }); - it("documents benchmark chart semantics", () => { expect(ROUTING_EXPLANATIONS.benchmarkChart).toContain( "benchmarked request latency", diff --git a/client/src/lib/routing-explanations.ts b/client/src/lib/routing-explanations.ts index 5a7d71ecc..3daf14810 100644 --- a/client/src/lib/routing-explanations.ts +++ b/client/src/lib/routing-explanations.ts @@ -8,8 +8,6 @@ export const ROUTING_EXPLANATIONS = { score: "Final weighted score across reliability, speed, intelligence, latency, boost, and penalties. A higher score routes first for score-based strategies.", reliability: - "Reliability comes from successful requests versus errors, rate limits, cooldowns, and degradation state. More clean traffic raises this axis.", - reliabilityWithHeartbeat: "Reliability is the share of API keys the heartbeat currently marks healthy for this model — cheap health-check pings cycled per key, independent of real traffic. A model with no live errors can still score low if its keys are down, and vice-versa.", speed: "Speed is output plus reasoning tokens per second from observed traffic. It answers how quickly useful tokens arrive after the first byte.", From c0adda37092071a172273f1593463a13e0df7917 Mon Sep 17 00:00:00 2001 From: vi Date: Thu, 16 Jul 2026 10:19:05 +0300 Subject: [PATCH 2/2] =?UTF-8?q?refactor:=20prune=20dead=20isHeartbeatEnabl?= =?UTF-8?q?ed=20branches=20=E2=80=94=20heartbeat=20is=20always=20on?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Heartbeat is now permanently enabled (team policy — heartbeat_enabled defaults true and is no longer toggled). Every isHeartbeatEnabled() call resolved to true, so all disabled-mode branches were dead code. - Remove the isHeartbeatEnabled() toggle and ~30 dead branches across heartbeat.ts, router.ts, proxy.ts, responses.ts, racing-mode.ts, health.ts - Collapse the reliability-scoring fork in router.ts to a single heartbeatReliability(platform, modelId) / 100 line (Beta-posterior fallbacks were unreachable) - Delete orphaned reliabilityPosteriorForModel export (model-priors.ts) - Drop the phantom heartbeat_enabled toggle + 11 parentToggle references in feature-settings.ts - Clean up tests: remove disabled-behavior assertions, drop isHeartbeatEnabled from test mocks, add missing getKeyStats mock export in key-affinity.test.ts Zero behavior change on the live (heartbeat-on) path. Cold keys now consistently report isKeyHealthy === false, matching the existing proportionHealthyKeys docstring. --- .../integration/key-affinity.test.ts | 4 +- .../src/__tests__/services/heartbeat.test.ts | 76 --------- .../__tests__/services/racing-mode.test.ts | 1 - .../__tests__/services/router-bandit.test.ts | 161 ------------------ .../services/router-strategy-override.test.ts | 2 - server/src/__tests__/services/router.test.ts | 34 ---- .../services/routing-exhaustion.test.ts | 7 +- server/src/routes/health.ts | 1 - server/src/routes/proxy.ts | 119 ++++++------- server/src/routes/responses.ts | 131 +++++++------- server/src/services/degradation.ts | 4 +- server/src/services/feature-settings.ts | 23 +-- server/src/services/heartbeat.ts | 30 +--- server/src/services/model-priors.ts | 19 --- server/src/services/racing-mode.ts | 5 +- server/src/services/router.ts | 69 ++------ 16 files changed, 144 insertions(+), 542 deletions(-) diff --git a/server/src/__tests__/integration/key-affinity.test.ts b/server/src/__tests__/integration/key-affinity.test.ts index e7850a1b5..fb56d7eb7 100644 --- a/server/src/__tests__/integration/key-affinity.test.ts +++ b/server/src/__tests__/integration/key-affinity.test.ts @@ -8,11 +8,11 @@ import { type RouteOptions, routeRequest } from "../../services/router.js"; // Mock heartbeat module to simplify testing vi.mock("../../services/heartbeat.js", () => ({ + heartbeatReliability: vi.fn(() => 100), // 100% healthy when not pre-configured isKeyHealthy: vi.fn(() => true), // All keys healthy by default - isHeartbeatEnabled: vi.fn(() => false), // Heartbeat disabled by default - markKeyUnhealthy: vi.fn(), recordActivity: vi.fn(), getAllKeyHealth: vi.fn(() => new Map()), + getKeyStats: vi.fn(), getKeyHealth: vi.fn(), })); diff --git a/server/src/__tests__/services/heartbeat.test.ts b/server/src/__tests__/services/heartbeat.test.ts index 15b839866..201b49b8e 100644 --- a/server/src/__tests__/services/heartbeat.test.ts +++ b/server/src/__tests__/services/heartbeat.test.ts @@ -336,15 +336,6 @@ describe("Provider Health Heartbeat", () => { expect(isKeyHealthy(999)).toBe(false); }); - it("isKeyHealthy returns true for a cold key when heartbeat is disabled (backward compat)", () => { - // Disable heartbeat - setSetting("heartbeat_enabled", "false"); - resetHeartbeatConfig(); - - // When heartbeat is off, cold keys are assumed healthy for backward compat - expect(isKeyHealthy(999)).toBe(true); - }); - it("isKeyHealthy returns true after a successful warmup ping", async () => { const { modelDbId, keyId } = setupProvider(); @@ -608,13 +599,6 @@ describe("Provider Health Heartbeat", () => { // ── Lifecycle ────────────────────────────────────────────────────────── describe("Lifecycle (start/stop)", () => { - it("startHeartbeat is a no-op when disabled", () => { - setSetting("heartbeat_enabled", "false"); - startHeartbeat(); - vi.advanceTimersByTime(30 * 60 * 1000); - expect(publishedEvents.length).toBe(0); - }); - it("stopHeartbeat is safe to call even if never started", () => { expect(() => stopHeartbeat()).not.toThrow(); }); @@ -754,14 +738,6 @@ describe("Provider Health Heartbeat", () => { // ── resetHeartbeatConfig ───────────────────────────────────────────── describe("pokeKey", () => { - it("returns true when heartbeat is disabled (backward compat)", async () => { - setSetting("heartbeat_enabled", "false"); - resetHeartbeatConfig(); - - const result = await pokeKey(1); - expect(result).toBe(true); - }); - it("pings a specific key and returns true when healthy", async () => { const { keyId } = setupProvider(); @@ -1186,58 +1162,6 @@ describe("Provider Health Heartbeat", () => { getPendingRechecks().get(healthKey(keyId, "test-model"))!.attempt, ).toBe(1); }); - - it("disabled heartbeat → no recheck scheduled", async () => { - // Need a fresh module with heartbeat disabled - vi.resetModules(); - vi.doMock("../../providers/index.js", async (importOriginal) => { - const actual = (await importOriginal()) as any; - return { - ...actual, - buildProviderFor: () => ({ name: "fake", chatCompletion: vi.fn() }), - }; - }); - vi.doMock("../../services/events.js", () => ({ - publish: vi.fn(), - })); - vi.doMock("../../lib/crypto.js", async (importOriginal) => { - const actual = (await importOriginal()) as any; - return { ...actual, decrypt: vi.fn(() => "mocked-api-key") }; - }); - - process.env.ENCRYPTION_KEY = "0".repeat(64); - const freshDb = await import("../../db/index.js"); - const freshHb = await import("../../services/heartbeat.js"); - const freshDegr = await import("../../services/degradation.js"); - - freshDb.initDb(":memory:"); - freshDegr.initDegradation(); - freshDb.setSetting("heartbeat_enabled", "false"); - freshHb.resetHeartbeatConfig(); - - // Setup provider in fresh DB - const db = freshDb.getDb(); - db.prepare( - "INSERT INTO models (platform, model_id, display_name, intelligence_rank, speed_rank, enabled) VALUES ('testprov', 'test-model', 'Test', 1, 1, 1)", - ).run(); - const id = ( - db - .prepare("SELECT id FROM models WHERE platform = 'testprov'") - .get() as any - ).id; - db.prepare( - "INSERT INTO fallback_config (model_db_id, priority, enabled) VALUES (?, 1, 1)", - ).run(id); - db.prepare( - "INSERT INTO api_keys (platform, label, encrypted_key, iv, auth_tag, status, enabled) VALUES ('testprov', 'Key 1', 'enc', 'iv', 'tag', 'healthy', 1)", - ).run(); - const keyRow = db - .prepare("SELECT id FROM api_keys WHERE platform = ? AND enabled = 1") - .get("testprov") as any; - - freshHb.markKeyUnhealthy(keyRow.id, "test-model", "429 rate limit"); - expect(freshHb.getPendingRechecks().size).toBe(0); - }); }); // ── Recheck Execution ────────────────────────────────────────────────── diff --git a/server/src/__tests__/services/racing-mode.test.ts b/server/src/__tests__/services/racing-mode.test.ts index 5db27fd45..9861ad6c8 100644 --- a/server/src/__tests__/services/racing-mode.test.ts +++ b/server/src/__tests__/services/racing-mode.test.ts @@ -39,7 +39,6 @@ vi.mock("../../services/ratelimit.js", () => ({ })); vi.mock("../../services/heartbeat.js", () => ({ - isHeartbeatEnabled: () => false, markKeyTrafficHealthy: () => {}, recordActivity: mocks.recordActivity, })); diff --git a/server/src/__tests__/services/router-bandit.test.ts b/server/src/__tests__/services/router-bandit.test.ts index 7b7ce9a77..432b33220 100644 --- a/server/src/__tests__/services/router-bandit.test.ts +++ b/server/src/__tests__/services/router-bandit.test.ts @@ -188,7 +188,6 @@ describe("bandit router", () => { getDb().exec( "DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests;", ); - setSetting("heartbeat_enabled", "false"); resetHeartbeatConfig(); vi.clearAllMocks(); (crypto.decrypt as any).mockReturnValue("mocked-api-key"); @@ -221,34 +220,6 @@ describe("bandit router", () => { }); }); - it("applies configured coding-agent model priors before live observations", () => { - addModel({ - platform: "nvidia", - modelId: "z-ai/glm-5.2", - name: "GLM-5.2", - intelligenceRank: 1, - sizeLabel: "Frontier", - budget: "~10M", - priority: 1, - }); - addModel({ - platform: "example", - modelId: "unlisted", - name: "Unlisted", - intelligenceRank: 1, - sizeLabel: "Frontier", - budget: "~10M", - priority: 2, - }); - - const scores = getRoutingScores().scores; - const configured = scores.find((s) => s.modelId === "z-ai/glm-5.2"); - const unlisted = scores.find((s) => s.modelId === "unlisted"); - - expect(configured?.reliability).toBeCloseTo(0.3995, 4); - expect(unlisted?.reliability).toBeCloseTo(0.5, 4); - }); - it("attaches aggressive per-arm timeouts to volatile StepFun routes", () => { setRoutingStrategy("priority"); addModel({ @@ -289,33 +260,6 @@ describe("bandit router", () => { expect(counts["a"]).toBe(50); // priority 1 always wins regardless of intelligence }); - it("balanced strategy favors the more reliable model", () => { - addModel({ - platform: "google", - modelId: "good", - name: "Good", - intelligenceRank: 3, - sizeLabel: "Large", - budget: "~50M", - priority: 1, - }); - addModel({ - platform: "groq", - modelId: "flaky", - name: "Flaky", - intelligenceRank: 3, - sizeLabel: "Large", - budget: "~50M", - priority: 2, - }); - addHistory("google", "good", { successes: 60, failures: 1 }); - addHistory("groq", "flaky", { successes: 5, failures: 40 }); - setRoutingStrategy("balanced"); - refreshStatsCache(getDb(), true); - const counts = pickCounts(300); - expect(counts["good"] ?? 0).toBeGreaterThan((counts["flaky"] ?? 0) * 3); - }); - it("uses heartbeat health proportion for flat-chain reliability when heartbeat is enabled", () => { addModel({ platform: "healthy-provider", @@ -344,7 +288,6 @@ describe("bandit router", () => { failures: 0, }); - setSetting("heartbeat_enabled", "true"); resetHeartbeatConfig(); setHeartbeatState("healthy-provider", "historically-bad", true); setHeartbeatState("sick-provider", "historically-good", false); @@ -358,75 +301,6 @@ describe("bandit router", () => { expect(byModel.get("historically-good")!.reliability).toBe(0); }); - it("falls back to historical reliability when heartbeat is disabled", () => { - addModel({ - platform: "healthy-provider", - modelId: "historically-bad", - name: "Historically Bad", - intelligenceRank: 3, - sizeLabel: "Large", - budget: "~50M", - priority: 1, - }); - addModel({ - platform: "sick-provider", - modelId: "historically-good", - name: "Historically Good", - intelligenceRank: 3, - sizeLabel: "Large", - budget: "~50M", - priority: 2, - }); - addHistory("healthy-provider", "historically-bad", { - successes: 1, - failures: 40, - }); - addHistory("sick-provider", "historically-good", { - successes: 60, - failures: 0, - }); - - setSetting("heartbeat_enabled", "false"); - resetHeartbeatConfig(); - setHeartbeatState("healthy-provider", "historically-bad", true); - setHeartbeatState("sick-provider", "historically-good", false); - setRoutingStrategy("balanced"); - refreshStatsCache(getDb(), true); - - const byModel = new Map( - getRoutingScores().scores.map((score) => [score.modelId, score]), - ); - expect(byModel.get("historically-good")!.reliability).toBeGreaterThan( - byModel.get("historically-bad")!.reliability, - ); - }); - - it("explores unseen models — both get picked at least once", () => { - addModel({ - platform: "google", - modelId: "x", - name: "X", - intelligenceRank: 3, - sizeLabel: "Large", - budget: "~50M", - priority: 1, - }); - addModel({ - platform: "groq", - modelId: "y", - name: "Y", - intelligenceRank: 3, - sizeLabel: "Large", - budget: "~50M", - priority: 2, - }); - setRoutingStrategy("balanced"); - refreshStatsCache(getDb(), true); - const counts = pickCounts(200); - expect(counts["x"] ?? 0).toBeGreaterThan(0); - expect(counts["y"] ?? 0).toBeGreaterThan(0); - }); - it("smartest vs fastest flips which model wins, at equal reliability", () => { // Smart: frontier tier, slow (high speed_rank = slow default). Fast: small tier, fast (low speed_rank). // Both get 60+2=62 requests, well above REAL_SPEED_CONFIDENCE_THRESHOLD (50), @@ -511,7 +385,6 @@ describe("bandit router", () => { addBenchmarkSamples(nearModelId, "groq", "near-slow-work", 520, 60); addPingSamples("groq", "near-slow-work", 20, 60); - setSetting("heartbeat_enabled", "true"); resetHeartbeatConfig(); setHeartbeatState("google", "far-fast-work", true); setHeartbeatState("groq", "near-slow-work", true); @@ -689,38 +562,4 @@ describe("bandit router", () => { latency: 0.1, }); }); - - it("getRoutingScores returns a per-axis breakdown ranked by score", () => { - addModel({ - platform: "google", - modelId: "m1", - name: "M1", - intelligenceRank: 1, - sizeLabel: "Frontier", - budget: "~50M", - priority: 1, - }); - addHistory("google", "m1", { - successes: 30, - failures: 0, - outTokens: 500, - latencyMs: 1000, - ttfbMs: 200, - }); - setRoutingStrategy("balanced"); - refreshStatsCache(getDb(), true); - const { strategy, weights, scores } = getRoutingScores(); - expect(strategy).toBe("balanced"); - expect(weights).toEqual({ - reliability: 0.4, - speed: 0.2, - intelligence: 0.2, - latency: 0.2, - }); - expect(scores).toHaveLength(1); - expect(scores[0]).toMatchObject({ modelId: "m1", enabled: true }); - expect(scores[0].reliability).toBeGreaterThan(0.9); - expect(scores[0].score).toBeGreaterThan(0); - expect(scores[0].score).toBeLessThanOrEqual(1); - }); }); diff --git a/server/src/__tests__/services/router-strategy-override.test.ts b/server/src/__tests__/services/router-strategy-override.test.ts index 44daa2e1c..e7211e519 100644 --- a/server/src/__tests__/services/router-strategy-override.test.ts +++ b/server/src/__tests__/services/router-strategy-override.test.ts @@ -58,8 +58,6 @@ describe("routeRequest(options.strategy) override", () => { getDb().exec( "DELETE FROM fallback_config; DELETE FROM api_keys; DELETE FROM models; DELETE FROM requests;", ); - setSetting("heartbeat_enabled", "false"); - resetHeartbeatConfig(); vi.clearAllMocks(); // Deterministic disagreement across strategies: diff --git a/server/src/__tests__/services/router.test.ts b/server/src/__tests__/services/router.test.ts index 7cb73f595..7a1f366d3 100644 --- a/server/src/__tests__/services/router.test.ts +++ b/server/src/__tests__/services/router.test.ts @@ -35,7 +35,6 @@ describe("Router", () => { // These cases assert the manual priority order specifically; pin it so the // bandit (now the default strategy) doesn't reorder by score. setRoutingStrategy("priority"); - setSetting("heartbeat_enabled", "false"); resetHeartbeatConfig(); keyHealthMap.clear(); db.prepare("DELETE FROM api_keys").run(); @@ -257,37 +256,6 @@ describe("Router", () => { expect(result.platform).toBe("groq"); }); - it("routes heartbeat-disabled keys even when stale health marks them unhealthy", () => { - const db = getDb(); - const platform = "heartbeat-off-stale"; - const modelId = "stale-model"; - addContextAwareModel({ - platform, - modelId, - displayName: "Stale Health Model", - priority: 1, - contextWindow: 100000, - }); - const key = db - .prepare("SELECT id FROM api_keys WHERE platform = ?") - .get(platform) as { id: number }; - setSetting("heartbeat_enabled", "false"); - resetHeartbeatConfig(); - keyHealthMap.set(healthKey(key.id, modelId), { - penalty: 1, - lastPingAt: Date.now(), - healthy: false, - lastError: "stale failure", - }); - - const result = routeRequest(100); - - expect(result.platform).toBe(platform); - expect(result.modelId).toBe(modelId); - expect(result.keyId).toBe(key.id); - result.release(); - }); - it("skips a model whose context window cannot hold the request (#167)", () => { const db = getDb(); const groqKey = encrypt("test-groq-key"); @@ -511,7 +479,6 @@ describe("Router", () => { expect(cold).toBeDefined(); expect(healthy).toBeDefined(); if (!cold || !healthy) throw new Error("missing racing test keys"); - setSetting("heartbeat_enabled", "true"); resetHeartbeatConfig(); keyHealthMap.set(healthKey(healthy.id, modelId), { penalty: 0, @@ -548,7 +515,6 @@ describe("Router", () => { describe("routeRacingRequest", () => { it("should include heartbeat-unhealthy keys (Tier 2) instead of returning empty when all keys are unhealthy", () => { - setSetting("heartbeat_enabled", "true"); const platform = "racing-tier2-all-unhealthy"; const modelId = "racing-model-2"; addContextAwareModel({ diff --git a/server/src/__tests__/services/routing-exhaustion.test.ts b/server/src/__tests__/services/routing-exhaustion.test.ts index 10f52539b..7acb013c6 100644 --- a/server/src/__tests__/services/routing-exhaustion.test.ts +++ b/server/src/__tests__/services/routing-exhaustion.test.ts @@ -4,16 +4,14 @@ import * as crypto from "../../lib/crypto.js"; import * as heartbeat from "../../services/heartbeat.js"; import { routeRequest, setRoutingStrategy } from "../../services/router.js"; -// Mock heartbeat to control key health ordering. We also mock -// isHeartbeatEnabled() so unmocked routeRequest() calls exercise the -// healthy-key filter path regardless of the heartbeat_enabled DB setting. +// Mock heartbeat to control key health ordering so unmocked routeRequest() +// calls exercise the healthy-key filter path with deterministic results. vi.mock("../../services/heartbeat.js", async () => { const actual = await vi.importActual("../../services/heartbeat.js"); return { ...actual, isKeyHealthy: vi.fn(() => true), getKeyHealth: vi.fn(() => undefined), - isHeartbeatEnabled: vi.fn(() => true), }; }); @@ -97,7 +95,6 @@ describe("Routing Key Exhaustion", () => { vi.mocked(crypto.decrypt).mockReturnValue("mocked-api-key"); vi.mocked(heartbeat.isKeyHealthy).mockReturnValue(true); vi.mocked(heartbeat.getKeyHealth).mockReturnValue(undefined); - vi.mocked(heartbeat.isHeartbeatEnabled).mockReturnValue(true); }); afterEach(() => { diff --git a/server/src/routes/health.ts b/server/src/routes/health.ts index 03bb24328..60161e696 100644 --- a/server/src/routes/health.ts +++ b/server/src/routes/health.ts @@ -3,7 +3,6 @@ import { Router } from "express"; import { getDb } from "../db/index.js"; import { hasProvider } from "../providers/index.js"; import { checkAllKeys, checkKeyHealth } from "../services/health.js"; -import { getKeyHealth, isHeartbeatEnabled } from "../services/heartbeat.js"; export const healthRouter = Router(); diff --git a/server/src/routes/proxy.ts b/server/src/routes/proxy.ts index 440ecf71e..0d6d88046 100644 --- a/server/src/routes/proxy.ts +++ b/server/src/routes/proxy.ts @@ -37,7 +37,6 @@ import { EmbeddingsError, runEmbeddings } from "../services/embeddings.js"; import { publish } from "../services/events.js"; import { getFeatureSetting } from "../services/feature-settings.js"; import { - isHeartbeatEnabled, markKeyTrafficHealthy, markKeyUnhealthy, recordActivity, @@ -1622,8 +1621,7 @@ proxyRouter.post("/chat/completions", async (req: Request, res: Response) => { promptTokens + completionTokens, ); recordSuccess(stepRoute.modelDbId); - if (isHeartbeatEnabled()) - markKeyTrafficHealthy(stepRoute.keyId, stepRoute.modelId); + markKeyTrafficHealthy(stepRoute.keyId, stepRoute.modelId); logRequest( stepRoute.platform, stepRoute.modelId, @@ -1663,8 +1661,7 @@ proxyRouter.post("/chat/completions", async (req: Request, res: Response) => { promptTokens + completionTokens, ); recordSuccess(stepRoute.modelDbId); - if (isHeartbeatEnabled()) - markKeyTrafficHealthy(stepRoute.keyId, stepRoute.modelId); + markKeyTrafficHealthy(stepRoute.keyId, stepRoute.modelId); logRequest( stepRoute.platform, stepRoute.modelId, @@ -2076,8 +2073,7 @@ proxyRouter.post("/chat/completions", async (req: Request, res: Response) => { totalTokens, ); recordSuccess(stepRoute.modelDbId); - if (isHeartbeatEnabled()) - markKeyTrafficHealthy(stepRoute.keyId, stepRoute.modelId); + markKeyTrafficHealthy(stepRoute.keyId, stepRoute.modelId); logRequest( stepRoute.platform, stepRoute.modelId, @@ -2117,8 +2113,7 @@ proxyRouter.post("/chat/completions", async (req: Request, res: Response) => { totalTokens, ); recordSuccess(stepRoute.modelDbId); - if (isHeartbeatEnabled()) - markKeyTrafficHealthy(stepRoute.keyId, stepRoute.modelId); + markKeyTrafficHealthy(stepRoute.keyId, stepRoute.modelId); logRequest( stepRoute.platform, stepRoute.modelId, @@ -2787,8 +2782,7 @@ proxyRouter.post("/chat/completions", async (req: Request, res: Response) => { estimatedInputTokens + injectedHandoffTokens + totalOutputTokens, ); recordSuccess(route.modelDbId); - if (isHeartbeatEnabled()) - markKeyTrafficHealthy(route.keyId, route.modelId); + markKeyTrafficHealthy(route.keyId, route.modelId); captureRequestSample({ modelDbId: route.modelDbId, messages, @@ -2969,8 +2963,7 @@ proxyRouter.post("/chat/completions", async (req: Request, res: Response) => { recordRequest(route.platform, route.modelId, route.keyId); recordTokens(route.platform, route.modelId, route.keyId, totalTokens); recordSuccess(route.modelDbId); - if (isHeartbeatEnabled()) - markKeyTrafficHealthy(route.keyId, route.modelId); + markKeyTrafficHealthy(route.keyId, route.modelId); captureRequestSample({ modelDbId: route.modelDbId, messages, @@ -3146,62 +3139,58 @@ proxyRouter.post("/chat/completions", async (req: Request, res: Response) => { // heartbeat is enabled, mark it unhealthy so the router // excludes it from the healthy pool. Only a successful ping // can restore it. - if (isHeartbeatEnabled()) { - const reason = isPaymentRequired - ? "payment_required" - : isDailyExhausted - ? "daily_exhausted" - : "rate_limited"; - const recheckDelayMs = - isDailyExhausted || isPaymentRequired - ? computeRetryCooldownMs( - isPaymentRequired, - route.platform, - route.modelId, - route.keyId, - { rpd: route.rpdLimit, tpd: route.tpdLimit }, - lastError?.retryAfterMs, - ) - : undefined; // transient uses default recheckSec - markKeyUnhealthy( - route.keyId, - route.modelId, - err?.message?.slice(0, 120) ?? "error", - isTransient, - recheckDelayMs, - ); - publish({ - type: isTransient ? "routing.key_transient" : "routing.key_evicted", - id: requestId, - provider: route.platform, - keyId: route.keyId, - model: route.modelId, - reason, - at: Date.now(), - }); - } + const reason = isPaymentRequired + ? "payment_required" + : isDailyExhausted + ? "daily_exhausted" + : "rate_limited"; + const recheckDelayMs = + isDailyExhausted || isPaymentRequired + ? computeRetryCooldownMs( + isPaymentRequired, + route.platform, + route.modelId, + route.keyId, + { rpd: route.rpdLimit, tpd: route.tpdLimit }, + lastError?.retryAfterMs, + ) + : undefined; // transient uses default recheckSec + markKeyUnhealthy( + route.keyId, + route.modelId, + err?.message?.slice(0, 120) ?? "error", + isTransient, + recheckDelayMs, + ); + publish({ + type: isTransient ? "routing.key_transient" : "routing.key_evicted", + id: requestId, + provider: route.platform, + keyId: route.keyId, + model: route.modelId, + reason, + at: Date.now(), + }); lastError = err; } else { // Non-retryable error (auth, 4xx, etc.): also evict from // healthy pool when heartbeat is enabled, then skip key. - if (isHeartbeatEnabled()) { - markKeyUnhealthy( - route.keyId, - route.modelId, - err?.message?.slice(0, 120) ?? "auth error", - false, - undefined, - ); - publish({ - type: "routing.key_evicted", - id: requestId, - provider: route.platform, - keyId: route.keyId, - model: route.modelId, - reason: "auth_error", - at: Date.now(), - }); - } + markKeyUnhealthy( + route.keyId, + route.modelId, + err?.message?.slice(0, 120) ?? "auth error", + false, + undefined, + ); + publish({ + type: "routing.key_evicted", + id: requestId, + provider: route.platform, + keyId: route.keyId, + model: route.modelId, + reason: "auth_error", + at: Date.now(), + }); lastError = err; } } finally { diff --git a/server/src/routes/responses.ts b/server/src/routes/responses.ts index ccec762c5..bef4fa490 100644 --- a/server/src/routes/responses.ts +++ b/server/src/routes/responses.ts @@ -26,7 +26,6 @@ import { } from "../services/degradation.js"; import { getFeatureSetting } from "../services/feature-settings.js"; import { - isHeartbeatEnabled, markKeyTrafficHealthy, markKeyUnhealthy, } from "../services/heartbeat.js"; @@ -658,14 +657,12 @@ async function handleResponsesStream(opts: { ), ); recordFailure(route.modelDbId, "minor"); - if (isHeartbeatEnabled()) { - markKeyUnhealthy( - route.keyId, - route.modelId, - "model skip: unparseable dialect", - true, - ); - } + markKeyUnhealthy( + route.keyId, + route.modelId, + "model skip: unparseable dialect", + true, + ); lastError.value = new Error( `unparseable inline tool-call dialect from ${route.displayName}`, ); @@ -732,14 +729,12 @@ async function handleResponsesStream(opts: { ), ); recordFailure(route.modelDbId, "minor"); - if (isHeartbeatEnabled()) { - markKeyUnhealthy( - route.keyId, - route.modelId, - "model skip: empty completion", - true, - ); - } + markKeyUnhealthy( + route.keyId, + route.modelId, + "model skip: empty completion", + true, + ); lastError.value = new Error(`empty completion from ${route.displayName}`); return false; // retry } @@ -817,7 +812,7 @@ async function handleResponsesStream(opts: { estimatedInputTokens + accCapturedOutputTokens, ); recordSuccess(route.modelDbId); - if (isHeartbeatEnabled()) markKeyTrafficHealthy(route.keyId, route.modelId); + markKeyTrafficHealthy(route.keyId, route.modelId); setStickyModel(messages, route.modelDbId, sessionIdHeader); logRequest( route.platform, @@ -947,14 +942,12 @@ async function handleResponsesCompletion(opts: { ), ); recordFailure(route.modelDbId, "minor"); - if (isHeartbeatEnabled()) { - markKeyUnhealthy( - route.keyId, - route.modelId, - "model skip: empty completion", - true, - ); - } + markKeyUnhealthy( + route.keyId, + route.modelId, + "model skip: empty completion", + true, + ); lastError.value = new Error(`empty completion from ${route.displayName}`); return false; // retry (loop continues) } @@ -967,7 +960,7 @@ async function handleResponsesCompletion(opts: { result.usage?.total_tokens ?? 0, ); recordSuccess(route.modelDbId); - if (isHeartbeatEnabled()) markKeyTrafficHealthy(route.keyId, route.modelId); + markKeyTrafficHealthy(route.keyId, route.modelId); setStickyModel(messages, route.modelDbId, sessionIdHeader); res.setHeader("X-Routed-Via", `${route.platform}/${route.modelId}`); @@ -1079,56 +1072,50 @@ async function handleResponsesError(opts: { const respTier = classifyError(err); if (respTier) recordFailure(route.modelDbId, respTier); - if (isHeartbeatEnabled()) { - const isDailyExhausted = isDailyLimitExhausted( - route.platform, - route.modelId, - route.keyId, - { rpd: route.rpdLimit, tpd: route.tpdLimit }, - ); - const isPaymentRequired = isPaymentRequiredError(err); - const isTransient = - classifyError(err) === "minor" && - !isPaymentRequired && - !isDailyExhausted; - const reason = isPaymentRequired - ? "payment_required" - : isDailyExhausted - ? "daily_exhausted" - : "rate_limited"; - const recheckDelayMs = - isDailyExhausted || isPaymentRequired - ? computeRetryCooldownMs( - isPaymentRequired, - route.platform, - route.modelId, - route.keyId, - { rpd: route.rpdLimit, tpd: route.tpdLimit }, - err?.retryAfterMs, - ) - : undefined; - markKeyUnhealthy( - route.keyId, - route.modelId, - err?.message?.slice(0, 120) ?? "error", - isTransient, - recheckDelayMs, - ); - } - lastError.value = err; - return "retry"; - } - - // Non-retryable error - if (isHeartbeatEnabled()) { + const isDailyExhausted = isDailyLimitExhausted( + route.platform, + route.modelId, + route.keyId, + { rpd: route.rpdLimit, tpd: route.tpdLimit }, + ); + const isPaymentRequired = isPaymentRequiredError(err); + const isTransient = + classifyError(err) === "minor" && !isPaymentRequired && !isDailyExhausted; + const reason = isPaymentRequired + ? "payment_required" + : isDailyExhausted + ? "daily_exhausted" + : "rate_limited"; + const recheckDelayMs = + isDailyExhausted || isPaymentRequired + ? computeRetryCooldownMs( + isPaymentRequired, + route.platform, + route.modelId, + route.keyId, + { rpd: route.rpdLimit, tpd: route.tpdLimit }, + err?.retryAfterMs, + ) + : undefined; markKeyUnhealthy( route.keyId, route.modelId, - err?.message?.slice(0, 120) ?? "auth error", - false, - undefined, + err?.message?.slice(0, 120) ?? "error", + isTransient, + recheckDelayMs, ); + lastError.value = err; + return "retry"; } + + // Non-retryable error + markKeyUnhealthy( + route.keyId, + route.modelId, + err?.message?.slice(0, 120) ?? "auth error", + false, + undefined, + ); res.status(502).json({ error: { message: `Provider error (${route.displayName}): ${safeError}`, diff --git a/server/src/services/degradation.ts b/server/src/services/degradation.ts index 923ff42f7..4c654bcc3 100644 --- a/server/src/services/degradation.ts +++ b/server/src/services/degradation.ts @@ -397,8 +397,8 @@ export function recordSuccess(modelDbId: number): void { // memory even with penalty 0. Deleting it would silently clear the // latch — the decay path (sub-threshold penalty) can race ahead // of the success counter if cheap heartbeat "hi" pings keep - // succeeding but never clear the latch. - if (state.penalty <= 0 && !state.hardCooled) { + // succeeding but never clear the latch. + if (state.penalty <= 0 && !state.hardCooled) { degradationStates.delete(modelDbId); } else { state.dirty = true; diff --git a/server/src/services/feature-settings.ts b/server/src/services/feature-settings.ts index 9cf4c5073..a5209949e 100644 --- a/server/src/services/feature-settings.ts +++ b/server/src/services/feature-settings.ts @@ -63,17 +63,7 @@ export const REGISTRY: FeatureSettingDef[] = [ group: "Resilience", parentToggle: "provider_fastfail_enabled", }, - { - key: "heartbeat_enabled", - label: "Provider Health Heartbeat", - description: - "Send periodic health-check pings to each provider. Feeds the degradation engine so the router avoids sick providers proactively.", - type: "boolean", - default: true, - envVar: "HEARTBEAT_ENABLED", - effect: "restart", - group: "Resilience", - }, + { key: "heartbeat_interval_min", label: "Heartbeat Interval", @@ -85,7 +75,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_INTERVAL_MIN", effect: "restart", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_timeout_ms", @@ -99,7 +88,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_TIMEOUT_MS", effect: "restart", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_stagger_ms", @@ -113,7 +101,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_STAGGER_MS", effect: "restart", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_concurrency", @@ -127,7 +114,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_CONCURRENCY", effect: "restart", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_persist_pings", @@ -139,7 +125,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_PERSIST_PINGS", effect: "live", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_exhausted_recheck_sec", @@ -153,7 +138,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_EXHAUSTED_RECHECK_SEC", effect: "restart", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_exhausted_max_rechecks", @@ -167,7 +151,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_EXHAUSTED_MAX_RECHECKS", effect: "restart", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_sample_replay_enabled", @@ -179,7 +162,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_SAMPLE_REPLAY_ENABLED", effect: "live", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_sample_canary_every_n_cycles", @@ -193,7 +175,6 @@ export const REGISTRY: FeatureSettingDef[] = [ envVar: "HEARTBEAT_SAMPLE_CANARY_EVERY_N_CYCLES", effect: "live", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_benchmark_mode", @@ -205,7 +186,6 @@ export const REGISTRY: FeatureSettingDef[] = [ options: ["off", "sample_only", "synthetic_only", "sample_first"], effect: "live", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "heartbeat_benchmark_max_tokens", @@ -218,7 +198,6 @@ export const REGISTRY: FeatureSettingDef[] = [ max: 2048, effect: "live", group: "Resilience", - parentToggle: "heartbeat_enabled", }, { key: "degrade_hard_cooldown_threshold", diff --git a/server/src/services/heartbeat.ts b/server/src/services/heartbeat.ts index 74e50becd..34eec699b 100644 --- a/server/src/services/heartbeat.ts +++ b/server/src/services/heartbeat.ts @@ -13,7 +13,7 @@ * side-by-side correlation. * Pings every key for every enabled model in the fallback chain. * - * Opt-in: disabled by default (heartbeat_enabled=false). + * Always enabled — health-check pings run every configured interval. */ import type { @@ -138,7 +138,7 @@ export function getKeyHealth( export function isKeyHealthy(keyId: number, modelId?: string): boolean { if (modelId) { const h = keyHealthMap.get(healthKey(keyId, modelId)); - if (!h) return !isHeartbeatEnabled(); // cold entry + if (!h) return false; // cold entry — unproven return h.healthy; } // No model specified: healthy if ANY model is healthy @@ -152,7 +152,7 @@ function isKeyHealthyOnAnyModel(keyId: number): boolean { // Exact match: key must be "${keyId}:${modelId}" (split by ":" first part === keyId) if (key.split(":")[0] === keyIdStr && health.healthy) return true; } - return !isHeartbeatEnabled(); + return false; } function isKeyKnownUnhealthyForAllEnabledModels( @@ -185,7 +185,6 @@ export function markKeyUnhealthy( transient = false, recheckDelayMs?: number, ): void { - if (!isHeartbeatEnabled()) return; // Transient 429s (per-minute/quota-window) should NOT evict the key from // the healthy pool or increment its penalty — they are expected and // self-resolving. We still schedule a recheck so the key gets verified @@ -230,7 +229,6 @@ export function markKeyUnhealthy( * Mirrors the ping-success state: healthy=true, penalty=0. */ export function markKeyTrafficHealthy(keyId: number, modelId: string): void { - if (!isHeartbeatEnabled()) return; const key = healthKey(keyId, modelId); const prev = keyHealthMap.get(key); keyHealthMap.set(key, { @@ -273,9 +271,6 @@ export function markKeyTrafficHealthy(keyId: number, modelId: string): void { * Returns false when heartbeat is disabled, meaning all keys are usable * without prewarming (backward-compatible mode for router.ts). */ -export function isHeartbeatEnabled(): boolean { - return readConfig().enabled; -} /** Get all key health states (for dashboard/debugging). Key is composite `${keyId}:${modelId}`. */ export function getAllKeyHealth(): Map { @@ -583,7 +578,6 @@ export function getPendingRechecks(): ReadonlyMap< // Configuration (lazy-initialized from feature-settings on first use) // ────────────────────────────────────────────────────────────────────── -let _enabled: boolean | null = null; let _intervalMs: number | null = null; let _persistPings: boolean | null = null; let _pingTimeoutMs: number | null = null; @@ -593,8 +587,7 @@ let _recheckSec: number | null = null; let _maxRechecks: number | null = null; function readConfig() { - if (_enabled === null) { - _enabled = getFeatureSetting("heartbeat_enabled") as boolean; + if (_intervalMs === null) { _intervalMs = (getFeatureSetting("heartbeat_interval_min") as number) * 60 * 1000; _persistPings = getFeatureSetting("heartbeat_persist_pings") as boolean; @@ -609,7 +602,6 @@ function readConfig() { ) as number; } return { - enabled: _enabled, intervalMs: _intervalMs!, persistPings: _persistPings!, pingTimeoutMs: _pingTimeoutMs!, @@ -622,7 +614,6 @@ function readConfig() { /** Reset the cached config (used in tests and after settings change). */ export function resetHeartbeatConfig(): void { - _enabled = null; _intervalMs = null; _persistPings = null; _pingTimeoutMs = null; @@ -681,14 +672,10 @@ export function recordActivity(): void { lastActivityAt = Date.now(); } -/** Called from server startup to begin the timer. No-op when disabled. */ +/** Called from server startup to begin the per-key-per-model timer. */ export function startHeartbeat(): void { try { - const { enabled, intervalMs } = readConfig(); - if (!enabled) { - console.log("[Heartbeat] Disabled — no timer started"); - return; - } + const { intervalMs } = readConfig(); if (timerRef) return; // already running console.log( `[Heartbeat] Starting per-key-per-model timer (interval=${intervalMs / 1000}s)`, @@ -750,14 +737,11 @@ export async function pokeAllKeys(): Promise<{ * highest-priority model for the key's platform (current behavior). * * Does NOT block on cycleInProgress — a single-key ping runs independently - * of the cycle state. When heartbeat is disabled, returns true immediately - * (backward compat: all keys assumed healthy). */ + * of the cycle state. Performs an on-demand health ping and returns post-ping health. */ export async function pokeKey( keyId: number, modelId?: string, ): Promise { - if (!isHeartbeatEnabled()) return true; - const db = getDb(); const keyRow = db .prepare("SELECT * FROM api_keys WHERE id = ? AND enabled = 1") diff --git a/server/src/services/model-priors.ts b/server/src/services/model-priors.ts index 3a951ac22..e8f65513b 100644 --- a/server/src/services/model-priors.ts +++ b/server/src/services/model-priors.ts @@ -396,25 +396,6 @@ export function getModelRoutingPrior( return MODEL_PRIOR_BY_KEY.get(`${platform}\u0000${modelId}`); } -export function reliabilityPosteriorForModel( - platform: string, - modelId: string, - successes: number, - failures: number, -): { alpha: number; beta: number } { - const prior = getModelRoutingPrior(platform, modelId); - if (!prior) { - return { - alpha: Math.max(0, successes) + 1, - beta: Math.max(0, failures) + 1, - }; - } - return { - alpha: Math.max(0, successes) + prior.alpha0, - beta: Math.max(0, failures) + prior.beta0, - }; -} - export function routeTimeoutMsForModel( platform: string, modelId: string, diff --git a/server/src/services/racing-mode.ts b/server/src/services/racing-mode.ts index 7ce64d37b..605000cf1 100644 --- a/server/src/services/racing-mode.ts +++ b/server/src/services/racing-mode.ts @@ -18,7 +18,6 @@ import { recordFailure, recordSuccess } from "../services/degradation.js"; import { publish } from "../services/events.js"; import { getFeatureSetting } from "../services/feature-settings.js"; import { - isHeartbeatEnabled, markKeyTrafficHealthy, recordActivity, } from "../services/heartbeat.js"; @@ -472,7 +471,7 @@ async function handleRacingStream( estimatedInputTokens + totalOutputTokens, ); recordSuccess(won.modelDbId); - if (isHeartbeatEnabled()) markKeyTrafficHealthy(won.keyId, won.modelId); + markKeyTrafficHealthy(won.keyId, won.modelId); recordActivity(); publish({ type: "request.done", @@ -626,7 +625,7 @@ async function handleRacingNonStream( response.usage?.total_tokens ?? promptTokens + completionTokens, ); recordSuccess(won.modelDbId); - if (isHeartbeatEnabled()) markKeyTrafficHealthy(won.keyId, won.modelId); + markKeyTrafficHealthy(won.keyId, won.modelId); recordActivity(); publish({ type: "request.done", diff --git a/server/src/services/router.ts b/server/src/services/router.ts index 8cef1bf05..82834dfdb 100644 --- a/server/src/services/router.ts +++ b/server/src/services/router.ts @@ -22,15 +22,11 @@ import { getKeyHealth, getKeyStats, heartbeatReliability, - isHeartbeatEnabled, isKeyHealthy, type KeyStats, } from "./heartbeat.js"; import { isExhausted } from "./key-exhaustion.js"; -import { - reliabilityPosteriorForModel, - routeTimeoutMsForModel, -} from "./model-priors.js"; +import { routeTimeoutMsForModel } from "./model-priors.js"; import { getProviderStrategy } from "./provider-strategy.js"; import { type TransportId, @@ -46,7 +42,6 @@ import { latencyCompositeFromSize, type RoutingStrategy, type RoutingWeights, - sampleBeta, speedCompositeFromRank, } from "./scoring.js"; @@ -712,26 +707,8 @@ function scoreChainEntry( const successes = stats?.successes ?? 0; const failures = stats?.failures ?? 0; - let reliability: number; - if (isHeartbeatEnabled()) { - reliability = heartbeatReliability(entry.platform, entry.model_id) / 100; - } else if (sampled) { - const { alpha, beta } = reliabilityPosteriorForModel( - entry.platform, - entry.model_id, - successes, - failures, - ); - reliability = sampleBeta(alpha, beta); - } else { - const { alpha, beta } = reliabilityPosteriorForModel( - entry.platform, - entry.model_id, - successes, - failures, - ); - reliability = alpha / (alpha + beta); - } + const reliability = + heartbeatReliability(entry.platform, entry.model_id) / 100; // Compute a default speed score from the manual speed_rank so we have a // fallback when no real perf data exists yet. Uses the same min-max @@ -911,14 +888,7 @@ interface PartitionedKeys { unhealthy: KeyRow[]; } -function partitionKeys( - keys: KeyRow[], - modelId: string, - heartbeatEnabled: boolean, -): PartitionedKeys { - if (!heartbeatEnabled) { - return { healthy: keys, cold: [], unhealthy: [] }; - } +function partitionKeys(keys: KeyRow[], modelId: string): PartitionedKeys { return { healthy: keys.filter((k) => isKeyHealthy(k.id, modelId)), cold: keys.filter((k) => !getKeyHealth(k.id, modelId)), @@ -956,12 +926,11 @@ function tryRouteFlatEntry( (providerStickyEnabled && options?.stickySessionKey)) && options?.stickySessionKey; - const heartbeatEnabled = isHeartbeatEnabled(); const { healthy: healthyKeys, cold: coldKeys, unhealthy: unhealthyKeys, - } = partitionKeys(keys, entry.model_id, heartbeatEnabled); + } = partitionKeys(keys, entry.model_id); if ( unhealthyKeys.length === 0 && healthyKeys.length === 0 && @@ -981,9 +950,11 @@ function tryRouteFlatEntry( idx = hashInt % keyOrder.length; } else { rrIdx = roundRobinIndex.get(rrKey) ?? 0; - const orderedHealthyKeys = heartbeatEnabled - ? speedWeightedShuffle(healthyKeys, entry.model_id, entry.platform) - : rotateArray(healthyKeys, rrIdx); + const orderedHealthyKeys = speedWeightedShuffle( + healthyKeys, + entry.model_id, + entry.platform, + ); keyOrder = [ ...orderedHealthyKeys, ...rotateArray(coldKeys, rrIdx), @@ -1229,21 +1200,11 @@ export function routeRacingRequest( "SELECT * FROM api_keys WHERE platform = ? AND enabled = 1 AND status IN ('healthy', 'unknown', 'error')", ) .all(platform) as KeyRow[]; - const heartbeatEnabled = isHeartbeatEnabled(); - - let candidates: KeyRow[]; - if (heartbeatEnabled) { - const { healthy, cold, unhealthy } = partitionKeys( - keys, - modelId, - heartbeatEnabled, - ); - if (healthy.length === 0 && cold.length === 0 && unhealthy.length === 0) - return null; - candidates = [...healthy, ...cold, ...unhealthy]; - } else { - candidates = keys; - } + + const { healthy, cold, unhealthy } = partitionKeys(keys, modelId); + if (healthy.length === 0 && cold.length === 0 && unhealthy.length === 0) + return null; + const candidates = [...healthy, ...cold, ...unhealthy]; // Round-robin selection — walk candidates from rrIdx, skip exhausted const rrKey = `${platform}:${modelId}`;