refactor(metrics): centralize registry and lifecycle tracking

.cargo/config.toml

@@ -1,2 +1,3 @@
 [build]
-rustflags = ["-Zthreads=8", "-Zshare-generics=y"]
+# The nightly global next solver exceeds CI memory on synctv-api-http; see rust-lang/rust#161748.
+rustflags = ["-Zthreads=8", "-Zshare-generics=y", "-Znext-solver=coherence"]

.github/workflows/ci.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
           components: rustfmt
 
       - name: Check formatting
@@ -55,14 +55,17 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
           components: clippy
 
       - name: Install build dependencies
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler nasm libclang-dev
 
       - name: Cache Rust build artifacts
         uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          cache-workspace-crates: true
 
       - name: Run Clippy
         timeout-minutes: 60
@@ -79,13 +82,16 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install build dependencies
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler nasm libclang-dev
 
       - name: Cache Rust build artifacts
         uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          cache-workspace-crates: true
 
       - name: Build
         run: make build-workspace
@@ -100,7 +106,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install build dependencies
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler nasm libclang-dev
@@ -122,13 +128,16 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install build dependencies
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler nasm libclang-dev
 
       - name: Cache Rust build artifacts
         uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          cache-workspace-crates: true
 
       - name: Check SQLx offline metadata
         run: make check-all-targets
@@ -150,7 +159,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install cargo-nextest
         uses: taiki-e/install-action@nextest
@@ -173,6 +182,9 @@ jobs:
 
       - name: Cache Rust build artifacts
         uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          cache-workspace-crates: true
 
       - name: Run non-ignored tests with nextest
         timeout-minutes: 60
@@ -199,7 +211,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install cargo-nextest
         uses: taiki-e/install-action@nextest
@@ -210,6 +222,9 @@ jobs:
 
       - name: Cache Rust build artifacts
         uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          cache-workspace-crates: true
 
       - name: Run ignored tests with nextest
         timeout-minutes: 60
@@ -225,7 +240,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install cargo-audit
         run: make install-cargo-audit
@@ -248,7 +263,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install cargo-deny
         run: make install-cargo-deny
@@ -282,11 +297,17 @@ jobs:
       - name: Install Rust nightly
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install build dependencies
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler nasm libclang-dev
 
+      - name: Cache Rust build artifacts
+        uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+          cache-workspace-crates: true
+
       - name: Install cargo-udeps
         run: make install-cargo-udeps
 

.github/workflows/docker.yml

@@ -82,7 +82,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install Flutter
         uses: subosito/flutter-action@1a449444c387b1966244ae4d4f8c696479add0b2 # v2

.github/workflows/helm-ci.yml

@@ -47,11 +47,18 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install build dependencies
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler nasm libclang-dev
 
+      - name: Cache Rust build artifacts
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: build
+          cache-on-failure: true
+          cache-workspace-crates: true
+
       - name: Validate chart
         run: make validate-helm
 

.github/workflows/helm.yml

@@ -41,7 +41,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Install build dependencies
         run: sudo apt-get update && sudo apt-get install -y protobuf-compiler nasm libclang-dev

.github/workflows/prepare-release.yml

@@ -30,7 +30,7 @@ jobs:
       - name: Install Rust toolchain
         uses: dtolnay/rust-toolchain@master
         with:
-          toolchain: nightly-2026-08-21
+          toolchain: nightly-2026-08-25
 
       - name: Normalize release version
         id: version

Dockerfile

@@ -96,10 +96,13 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
     if [ -n "$SYNCTV_BUILD_FEATURES" ]; then \
         build_flags="$build_flags --features $SYNCTV_BUILD_FEATURES"; \
     fi; \
-    RUSTFLAGS="-Clink-arg=-fuse-ld=lld -Clink-arg=-Wl,-z,pack-relative-relocs" \
     cargo \
-        build $build_flags \
-        --bin synctv && \
+        rustc $build_flags \
+        -p synctv \
+        --bin synctv \
+        -- \
+        -Clink-arg=-fuse-ld=lld \
+        -Clink-arg=-Wl,-z,pack-relative-relocs && \
     cp "target/$target_profile_dir/synctv" /synctv
 
 # Stage 2: Runtime image

Makefile

@@ -373,7 +373,7 @@ clippy: ## Apply Clippy fixes, then require a clean workspace lint pass.
 	SQLX_OFFLINE=true $(CARGO) clippy $(CARGO_WORKSPACE_ALL_TARGETS_BUILD_ARGS) --fix --allow-dirty
 
 clippy-check: ## Run locked workspace Clippy checks without modifying files.
-	SQLX_OFFLINE=true $(CARGO) clippy $(CARGO_WORKSPACE_ALL_TARGETS_BUILD_ARGS)
+	SQLX_OFFLINE=true $(CARGO) clippy $(CARGO_WORKSPACE_ALL_TARGETS_BUILD_ARGS) -- -D warnings
 
 install-cargo-audit: ## Install cargo-audit for CI security checks.
 	$(CARGO) install cargo-audit $(CARGO_LOCKED)

docs/src/content/docs/configuration/metrics.mdx

@@ -14,6 +14,8 @@ metrics:
 
 生产环境可以开启，但不要直接暴露公网。Linux 构建会把进程级指标和业务指标放在同一个 registry 中暴露。
 
+metrics listener 启动时会校验并注册全部指标定义。重复或无效定义会中止启动，避免端点静默暴露不完整的 registry。抓取时发生编码错误会返回 HTTP `500`；应在 Prometheus 中配置抓取失败告警。
+
 ## 常用示例
 
 Bearer token：

docs/src/content/docs/en/configuration/metrics.mdx

@@ -18,6 +18,8 @@ Production deployments should enable metrics but avoid exposing them publicly.
 
 Linux builds also register process-level metrics in the same Prometheus registry, including CPU, memory, file descriptors, and process start time. They are exposed through `/metrics` together with application metrics and need no extra configuration.
 
+All metric definitions are validated and registered when the metrics listener starts. A duplicate or invalid definition stops startup so the endpoint cannot silently expose a partial registry. A scrape-time encoding failure returns HTTP `500`; configure scrape-failure alerts in Prometheus.
+
 ## Common Examples
 
 Bearer token:

docs/src/content/docs/en/reference/metrics-catalog.mdx

@@ -19,6 +19,13 @@ curl -fsS \
 Disabled or unused features may not emit their metrics. Do not expose the metrics listener directly to the public internet.
 </Aside>
 
+## Instrumentation Contract
+
+- SyncTV registers every metric definition when the metrics listener starts. Invalid or duplicate definitions stop startup instead of producing a partial registry.
+- Labels use bounded values. HTTP `path` is an Axum route template such as `/api/rooms/{room_id}`; resource IDs, query strings, and raw error messages are excluded.
+- Existing metric names and label sets are compatibility-sensitive. Review dashboards and alerts before changing them.
+- A vector metric may remain absent until its feature records the first labeled sample.
+
 ## HTTP And WebSocket
 
 | Metric | Type | Labels | Meaning |
@@ -38,11 +45,28 @@ Disabled or unused features may not emit their metrics. Do not expose the metric
 | --- | --- | --- | --- |
 | `db_connections_active` | gauge | none | Active DB connections |
 | `db_connections_idle` | gauge | none | Idle DB connections |
+| `db_pool_size_max` | gauge | none | Configured maximum DB pool size across pools |
 | `db_pool_utilization_ratio` | gauge | `pool` | Pool utilization, from 0 to 1 |
 | `cache_hits_total` | counter | `cache_type`, `level` | Cache hits |
 | `cache_misses_total` | counter | `cache_type`, `level` | Cache misses |
 | `cache_evictions_total` | counter | `cache_type` | Cache evictions |
 | `cache_errors_total` | counter | `cache_type`, `operation` | Cache operation errors |
+| `cache_invalidations_total` | counter | `cache_type` | Cache invalidations |
+| `cache_operation_duration_seconds` | histogram | `operation` | Cache operation duration |
+| `cache_lag_flush_total` | counter | `component` | Full L1 flushes after invalidation-channel lag |
+| `cache_fence_operations_total` | counter | `domain`, `operation`, `result` | Version-fence operations |
+| `cache_db_fallback_total` | counter | `domain`, `reason` | Strong reads that fell back to PostgreSQL |
+| `cache_stale_write_reject_total` | counter | `cache_type`, `level` | Rejected stale cache writes |
+| `cache_fence_pending` | gauge | `domain` | Domains with a pending version fence |
+| `cache_fence_repair_total` | counter | `domain`, `result` | Read-time fence repair outcomes |
+| `cache_fence_db_compare` | gauge | `domain`, `relation` | Latest DB-to-fence patrol comparison |
+
+## Remote Transport
+
+| Metric | Type | Labels | Meaning |
+| --- | --- | --- | --- |
+| `grpc_requests_total` | counter | `service`, `method`, `status` | Completed gRPC requests |
+| `grpc_request_duration_seconds` | histogram | `service`, `method`, `status` | gRPC request duration |
 
 ## Business And Rate Limits
 
@@ -56,6 +80,7 @@ Disabled or unused features may not emit their metrics. Do not expose the metric
 | `webrtc_peers_active` | gauge | none | Active WebRTC peers |
 | `active_connections` | gauge | none | Active connections |
 | `spawned_task_panics_total` | counter | `task_name` | Background task panics caught by `spawn_monitored` |
+| `logging_dropped_lines_total` | counter | `component` | Log lines dropped by full non-blocking queues |
 | `email_delivery_queue_depth` | gauge | none | Email jobs awaiting delivery in the PostgreSQL outbox |
 | `email_delivery_in_flight` | gauge | none | Email jobs being delivered by this instance |
 | `email_delivery_jobs_total` | counter | `kind`, `status` | Email job outcomes: `sent`, `retry`, `dead`, `superseded`, `fenced`, `persist_failed`, or `ack_failed` |
@@ -76,6 +101,10 @@ Disabled or unused features may not emit their metrics. Do not expose the metric
 | `synctv_cluster_leader_election_epoch` | gauge | none | Current leader epoch |
 | `synctv_cluster_leader_election_consecutive_failures` | gauge | none | Consecutive leader election failures |
 | `synctv_cluster_epoch_mismatch_quarantine` | gauge | none | Epoch mismatch quarantine state |
+| `synctv_cluster_leader_election_mode` | gauge | none | Election mode: 0 standalone, 1 Redis, 2 Kubernetes Lease |
+| `synctv_cluster_distributed_counter_ttl_refreshes_total` | counter | `result` | Distributed counter TTL refresh outcomes |
+| `synctv_cluster_distributed_counter_ttl_keys_refreshed` | gauge | none | Keys refreshed in the latest TTL cycle |
+| `synctv_cluster_distributed_counter_ttl_consecutive_failures` | gauge | none | Consecutive TTL refresh failures |
 
 ## Media And Livestream
 
@@ -85,6 +114,8 @@ Disabled or unused features may not emit their metrics. Do not expose the metric
 | `active_relay_streams` | gauge | none | Active relay streams |
 | `stream_errors_total` | counter | `stream_type`, `error_type` | Stream errors |
 | `streamhub_restarts_total` | counter | `reason` | StreamHub event loop restarts |
+| `streams_active` | gauge | none | Active tracked streams |
+| `synctv_publisher_heartbeat_failures_total` | counter | none | Publisher cleanups after heartbeat failure |
 | `livestream_active_publishers` | gauge | none | Active livestream publishers |
 | `livestream_active_viewers` | gauge | none | Active livestream viewers |
 | `livestream_relay_frame_drops_total` | counter | none | Relay frame drops caused by backpressure |

docs/src/content/docs/reference/metrics-catalog.mdx

@@ -19,6 +19,13 @@ curl -fsS \
 没有启用的功能可能不会产生对应指标。不要把 metrics listener 直接暴露公网。
 </Aside>
 
+## 埋点契约
+
+- metrics listener 启动时会注册全部指标定义。无效或重复定义会中止启动，避免暴露不完整的 registry。
+- Label 只能使用有限集合。HTTP `path` 使用 Axum 路由模板，例如 `/api/rooms/{room_id}`；不得包含资源 ID、query string 或原始错误消息。
+- 现有指标名和 label 集合属于兼容接口。修改前必须检查 dashboard 和 alert。
+- vector 指标可能在对应功能首次记录带 label 的样本前保持缺失。
+
 ## HTTP 和 WebSocket
 
 | 指标 | 类型 | Labels | 含义 |
@@ -38,11 +45,28 @@ curl -fsS \
 | --- | --- | --- | --- |
 | `db_connections_active` | gauge | none | 活跃数据库连接 |
 | `db_connections_idle` | gauge | none | 空闲数据库连接 |
+| `db_pool_size_max` | gauge | none | 所有数据库连接池配置的最大连接数 |
 | `db_pool_utilization_ratio` | gauge | `pool` | 连接池利用率，取值 0 到 1 |
 | `cache_hits_total` | counter | `cache_type`, `level` | 缓存命中数 |
 | `cache_misses_total` | counter | `cache_type`, `level` | 缓存未命中数 |
 | `cache_evictions_total` | counter | `cache_type` | 缓存淘汰数 |
 | `cache_errors_total` | counter | `cache_type`, `operation` | 缓存操作错误数 |
+| `cache_invalidations_total` | counter | `cache_type` | 缓存失效次数 |
+| `cache_operation_duration_seconds` | histogram | `operation` | 缓存操作耗时 |
+| `cache_lag_flush_total` | counter | `component` | 失效 channel 延迟触发的 L1 全量清理次数 |
+| `cache_fence_operations_total` | counter | `domain`, `operation`, `result` | version fence 操作次数 |
+| `cache_db_fallback_total` | counter | `domain`, `reason` | 强一致读取回退 PostgreSQL 的次数 |
+| `cache_stale_write_reject_total` | counter | `cache_type`, `level` | 被拒绝的过期缓存写入次数 |
+| `cache_fence_pending` | gauge | `domain` | 存在待处理 version fence 的 domain |
+| `cache_fence_repair_total` | counter | `domain`, `result` | 读取时 fence 修复结果 |
+| `cache_fence_db_compare` | gauge | `domain`, `relation` | 最近一次 DB 与 fence 巡检比较结果 |
+
+## 远程传输
+
+| 指标 | 类型 | Labels | 含义 |
+| --- | --- | --- | --- |
+| `grpc_requests_total` | counter | `service`, `method`, `status` | 已完成 gRPC 请求数 |
+| `grpc_request_duration_seconds` | histogram | `service`, `method`, `status` | gRPC 请求耗时 |
 
 ## 业务和限流
 
@@ -56,6 +80,7 @@ curl -fsS \
 | `webrtc_peers_active` | gauge | none | 活跃 WebRTC peer |
 | `active_connections` | gauge | none | 活跃连接 |
 | `spawned_task_panics_total` | counter | `task_name` | `spawn_monitored` 捕获的后台任务 panic |
+| `logging_dropped_lines_total` | counter | `component` | 非阻塞日志队列满时丢弃的日志行数 |
 | `email_delivery_queue_depth` | gauge | none | PostgreSQL outbox 中等待投递的邮件任务数 |
 | `email_delivery_in_flight` | gauge | none | 当前实例正在投递的邮件任务数 |
 | `email_delivery_jobs_total` | counter | `kind`, `status` | 邮件任务结果数；状态包括 `sent`、`retry`、`dead`、`superseded`、`fenced`、`persist_failed` 和 `ack_failed` |
@@ -76,6 +101,10 @@ curl -fsS \
 | `synctv_cluster_leader_election_epoch` | gauge | none | 当前 leader epoch |
 | `synctv_cluster_leader_election_consecutive_failures` | gauge | none | 连续选主失败数 |
 | `synctv_cluster_epoch_mismatch_quarantine` | gauge | none | epoch mismatch 隔离状态 |
+| `synctv_cluster_leader_election_mode` | gauge | none | 选主模式：0 standalone、1 Redis、2 Kubernetes Lease |
+| `synctv_cluster_distributed_counter_ttl_refreshes_total` | counter | `result` | 分布式计数器 TTL 刷新结果 |
+| `synctv_cluster_distributed_counter_ttl_keys_refreshed` | gauge | none | 最近一轮 TTL 刷新的 key 数 |
+| `synctv_cluster_distributed_counter_ttl_consecutive_failures` | gauge | none | 连续 TTL 刷新失败数 |
 
 ## 媒体和直播
 
@@ -85,6 +114,8 @@ curl -fsS \
 | `active_relay_streams` | gauge | none | 活跃 relay stream |
 | `stream_errors_total` | counter | `stream_type`, `error_type` | stream 错误数 |
 | `streamhub_restarts_total` | counter | `reason` | StreamHub event loop 重启次数 |
+| `streams_active` | gauge | none | 当前被跟踪的活跃 stream |
+| `synctv_publisher_heartbeat_failures_total` | counter | none | heartbeat 失败后清理 publisher 的次数 |
 | `livestream_active_publishers` | gauge | none | 活跃直播 publisher |
 | `livestream_active_viewers` | gauge | none | 活跃直播 viewer |
 | `livestream_relay_frame_drops_total` | counter | none | backpressure 导致的 relay 丢帧 |

rust-toolchain.toml

@@ -1,4 +1,4 @@
 [toolchain]
-channel = "nightly-2026-08-21"
+channel = "nightly-2026-08-25"
 profile = "minimal"
 components = ["clippy", "rustfmt"]