feat: add experimental support for promql via CH timeseries table

.changeset/clever-suns-nail.md

@@ -0,0 +1,9 @@
+---
+"@hyperdx/otel-collector": minor
+"@hyperdx/common-utils": minor
+"@hyperdx/api": minor
+"@hyperdx/app": minor
+"@hyperdx/cli": minor
+---
+
+feat: experimental promql support

docker-compose.dev.yml

@@ -40,6 +40,7 @@ services:
       # Uncomment to enable stdout logging for the OTel collector
       OTEL_SUPERVISOR_LOGS: 'true'
       HYPERDX_OTEL_EXPORTER_TABLES_TTL: '24h'
+      ENABLE_PROMQL: 'true'
     volumes:
       - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml
       - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl
@@ -100,7 +101,7 @@ services:
       hdx.dev.service: clickhouse
       hdx.dev.port: '${HDX_DEV_CH_HTTP_PORT:-8123}'
       hdx.dev.url: 'http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}'
-    image: clickhouse/clickhouse-server:26.2-alpine
+    image: clickhouse/clickhouse-server:26.4-alpine
     ports:
       - '${HDX_DEV_CH_HTTP_PORT:-8123}:8123' # http api
       - '${HDX_DEV_CH_NATIVE_PORT:-9000}:9000' # native
@@ -150,5 +151,23 @@ services:
   #   network_mode: host
   #   restart: always
 
+  prometheus:
+    labels:
+      <<: *hdx-labels
+      hdx.dev.service: prometheus
+      hdx.dev.port: '${HDX_DEV_PROMETHEUS_PORT:-9090}'
+      hdx.dev.url: 'http://localhost:${HDX_DEV_PROMETHEUS_PORT:-9090}'
+    profiles:
+      - prometheus
+    image: prom/prometheus:latest
+    ports:
+      - '${HDX_DEV_PROMETHEUS_PORT:-9090}:9090'
+    volumes:
+      - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - .volumes/prometheus_data_dev_${HDX_DEV_SLOT:-0}:/prometheus
+    networks:
+      - internal
+    restart: on-failure
+
 networks:
   internal:

docker-compose.yml

@@ -34,6 +34,8 @@ services:
       OPAMP_SERVER_URL: 'http://app:${HYPERDX_OPAMP_PORT}'
       # TODO: use new schema
       HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA: 'true'
+      # Uncomment to enable PromQL schema (must match app service's ENABLE_PROMQL)
+      # ENABLE_PROMQL: 'true'
     ports:
       - '13133:13133' # health_check extension
       - '24225:24225' # fluentd receiver
@@ -67,6 +69,9 @@ services:
       # OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: 'http://otel-collector:4318/v1/logs'
       OTEL_SERVICE_NAME: 'hdx-oss-app'
       USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-true}
+      # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics)
+      # ENABLE_PROMQL: 'true'
+      # NEXT_PUBLIC_ENABLE_PROMQL: 'true'
       DEFAULT_CONNECTIONS:
         '[{"name":"Local
         ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]'

docker/clickhouse/local/config.xml

@@ -34,14 +34,43 @@
     <timezone>UTC</timezone>
     <mlock_executable>false</mlock_executable>
 
-    <!-- Prometheus exporter -->
+    <access_control_path>/var/lib/clickhouse/access</access_control_path>
+
+    <!-- Prometheus metrics exporter + remote write/read ingestion -->
     <prometheus>
-        <endpoint>/metrics</endpoint>
         <port>9363</port>
         <metrics>true</metrics>
         <events>true</events>
         <asynchronous_metrics>true</asynchronous_metrics>
         <errors>true</errors>
+        <handlers>
+            <metrics_handler>
+                <url>/metrics</url>
+                <handler>
+                    <type>expose_metrics</type>
+                    <metrics>true</metrics>
+                    <asynchronous_metrics>true</asynchronous_metrics>
+                    <events>true</events>
+                    <errors>true</errors>
+                </handler>
+            </metrics_handler>
+            <remote_write_handler>
+                <url>/write</url>
+                <handler>
+                    <type>remote_write</type>
+                    <database>default</database>
+                    <table>otel_metrics_ts</table>
+                </handler>
+            </remote_write_handler>
+            <remote_read_handler>
+                <url>/read</url>
+                <handler>
+                    <type>remote_read</type>
+                    <database>default</database>
+                    <table>otel_metrics_ts</table>
+                </handler>
+            </remote_read_handler>
+        </handlers>
     </prometheus>
 
     <!-- Query log. Used only for queries with setting log_queries = 1. -->

docker/clickhouse/local/users.xml

@@ -6,6 +6,7 @@
             <use_uncompressed_cache>0</use_uncompressed_cache>
             <load_balancing>in_order</load_balancing>
             <log_queries>1</log_queries>
+            <allow_experimental_time_series_table>1</allow_experimental_time_series_table>
         </default>
     </profiles>
 

docker/otel-collector/config.standalone.yaml

@@ -53,6 +53,12 @@ exporters:
       initial_interval: 5s
       max_interval: 30s
       max_elapsed_time: 300s
+  prometheusremotewrite:
+    endpoint: http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write
+    tls:
+      insecure: true
+    resource_to_telemetry_conversion:
+      enabled: true
 
 service:
   pipelines:
@@ -64,6 +70,10 @@ service:
       receivers: [otlp/hyperdx]
       processors: [memory_limiter, batch]
       exporters: [clickhouse]
+    metrics/promql:
+      receivers: [otlp/hyperdx]
+      processors: [memory_limiter, batch]
+      exporters: [prometheusremotewrite]
     logs/in:
       receivers: [otlp/hyperdx]
       exporters: [routing/logs]

docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql

@@ -0,0 +1,4 @@
+-- +goose Up
+CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_ts
+ENGINE = TimeSeries
+SETTINGS allow_experimental_time_series_table = 1;

docker/prometheus/prometheus.yml

@@ -0,0 +1,20 @@
+global:
+  scrape_interval: 15s
+  evaluation_interval: 15s
+
+scrape_configs:
+  # Scrape Prometheus itself
+  - job_name: 'prometheus'
+    static_configs:
+      - targets: ['localhost:9090']
+
+  # Scrape the OTel collector's internal metrics
+  - job_name: 'otel-collector'
+    static_configs:
+      - targets: ['otel-collector:8888']
+
+  # Scrape ClickHouse metrics (if the /metrics handler is available)
+  - job_name: 'clickhouse'
+    static_configs:
+      - targets: ['ch-server:9363']
+    metrics_path: '/metrics'

package.json

@@ -40,7 +40,7 @@
     "app:storybook": "nx run @hyperdx/app:storybook",
     "build:clickhouse": "nx run @hyperdx/common-utils:build && nx run @hyperdx/app:build:clickhouse",
     "run:clickhouse": "nx run @hyperdx/app:run:clickhouse",
-    "dev": "sh -c '. ./scripts/dev-env.sh && yarn build:common-utils && dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml up -d && yarn app:dev; dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down'",
+    "dev": "sh -c '. ./scripts/dev-env.sh && yarn build:common-utils && dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml up -d --build && yarn app:dev; dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down'",
     "dev:local": "IS_LOCAL_APP_MODE='DANGEROUSLY_is_local_app_mode💀' yarn dev",
     "cli:dev": "yarn workspace @hyperdx/cli dev",
     "dev:down": "sh -c '. ./scripts/dev-env.sh && docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down && sh ./scripts/dev-kill-ports.sh'",

packages/api/.env.development

@@ -25,3 +25,4 @@ DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kin
 INGESTION_API_KEY="super-secure-ingestion-api-key"
 HYPERDX_API_KEY=$INGESTION_API_KEY
 ANTHROPIC_API_KEY="your-anthropic-api-key-here"
+ENABLE_PROMQL=true

packages/api/.env.test

@@ -15,4 +15,5 @@ OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320}
 # Default to only logging errors. Adjust if you need more verbosity.
 # Note: the logger module is mocked in jest.setup.ts to suppress expected
 # operational noise (validation errors, MCP tool errors, etc.) during tests.
-HYPERDX_LOG_LEVEL=error
+HYPERDX_LOG_LEVEL=error
+ENABLE_PROMQL=true

packages/api/src/api-app.ts

@@ -108,6 +108,9 @@ app.use('/saved-search', isUserAuthenticated, savedSearchRouter);
 app.use('/favorites', isUserAuthenticated, favoritesRouter);
 app.use('/pinned-filters', isUserAuthenticated, pinnedFiltersRouter);
 app.use('/clickhouse-proxy', isUserAuthenticated, clickhouseProxyRouter);
+if (config.IS_PROMQL_ENABLED) {
+  app.use('/v1/prometheus', isUserAuthenticated, routers.prometheusRouter);
+}
 // ---------------------------------------------------------------------
 
 // TODO: Separate external API routers from internal routers

packages/api/src/config.ts

@@ -49,6 +49,8 @@ export const IS_LOCAL_APP_MODE =
 export const DEFAULT_CONNECTIONS = env.DEFAULT_CONNECTIONS;
 export const DEFAULT_SOURCES = env.DEFAULT_SOURCES;
 
+export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true';
+
 // FOR CI ONLY
 export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string;
 export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string;

packages/api/src/controllers/sources.ts

@@ -4,6 +4,7 @@ import {
   ISourceInput,
   LogSource,
   MetricSource,
+  PromqlSource,
   SessionSource,
   Source,
   TraceSource,
@@ -22,6 +23,8 @@ function getModelForKind(kind: SourceKind) {
       return SessionSource;
     case SourceKind.Metric:
       return MetricSource;
+    case SourceKind.Promql:
+      return PromqlSource;
     default:
       kind satisfies never;
       throw new Error(`${kind} is not a valid SourceKind`);

packages/api/src/mcp/tools/query/helpers.ts

@@ -1,7 +1,10 @@
 import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node';
 import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata';
 import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils';
-import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards';
+import {
+  isBuilderSavedChartConfig,
+  isRawSqlSavedChartConfig,
+} from '@hyperdx/common-utils/dist/guards';
 import type {
   ChartConfigWithDateRange,
   MetricTable,
@@ -133,7 +136,7 @@ export async function runConfigTile(
   const internalTile = convertToInternalTileConfig(tile);
   const savedConfig = internalTile.config;
 
-  if (!isRawSqlSavedChartConfig(savedConfig)) {
+  if (isBuilderSavedChartConfig(savedConfig)) {
     const builderConfig = savedConfig;
 
     if (

packages/api/src/models/connection.ts

@@ -12,6 +12,10 @@ export interface IConnection {
   username: string;
   team: ObjectId;
   hyperdxSettingPrefix?: string;
+  /** Optional Prometheus-compatible API endpoint (e.g. http://prometheus:9090).
+   *  When set, PromQL queries are proxied to this endpoint instead of using
+   *  ClickHouse's prometheusQuery() function. */
+  prometheusEndpoint?: string;
 }
 
 export default mongoose.model<IConnection>(
@@ -31,6 +35,7 @@ export default mongoose.model<IConnection>(
         select: false,
       },
       hyperdxSettingPrefix: String,
+      prometheusEndpoint: String,
     },
     {
       timestamps: true,

packages/api/src/models/source.ts

@@ -3,6 +3,7 @@ import {
   LogSourceSchema,
   MetricsDataType,
   MetricSourceSchema,
+  PromqlSourceSchema,
   QuerySettings,
   SessionSourceSchema,
   SourceKind,
@@ -34,6 +35,10 @@ export const ISourceSchema = z.discriminatedUnion('kind', [
     team: objectIdSchema,
     connection: objectIdSchema.or(z.string()),
   }),
+  PromqlSourceSchema.omit({ connection: true }).extend({
+    team: objectIdSchema,
+    connection: objectIdSchema.or(z.string()),
+  }),
 ]);
 export type ISource = z.infer<typeof ISourceSchema>;
 export type ISourceInput = z.input<typeof ISourceSchema>;
@@ -251,3 +256,12 @@ export const MetricSource = Source.discriminator<IMetricSource>(
     logSourceId: String,
   }),
 );
+
+// --------------------------
+// PromQL discriminator
+// --------------------------
+type IPromqlSource = Extract<ISource, { kind: SourceKind.Promql }>;
+export const PromqlSource = Source.discriminator<IPromqlSource>(
+  SourceKind.Promql,
+  new Schema<Extract<ISource, { kind: SourceKind.Promql }>>({}),
+);

packages/api/src/opamp/controllers/opampController.ts

@@ -106,6 +106,15 @@ type CollectorConfig = {
         max_elapsed_time: string;
       };
     };
+    prometheusremotewrite?: {
+      endpoint: string;
+      tls: {
+        insecure: boolean;
+      };
+      resource_to_telemetry_conversion: {
+        enabled: boolean;
+      };
+    };
   };
   service: {
     extensions: string[];
@@ -275,6 +284,23 @@ export const buildOtelCollectorConfig = (
       'otlp/hyperdx',
     );
 
+    if (config.IS_PROMQL_ENABLED && otelCollectorConfig.exporters) {
+      otelCollectorConfig.exporters.prometheusremotewrite = {
+        endpoint: 'http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write',
+        tls: {
+          insecure: true,
+        },
+        resource_to_telemetry_conversion: {
+          enabled: true,
+        },
+      };
+      otelCollectorConfig.service.pipelines['metrics/promql'] = {
+        receivers: ['otlp/hyperdx'],
+        processors: ['memory_limiter', 'batch'],
+        exporters: ['prometheusremotewrite'],
+      };
+    }
+
     if (collectorAuthenticationEnforced) {
       if (otelCollectorConfig.receivers['otlp/hyperdx'] == null) {
         // should never happen