diff --git a/.changeset/clever-suns-nail.md b/.changeset/clever-suns-nail.md new file mode 100644 index 0000000000..af19fd8b47 --- /dev/null +++ b/.changeset/clever-suns-nail.md @@ -0,0 +1,9 @@ +--- +"@hyperdx/otel-collector": minor +"@hyperdx/common-utils": minor +"@hyperdx/api": minor +"@hyperdx/app": minor +"@hyperdx/cli": minor +--- + +feat: experimental promql support diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index df1ba0e00b..effe5d0877 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -40,6 +40,7 @@ services: # Uncomment to enable stdout logging for the OTel collector OTEL_SUPERVISOR_LOGS: 'true' HYPERDX_OTEL_EXPORTER_TABLES_TTL: '24h' + ENABLE_PROMQL: 'true' volumes: - ./docker/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml - ./docker/otel-collector/supervisor_docker.yaml.tmpl:/etc/otel/supervisor.yaml.tmpl @@ -100,7 +101,7 @@ services: hdx.dev.service: clickhouse hdx.dev.port: '${HDX_DEV_CH_HTTP_PORT:-8123}' hdx.dev.url: 'http://localhost:${HDX_DEV_CH_HTTP_PORT:-8123}' - image: clickhouse/clickhouse-server:26.2-alpine + image: clickhouse/clickhouse-server:26.4-alpine ports: - '${HDX_DEV_CH_HTTP_PORT:-8123}:8123' # http api - '${HDX_DEV_CH_NATIVE_PORT:-9000}:9000' # native @@ -150,5 +151,23 @@ services: # network_mode: host # restart: always + prometheus: + labels: + <<: *hdx-labels + hdx.dev.service: prometheus + hdx.dev.port: '${HDX_DEV_PROMETHEUS_PORT:-9090}' + hdx.dev.url: 'http://localhost:${HDX_DEV_PROMETHEUS_PORT:-9090}' + profiles: + - prometheus + image: prom/prometheus:latest + ports: + - '${HDX_DEV_PROMETHEUS_PORT:-9090}:9090' + volumes: + - ./docker/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - .volumes/prometheus_data_dev_${HDX_DEV_SLOT:-0}:/prometheus + networks: + - internal + restart: on-failure + networks: internal: diff --git a/docker-compose.yml b/docker-compose.yml index 678a111219..7c2d5678a5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,6 +34,8 @@ services: OPAMP_SERVER_URL: 'http://app:${HYPERDX_OPAMP_PORT}' # TODO: use new schema HYPERDX_OTEL_EXPORTER_CREATE_LEGACY_SCHEMA: 'true' + # Uncomment to enable PromQL schema (must match app service's ENABLE_PROMQL) + # ENABLE_PROMQL: 'true' ports: - '13133:13133' # health_check extension - '24225:24225' # fluentd receiver @@ -67,6 +69,9 @@ services: # OTEL_EXPORTER_OTLP_LOGS_ENDPOINT: 'http://otel-collector:4318/v1/logs' OTEL_SERVICE_NAME: 'hdx-oss-app' USAGE_STATS_ENABLED: ${USAGE_STATS_ENABLED:-true} + # Uncomment the next two lines to enable PromQL (Prometheus-compatible metrics) + # ENABLE_PROMQL: 'true' + # NEXT_PUBLIC_ENABLE_PROMQL: 'true' DEFAULT_CONNECTIONS: '[{"name":"Local ClickHouse","host":"http://ch-server:8123","username":"default","password":""}]' diff --git a/docker/clickhouse/local/config.xml b/docker/clickhouse/local/config.xml index 7b45ba5b92..aac116287c 100644 --- a/docker/clickhouse/local/config.xml +++ b/docker/clickhouse/local/config.xml @@ -34,14 +34,43 @@ UTC false - + /var/lib/clickhouse/access + + - /metrics 9363 true true true true + + + /metrics + + expose_metrics + true + true + true + true + + + + /write + + remote_write + default + otel_metrics_ts
+ + + + /read + + remote_read + default + otel_metrics_ts
+ + + diff --git a/docker/clickhouse/local/users.xml b/docker/clickhouse/local/users.xml index 80c8d4fa7b..0837cbaf9a 100644 --- a/docker/clickhouse/local/users.xml +++ b/docker/clickhouse/local/users.xml @@ -6,6 +6,7 @@ 0 in_order 1 + 1 diff --git a/docker/otel-collector/config.standalone.yaml b/docker/otel-collector/config.standalone.yaml index 46e3958bf6..ccd02dd25d 100644 --- a/docker/otel-collector/config.standalone.yaml +++ b/docker/otel-collector/config.standalone.yaml @@ -53,6 +53,12 @@ exporters: initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + prometheusremotewrite: + endpoint: http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write + tls: + insecure: true + resource_to_telemetry_conversion: + enabled: true service: pipelines: @@ -64,6 +70,10 @@ service: receivers: [otlp/hyperdx] processors: [memory_limiter, batch] exporters: [clickhouse] + metrics/promql: + receivers: [otlp/hyperdx] + processors: [memory_limiter, batch] + exporters: [prometheusremotewrite] logs/in: receivers: [otlp/hyperdx] exporters: [routing/logs] diff --git a/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql b/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql new file mode 100644 index 0000000000..50323f3d05 --- /dev/null +++ b/docker/otel-collector/schema/seed/00008_otel_metrics_timeseries.sql @@ -0,0 +1,4 @@ +-- +goose Up +CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_metrics_ts +ENGINE = TimeSeries +SETTINGS allow_experimental_time_series_table = 1; diff --git a/docker/prometheus/prometheus.yml b/docker/prometheus/prometheus.yml new file mode 100644 index 0000000000..94877ac65d --- /dev/null +++ b/docker/prometheus/prometheus.yml @@ -0,0 +1,20 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + # Scrape Prometheus itself + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # Scrape the OTel collector's internal metrics + - job_name: 'otel-collector' + static_configs: + - targets: ['otel-collector:8888'] + + # Scrape ClickHouse metrics (if the /metrics handler is available) + - job_name: 'clickhouse' + static_configs: + - targets: ['ch-server:9363'] + metrics_path: '/metrics' diff --git a/package.json b/package.json index 0d2e0827d9..e2133c7e01 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "app:storybook": "nx run @hyperdx/app:storybook", "build:clickhouse": "nx run @hyperdx/common-utils:build && nx run @hyperdx/app:build:clickhouse", "run:clickhouse": "nx run @hyperdx/app:run:clickhouse", - "dev": "sh -c '. ./scripts/dev-env.sh && yarn build:common-utils && dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml up -d && yarn app:dev; dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down'", + "dev": "sh -c '. ./scripts/dev-env.sh && yarn build:common-utils && dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml up -d --build && yarn app:dev; dotenvx run --convention=nextjs -- docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down'", "dev:local": "IS_LOCAL_APP_MODE='DANGEROUSLY_is_local_app_modeπŸ’€' yarn dev", "cli:dev": "yarn workspace @hyperdx/cli dev", "dev:down": "sh -c '. ./scripts/dev-env.sh && docker compose -p \"$HDX_DEV_PROJECT\" -f docker-compose.dev.yml down && sh ./scripts/dev-kill-ports.sh'", diff --git a/packages/api/.env.development b/packages/api/.env.development index 9d9761b87d..8ba0a6c0c8 100644 --- a/packages/api/.env.development +++ b/packages/api/.env.development @@ -25,3 +25,4 @@ DEFAULT_SOURCES=[{"from":{"databaseName":"default","tableName":"otel_logs"},"kin INGESTION_API_KEY="super-secure-ingestion-api-key" HYPERDX_API_KEY=$INGESTION_API_KEY ANTHROPIC_API_KEY="your-anthropic-api-key-here" +ENABLE_PROMQL=true diff --git a/packages/api/.env.test b/packages/api/.env.test index 2bc5fa808a..0b67f560d1 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -15,4 +15,5 @@ OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320} # Default to only logging errors. Adjust if you need more verbosity. # Note: the logger module is mocked in jest.setup.ts to suppress expected # operational noise (validation errors, MCP tool errors, etc.) during tests. -HYPERDX_LOG_LEVEL=error \ No newline at end of file +HYPERDX_LOG_LEVEL=error +ENABLE_PROMQL=true \ No newline at end of file diff --git a/packages/api/src/api-app.ts b/packages/api/src/api-app.ts index bec663c72e..5f4c3181d4 100644 --- a/packages/api/src/api-app.ts +++ b/packages/api/src/api-app.ts @@ -108,6 +108,9 @@ app.use('/saved-search', isUserAuthenticated, savedSearchRouter); app.use('/favorites', isUserAuthenticated, favoritesRouter); app.use('/pinned-filters', isUserAuthenticated, pinnedFiltersRouter); app.use('/clickhouse-proxy', isUserAuthenticated, clickhouseProxyRouter); +if (config.IS_PROMQL_ENABLED) { + app.use('/v1/prometheus', isUserAuthenticated, routers.prometheusRouter); +} // --------------------------------------------------------------------- // TODO: Separate external API routers from internal routers diff --git a/packages/api/src/config.ts b/packages/api/src/config.ts index b24ac4790b..6677989124 100644 --- a/packages/api/src/config.ts +++ b/packages/api/src/config.ts @@ -49,6 +49,8 @@ export const IS_LOCAL_APP_MODE = export const DEFAULT_CONNECTIONS = env.DEFAULT_CONNECTIONS; export const DEFAULT_SOURCES = env.DEFAULT_SOURCES; +export const IS_PROMQL_ENABLED = env.ENABLE_PROMQL === 'true'; + // FOR CI ONLY export const CLICKHOUSE_HOST = env.CLICKHOUSE_HOST as string; export const CLICKHOUSE_USER = env.CLICKHOUSE_USER as string; diff --git a/packages/api/src/controllers/sources.ts b/packages/api/src/controllers/sources.ts index e2c614e740..a34f931dff 100644 --- a/packages/api/src/controllers/sources.ts +++ b/packages/api/src/controllers/sources.ts @@ -4,6 +4,7 @@ import { ISourceInput, LogSource, MetricSource, + PromqlSource, SessionSource, Source, TraceSource, @@ -22,6 +23,8 @@ function getModelForKind(kind: SourceKind) { return SessionSource; case SourceKind.Metric: return MetricSource; + case SourceKind.Promql: + return PromqlSource; default: kind satisfies never; throw new Error(`${kind} is not a valid SourceKind`); diff --git a/packages/api/src/mcp/tools/query/helpers.ts b/packages/api/src/mcp/tools/query/helpers.ts index e73502aebc..e065c4c642 100644 --- a/packages/api/src/mcp/tools/query/helpers.ts +++ b/packages/api/src/mcp/tools/query/helpers.ts @@ -1,7 +1,10 @@ import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; import { getMetadata } from '@hyperdx/common-utils/dist/core/metadata'; import { getFirstTimestampValueExpression } from '@hyperdx/common-utils/dist/core/utils'; -import { isRawSqlSavedChartConfig } from '@hyperdx/common-utils/dist/guards'; +import { + isBuilderSavedChartConfig, + isRawSqlSavedChartConfig, +} from '@hyperdx/common-utils/dist/guards'; import type { ChartConfigWithDateRange, MetricTable, @@ -133,7 +136,7 @@ export async function runConfigTile( const internalTile = convertToInternalTileConfig(tile); const savedConfig = internalTile.config; - if (!isRawSqlSavedChartConfig(savedConfig)) { + if (isBuilderSavedChartConfig(savedConfig)) { const builderConfig = savedConfig; if ( diff --git a/packages/api/src/models/connection.ts b/packages/api/src/models/connection.ts index a0532d546f..324390df6c 100644 --- a/packages/api/src/models/connection.ts +++ b/packages/api/src/models/connection.ts @@ -12,6 +12,10 @@ export interface IConnection { username: string; team: ObjectId; hyperdxSettingPrefix?: string; + /** Optional Prometheus-compatible API endpoint (e.g. http://prometheus:9090). + * When set, PromQL queries are proxied to this endpoint instead of using + * ClickHouse's prometheusQuery() function. */ + prometheusEndpoint?: string; } export default mongoose.model( @@ -31,6 +35,7 @@ export default mongoose.model( select: false, }, hyperdxSettingPrefix: String, + prometheusEndpoint: String, }, { timestamps: true, diff --git a/packages/api/src/models/source.ts b/packages/api/src/models/source.ts index 319e95c0b5..b7e7d55d33 100644 --- a/packages/api/src/models/source.ts +++ b/packages/api/src/models/source.ts @@ -3,6 +3,7 @@ import { LogSourceSchema, MetricsDataType, MetricSourceSchema, + PromqlSourceSchema, QuerySettings, SessionSourceSchema, SourceKind, @@ -34,6 +35,10 @@ export const ISourceSchema = z.discriminatedUnion('kind', [ team: objectIdSchema, connection: objectIdSchema.or(z.string()), }), + PromqlSourceSchema.omit({ connection: true }).extend({ + team: objectIdSchema, + connection: objectIdSchema.or(z.string()), + }), ]); export type ISource = z.infer; export type ISourceInput = z.input; @@ -251,3 +256,12 @@ export const MetricSource = Source.discriminator( logSourceId: String, }), ); + +// -------------------------- +// PromQL discriminator +// -------------------------- +type IPromqlSource = Extract; +export const PromqlSource = Source.discriminator( + SourceKind.Promql, + new Schema>({}), +); diff --git a/packages/api/src/opamp/controllers/opampController.ts b/packages/api/src/opamp/controllers/opampController.ts index 1b1ca94121..e6b30d3c2b 100644 --- a/packages/api/src/opamp/controllers/opampController.ts +++ b/packages/api/src/opamp/controllers/opampController.ts @@ -106,6 +106,15 @@ type CollectorConfig = { max_elapsed_time: string; }; }; + prometheusremotewrite?: { + endpoint: string; + tls: { + insecure: boolean; + }; + resource_to_telemetry_conversion: { + enabled: boolean; + }; + }; }; service: { extensions: string[]; @@ -275,6 +284,23 @@ export const buildOtelCollectorConfig = ( 'otlp/hyperdx', ); + if (config.IS_PROMQL_ENABLED && otelCollectorConfig.exporters) { + otelCollectorConfig.exporters.prometheusremotewrite = { + endpoint: 'http://${env:CLICKHOUSE_PROMETHEUS_METRICS_ENDPOINT}/write', + tls: { + insecure: true, + }, + resource_to_telemetry_conversion: { + enabled: true, + }, + }; + otelCollectorConfig.service.pipelines['metrics/promql'] = { + receivers: ['otlp/hyperdx'], + processors: ['memory_limiter', 'batch'], + exporters: ['prometheusremotewrite'], + }; + } + if (collectorAuthenticationEnforced) { if (otelCollectorConfig.receivers['otlp/hyperdx'] == null) { // should never happen diff --git a/packages/api/src/routers/api/__tests__/connections.test.ts b/packages/api/src/routers/api/__tests__/connections.test.ts new file mode 100644 index 0000000000..e168de4328 --- /dev/null +++ b/packages/api/src/routers/api/__tests__/connections.test.ts @@ -0,0 +1,83 @@ +import * as config from '@/config'; +import { getLoggedInAgent, getServer } from '@/fixtures'; +import Connection from '@/models/connection'; + +describe('connections router', () => { + const server = getServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await server.clearDBs(); + }); + + afterAll(async () => { + await server.stop(); + }); + + it('persists prometheusEndpoint through POST /connections', async () => { + const { agent } = await getLoggedInAgent(server); + + const res = await agent + .post('/connections') + .send({ + name: 'Prom-enabled', + host: config.CLICKHOUSE_HOST, + username: 'default', + password: '', + prometheusEndpoint: 'http://prom.example.com', + }) + .expect(200); + + const stored = await Connection.findById(res.body.id).select( + '+prometheusEndpoint', + ); + expect(stored?.prometheusEndpoint).toBe('http://prom.example.com'); + }); + + it('rejects invalid prometheusEndpoint URLs with 400', async () => { + const { agent } = await getLoggedInAgent(server); + + await agent + .post('/connections') + .send({ + name: 'Bad-URL', + host: config.CLICKHOUSE_HOST, + username: 'default', + password: '', + prometheusEndpoint: 'not-a-url', + }) + .expect(400); + }); + + it('persists prometheusEndpoint through PUT /connections/:id', async () => { + const { agent, team } = await getLoggedInAgent(server); + + const created = await Connection.create({ + team: team._id, + name: 'No-prom', + host: config.CLICKHOUSE_HOST, + username: 'default', + password: '', + }); + + await agent + .put(`/connections/${created._id.toString()}`) + .send({ + id: created._id.toString(), + name: 'No-prom', + host: config.CLICKHOUSE_HOST, + username: 'default', + password: '', + prometheusEndpoint: 'http://prom-new.example.com', + }) + .expect(200); + + const stored = await Connection.findById(created._id).select( + '+prometheusEndpoint', + ); + expect(stored?.prometheusEndpoint).toBe('http://prom-new.example.com'); + }); +}); diff --git a/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts b/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts new file mode 100644 index 0000000000..8397a3dece --- /dev/null +++ b/packages/api/src/routers/api/__tests__/prometheus.integration.test.ts @@ -0,0 +1,262 @@ +import { Types } from 'mongoose'; + +import * as config from '@/config'; +import { getAgent, getLoggedInAgent, getServer } from '@/fixtures'; +import Connection from '@/models/connection'; + +const mockFetch = global.fetch as jest.Mock; + +describe('prometheus router', () => { + const server = getServer(); + + beforeAll(async () => { + await server.start(); + }); + + afterEach(async () => { + await server.clearDBs(); + mockFetch.mockReset(); + mockFetch.mockResolvedValue({ + ok: true, + text: jest.fn().mockResolvedValue(''), + json: jest.fn().mockResolvedValue({}), + } as any); + }); + + afterAll(async () => { + await server.stop(); + }); + + const seedPrometheusConnection = async (teamId: Types.ObjectId) => { + return Connection.create({ + team: teamId, + name: 'Prom', + host: 'http://ch-server:8123', + username: 'default', + password: '', + prometheusEndpoint: 'http://prom.example.com', + }); + }; + + const seedClickHouseConnection = async (teamId: Types.ObjectId) => { + return Connection.create({ + team: teamId, + name: 'CH', + host: config.CLICKHOUSE_HOST, + username: config.CLICKHOUSE_USER, + password: config.CLICKHOUSE_PASSWORD, + }); + }; + + describe('GET /v1/prometheus/query_range', () => { + it('rejects unauthenticated requests with 401', async () => { + const anon = getAgent(server); + await anon.get('/v1/prometheus/query_range').expect(401); + }); + + it('returns 400 when query parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_range') + .query({ connectionId: new Types.ObjectId().toString() }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('query'), + }); + }); + + it('returns 400 when connectionId parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_range') + .query({ query: 'up' }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('connectionId'), + }); + }); + + it('returns 404 when connection does not exist', async () => { + const { agent } = await getLoggedInAgent(server); + const res = await agent + .get('/v1/prometheus/query_range') + .query({ + query: 'up', + connectionId: new Types.ObjectId().toString(), + }) + .expect(404); + expect(res.body).toMatchObject({ + status: 'error', + error: 'Connection not found', + }); + }); + + it('proxies to upstream Prometheus when connection has prometheusEndpoint', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + const promResponse = { + status: 'success', + data: { resultType: 'matrix', result: [] }, + }; + mockFetch.mockResolvedValueOnce({ + ok: true, + text: jest.fn().mockResolvedValue(''), + json: jest.fn().mockResolvedValue(promResponse), + } as any); + + const res = await agent + .get('/v1/prometheus/query_range') + .query({ + query: 'up', + start: '1700000000', + end: '1700000060', + step: '15s', + connectionId: conn._id.toString(), + }) + .expect(200); + + expect(res.body).toEqual(promResponse); + expect(mockFetch).toHaveBeenCalledTimes(1); + const calledUrl = mockFetch.mock.calls[0][0] as string; + expect(calledUrl).toContain('http://prom.example.com'); + expect(calledUrl).toContain('/api/v1/query_range'); + expect(calledUrl).toContain('query=up'); + expect(calledUrl).not.toContain('connectionId'); + }); + + it('does NOT proxy to Prometheus when connection has no prometheusEndpoint', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedClickHouseConnection(team._id); + + // ClickHouse path: will likely fail with 400 because otel_metrics_ts + // is not seeded in the test CH, but the routing decision is what we + // care about β€” fetch must not be called. + await agent.get('/v1/prometheus/query_range').query({ + query: 'up', + start: '1700000000', + end: '1700000060', + connectionId: conn._id.toString(), + }); + + expect(mockFetch).not.toHaveBeenCalled(); + }); + + it('returns 400 with Prometheus-compatible error when resolution exceeds 11,000 points', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedClickHouseConnection(team._id); + + const res = await agent + .get('/v1/prometheus/query_range') + .query({ + query: 'up', + start: '0', + end: '1700000000', + step: '1s', + connectionId: conn._id.toString(), + }) + .expect(400); + expect(res.body).toMatchObject({ + status: 'error', + errorType: 'bad_data', + error: expect.stringContaining('11,000 points'), + }); + expect(mockFetch).not.toHaveBeenCalled(); + }); + }); + + describe('GET /v1/prometheus/query', () => { + it('returns 400 when query parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + await agent + .get('/v1/prometheus/query') + .query({ connectionId: new Types.ObjectId().toString() }) + .expect(400); + }); + + it('returns 400 when connectionId parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + await agent + .get('/v1/prometheus/query') + .query({ query: 'up' }) + .expect(400); + }); + + it('returns 404 when connection does not exist', async () => { + const { agent } = await getLoggedInAgent(server); + await agent + .get('/v1/prometheus/query') + .query({ + query: 'up', + connectionId: new Types.ObjectId().toString(), + }) + .expect(404); + }); + + it('proxies to upstream Prometheus when connection has prometheusEndpoint', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + const promResponse = { + status: 'success', + data: { resultType: 'vector', result: [] }, + }; + mockFetch.mockResolvedValueOnce({ + ok: true, + text: jest.fn().mockResolvedValue(''), + json: jest.fn().mockResolvedValue(promResponse), + } as any); + + const res = await agent + .get('/v1/prometheus/query') + .query({ query: 'up', connectionId: conn._id.toString() }) + .expect(200); + + expect(res.body).toEqual(promResponse); + expect(mockFetch).toHaveBeenCalledTimes(1); + const calledUrl = mockFetch.mock.calls[0][0] as string; + expect(calledUrl).toContain('/api/v1/query'); + expect(calledUrl).not.toContain('/api/v1/query_range'); + }); + }); + + describe('GET /v1/prometheus/label/:name/values', () => { + it('returns 400 when connectionId parameter is missing', async () => { + const { agent } = await getLoggedInAgent(server); + await agent.get('/v1/prometheus/label/__name__/values').expect(400); + }); + + it('returns 404 when connection does not exist', async () => { + const { agent } = await getLoggedInAgent(server); + await agent + .get('/v1/prometheus/label/__name__/values') + .query({ connectionId: new Types.ObjectId().toString() }) + .expect(404); + }); + + it('proxies to upstream Prometheus when connection has prometheusEndpoint', async () => { + const { agent, team } = await getLoggedInAgent(server); + const conn = await seedPrometheusConnection(team._id); + + const promResponse = { status: 'success', data: ['up', 'requests'] }; + mockFetch.mockResolvedValueOnce({ + ok: true, + text: jest.fn().mockResolvedValue(''), + json: jest.fn().mockResolvedValue(promResponse), + } as any); + + const res = await agent + .get('/v1/prometheus/label/__name__/values') + .query({ connectionId: conn._id.toString() }) + .expect(200); + + expect(res.body).toEqual(promResponse); + const calledUrl = mockFetch.mock.calls[0][0] as string; + expect(calledUrl).toContain('/api/v1/label/__name__/values'); + }); + }); +}); diff --git a/packages/api/src/routers/api/__tests__/prometheus.test.ts b/packages/api/src/routers/api/__tests__/prometheus.test.ts new file mode 100644 index 0000000000..4f33aff312 --- /dev/null +++ b/packages/api/src/routers/api/__tests__/prometheus.test.ts @@ -0,0 +1,135 @@ +import { + formatMatrixResponse, + formatVectorResponse, + parseDuration, + parseTimestamp, +} from '../prometheus'; + +describe('parseTimestamp', () => { + it('returns numeric inputs unchanged', () => { + expect(parseTimestamp(1700000000)).toBe(1700000000); + expect(parseTimestamp(1700000000.5)).toBe(1700000000.5); + }); + + it('parses numeric strings as unix seconds', () => { + expect(parseTimestamp('1700000000')).toBe(1700000000); + expect(parseTimestamp('1700000000.5')).toBe(1700000000.5); + }); + + it('parses RFC3339 strings to unix seconds', () => { + expect(parseTimestamp('2023-11-14T22:13:20.000Z')).toBe(1700000000); + }); + + it('throws on unparseable input', () => { + expect(() => parseTimestamp('not-a-date')).toThrow(/Invalid timestamp/); + }); +}); + +describe('parseDuration', () => { + it('returns numeric inputs unchanged', () => { + expect(parseDuration(60)).toBe(60); + }); + + it('parses bare numeric strings as seconds', () => { + expect(parseDuration('60')).toBe(60); + }); + + it.each([ + ['500ms', 0.5], + ['30s', 30], + ['5m', 300], + ['2h', 7200], + ['1d', 86400], + ['1w', 604800], + ['1y', 31536000], + ])('parses %s to %d seconds', (input, expected) => { + expect(parseDuration(input)).toBe(expected); + }); + + it('parses fractional durations', () => { + expect(parseDuration('1.5h')).toBe(5400); + }); + + it('throws on invalid units', () => { + expect(() => parseDuration('5x')).toThrow(/Invalid duration/); + }); + + it('throws on garbage input', () => { + expect(() => parseDuration('abc')).toThrow(/Invalid duration/); + }); +}); + +describe('formatMatrixResponse', () => { + it('converts ClickHouse rows into Prometheus matrix shape', () => { + const rows = [ + { + tags: [ + ['__name__', 'http_requests_total'], + ['method', 'GET'], + ] as [string, string][], + time_series: [ + [1700000000, 5], + [1700000060, 7], + ] as [string | number, number][], + }, + ]; + expect(formatMatrixResponse(rows as any)).toEqual([ + { + metric: { __name__: 'http_requests_total', method: 'GET' }, + values: [ + [1700000000, '5'], + [1700000060, '7'], + ], + }, + ]); + }); + + it('converts string timestamps to unix seconds', () => { + const rows = [ + { + tags: [] as [string, string][], + time_series: [['2023-11-14T22:13:20.000Z', 1]] as [ + string | number, + number, + ][], + }, + ]; + expect(formatMatrixResponse(rows as any)[0].values[0]).toEqual([ + 1700000000, + '1', + ]); + }); + + it('returns empty array for empty input', () => { + expect(formatMatrixResponse([])).toEqual([]); + }); +}); + +describe('formatVectorResponse', () => { + it('converts ClickHouse rows into Prometheus vector shape', () => { + const rows = [ + { + tags: [['service', 'api']] as [string, string][], + timestamp: 1700000000 as unknown as string, + value: 42, + }, + ]; + expect(formatVectorResponse(rows as any)).toEqual([ + { metric: { service: 'api' }, value: [1700000000, '42'] }, + ]); + }); + + it('converts string timestamps to unix seconds', () => { + const rows = [ + { + tags: [] as [string, string][], + timestamp: '2023-11-14T22:13:20.000Z', + value: 3, + }, + ]; + expect(formatVectorResponse(rows as any)[0].value).toEqual([ + 1700000000, + '3', + ]); + }); +}); diff --git a/packages/api/src/routers/api/index.ts b/packages/api/src/routers/api/index.ts index 92df271296..9e7b0c0322 100644 --- a/packages/api/src/routers/api/index.ts +++ b/packages/api/src/routers/api/index.ts @@ -2,6 +2,7 @@ import aiRouter from './ai'; import alertsRouter from './alerts'; import dashboardRouter from './dashboards'; import meRouter from './me'; +import prometheusRouter from './prometheus'; import rootRouter from './root'; import teamRouter from './team'; import webhooksRouter from './webhooks'; @@ -11,6 +12,7 @@ export default { alertsRouter, dashboardRouter, meRouter, + prometheusRouter, rootRouter, teamRouter, webhooksRouter, diff --git a/packages/api/src/routers/api/prometheus.ts b/packages/api/src/routers/api/prometheus.ts new file mode 100644 index 0000000000..6fd095d7c9 --- /dev/null +++ b/packages/api/src/routers/api/prometheus.ts @@ -0,0 +1,451 @@ +import { ClickhouseClient } from '@hyperdx/common-utils/dist/clickhouse/node'; +import express from 'express'; + +import { getConnectionById } from '@/controllers/connection'; +import { getNonNullUserWithTeam } from '@/middleware/auth'; +import logger from '@/utils/logger'; + +const router = express.Router(); + +// Accept URL-encoded form bodies (Prometheus standard) and JSON +router.use(express.urlencoded({ extended: true })); + +// -------------------------- +// Param parsing helpers +// -------------------------- + +/** Parse a Prometheus timestamp: RFC3339 string or unix seconds (float) */ +export function parseTimestamp(value: string | number): number { + if (typeof value === 'number') return value; + const num = Number(value); + if (!isNaN(num)) return num; + const date = new Date(value); + if (isNaN(date.getTime())) throw new Error(`Invalid timestamp: ${value}`); + return date.getTime() / 1000; +} + +/** Parse a Prometheus duration string (e.g. "15s", "1m", "1h") to seconds */ +export function parseDuration(value: string | number): number { + if (typeof value === 'number') return value; + const num = Number(value); + if (!isNaN(num)) return num; + const match = value.match(/^(\d+(?:\.\d+)?)(ms|s|m|h|d|w|y)$/); + if (!match) throw new Error(`Invalid duration: ${value}`); + const n = parseFloat(match[1]); + switch (match[2]) { + case 'ms': + return n / 1000; + case 's': + return n; + case 'm': + return n * 60; + case 'h': + return n * 3600; + case 'd': + return n * 86400; + case 'w': + return n * 604800; + case 'y': + return n * 31536000; + default: + return n; + } +} + +/** Merge query params and body (supports both GET and POST) */ +function getParams(req: express.Request): Record { + return { + ...(req.query as Record), + ...(req.body as Record), + }; +} + +// -------------------------- +// Prometheus-compatible response types +// -------------------------- + +type PrometheusMetric = Record; +type PrometheusMatrixResult = { + metric: PrometheusMetric; + values: [number, string][]; +}; +type PrometheusVectorResult = { + metric: PrometheusMetric; + value: [number, string]; +}; + +// -------------------------- +// ClickHouse β†’ Prometheus response formatters +// -------------------------- + +export function formatMatrixResponse( + rows: { tags: [string, string][]; time_series: [string, number][] }[], +): PrometheusMatrixResult[] { + return rows.map(row => { + const metric: PrometheusMetric = {}; + for (const [key, value] of row.tags) { + metric[key] = value; + } + const values: [number, string][] = row.time_series.map( + ([timestamp, value]) => { + const ts = + typeof timestamp === 'string' + ? new Date(timestamp).getTime() / 1000 + : Number(timestamp); + return [ts, String(value)]; + }, + ); + return { metric, values }; + }); +} + +export function formatVectorResponse( + rows: { tags: [string, string][]; timestamp: string; value: number }[], +): PrometheusVectorResult[] { + return rows.map(row => { + const metric: PrometheusMetric = {}; + for (const [key, value] of row.tags) { + metric[key] = value; + } + const ts = + typeof row.timestamp === 'string' + ? new Date(row.timestamp).getTime() / 1000 + : Number(row.timestamp); + return { metric, value: [ts, String(row.value)] }; + }); +} + +// -------------------------- +// Prometheus proxy (for real Prometheus backends) +// -------------------------- + +const PROMETHEUS_PROXY_TIMEOUT_MS = 30_000; +const PROMETHEUS_CH_TIMEOUT_MS = 30_000; +const PROMETHEUS_MAX_EXECUTION_SEC = 30; +const PROMETHEUS_MAX_RESULT_ROWS = '100000'; +const PROMETHEUS_MAX_RESOLUTION = 11_000; + +async function proxyToPrometheus( + prometheusEndpoint: string, + path: string, + params: Record, +): Promise { + const url = new URL(path, prometheusEndpoint); + for (const [k, v] of Object.entries(params)) { + if (['connectionId', 'database', 'table'].includes(k)) continue; + if (v != null) url.searchParams.set(k, v); + } + let resp: Response; + try { + resp = await fetch(url.toString(), { + signal: AbortSignal.timeout(PROMETHEUS_PROXY_TIMEOUT_MS), + }); + } catch (err) { + if (err instanceof Error && err.name === 'TimeoutError') { + throw new Error( + `Prometheus request timed out after ${PROMETHEUS_PROXY_TIMEOUT_MS}ms`, + ); + } + throw err; + } + if (!resp.ok) { + const text = await resp.text(); + throw new Error(`Prometheus returned ${resp.status}: ${text}`); + } + return resp.json(); +} + +// -------------------------- +// GET|POST /query_range +// -------------------------- + +const queryRangeHandler: express.RequestHandler = async (req, res) => { + try { + const { teamId } = getNonNullUserWithTeam(req); + const params = getParams(req); + + const query = params.query; + if (!query) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: query', + }); + } + + const connectionId = params.connectionId; + if (!connectionId) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: connectionId', + }); + } + + // Resolve connection to determine backend (Prometheus or ClickHouse) + const connection = await getConnectionById( + teamId.toString(), + connectionId, + true, + ); + if (!connection) { + return res.status(404).json({ + status: 'error', + errorType: 'bad_data', + error: 'Connection not found', + }); + } + + // If connection has a Prometheus endpoint, proxy directly + if (connection.prometheusEndpoint) { + const result = await proxyToPrometheus( + connection.prometheusEndpoint, + '/api/v1/query_range', + params, + ); + return res.json(result); + } + + // Otherwise, use ClickHouse prometheusQuery() + const start = parseTimestamp(params.start); + const end = parseTimestamp(params.end); + const step = parseDuration(params.step ?? '60s'); + const database = params.database ?? 'default'; + const table = params.table ?? 'otel_metrics_ts'; + + if (step <= 0 || (end - start) / step > PROMETHEUS_MAX_RESOLUTION) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: `exceeded maximum resolution of ${PROMETHEUS_MAX_RESOLUTION.toLocaleString('en-US')} points per timeseries. Try decreasing the query resolution (?step=XX)`, + }); + } + + const client = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, + }); + + const startMs = Math.floor(start * 1000); + const endMs = Math.floor(end * 1000); + const stepSec = Math.max(Math.floor(step), 1); + + const resp = await client.query({ + query: `SELECT tags, time_series FROM prometheusQueryRange({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({startMs:Int64}), fromUnixTimestamp64Milli({endMs:Int64}), toIntervalSecond({stepSec:UInt32})) SETTINGS allow_experimental_time_series_table = 1`, + query_params: { + db: database, + table, + expr: query, + startMs, + endMs, + stepSec, + }, + format: 'JSON', + clickhouse_settings: { + allow_experimental_time_series_table: 1, + max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC, + max_result_rows: PROMETHEUS_MAX_RESULT_ROWS, + }, + }); + + const json = await resp.json(); + const result = formatMatrixResponse(json.data); + + return res.json({ + status: 'success', + data: { + resultType: 'matrix', + result, + }, + }); + } catch (e) { + logger.error(e, 'Prometheus query_range error'); + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: e instanceof Error ? e.message : String(e), + }); + } +}; +router.get('/query_range', queryRangeHandler); +router.post('/query_range', queryRangeHandler); + +// -------------------------- +// GET|POST /query +// -------------------------- + +const queryHandler: express.RequestHandler = async (req, res) => { + try { + const { teamId } = getNonNullUserWithTeam(req); + const params = getParams(req); + + const query = params.query; + if (!query) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: query', + }); + } + + const connectionId = params.connectionId; + if (!connectionId) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: connectionId', + }); + } + + const connection = await getConnectionById( + teamId.toString(), + connectionId, + true, + ); + if (!connection) { + return res.status(404).json({ + status: 'error', + errorType: 'bad_data', + error: 'Connection not found', + }); + } + + if (connection.prometheusEndpoint) { + const result = await proxyToPrometheus( + connection.prometheusEndpoint, + '/api/v1/query', + params, + ); + return res.json(result); + } + + const time = params.time ? parseTimestamp(params.time) : undefined; + const database = params.database ?? 'default'; + const table = params.table ?? 'otel_metrics_ts'; + + const client = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, + }); + + const evalMs = time ? Math.floor(time * 1000) : Date.now(); + + const resp = await client.query({ + query: `SELECT tags, timestamp, value FROM prometheusQuery({db:String}, {table:String}, {expr:String}, fromUnixTimestamp64Milli({evalMs:Int64})) SETTINGS allow_experimental_time_series_table = 1`, + query_params: { db: database, table, expr: query, evalMs }, + format: 'JSON', + clickhouse_settings: { + allow_experimental_time_series_table: 1, + max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC, + max_result_rows: PROMETHEUS_MAX_RESULT_ROWS, + }, + }); + + const json = await resp.json(); + const result = formatVectorResponse(json.data); + + return res.json({ + status: 'success', + data: { + resultType: 'vector', + result, + }, + }); + } catch (e) { + logger.error(e, 'Prometheus query error'); + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: e instanceof Error ? e.message : String(e), + }); + } +}; +router.get('/query', queryHandler); +router.post('/query', queryHandler); + +// -------------------------- +// GET /label/:name/values +// -------------------------- + +router.get('/label/:name/values', async (req, res) => { + try { + const { teamId } = getNonNullUserWithTeam(req); + const labelName = req.params.name; + const params = req.query as Record; + + const connectionId = params.connectionId; + if (!connectionId) { + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: 'missing required parameter: connectionId', + }); + } + + const connection = await getConnectionById( + teamId.toString(), + connectionId, + true, + ); + if (!connection) { + return res.status(404).json({ + status: 'error', + errorType: 'bad_data', + error: 'Connection not found', + }); + } + + // Proxy to Prometheus if endpoint is set + if (connection.prometheusEndpoint) { + const result = await proxyToPrometheus( + connection.prometheusEndpoint, + `/api/v1/label/${labelName}/values`, + params, + ); + return res.json(result); + } + + const database = params.database ?? 'default'; + const table = params.table ?? 'otel_metrics_ts'; + + const client = new ClickhouseClient({ + host: connection.host, + username: connection.username, + password: connection.password, + requestTimeout: PROMETHEUS_CH_TIMEOUT_MS, + }); + + const tagsQuery = + labelName === '__name__' + ? `SELECT DISTINCT metric_name AS val FROM timeSeriesTags({db:String}, {table:String}) ORDER BY val SETTINGS allow_experimental_time_series_table = 1` + : `SELECT DISTINCT all_tags[{label:String}] AS val FROM timeSeriesTags({db:String}, {table:String}) WHERE mapContains(all_tags, {label:String}) ORDER BY val SETTINGS allow_experimental_time_series_table = 1`; + + const resp = await client.query({ + query: tagsQuery, + query_params: { db: database, table, label: labelName }, + format: 'JSON', + clickhouse_settings: { + allow_experimental_time_series_table: 1, + max_execution_time: PROMETHEUS_MAX_EXECUTION_SEC, + max_result_rows: PROMETHEUS_MAX_RESULT_ROWS, + }, + }); + const json = await resp.json(); + const values: string[] = json.data.map((r: any) => r.val); + + return res.json({ status: 'success', data: values }); + } catch (e) { + logger.error(e, 'Prometheus label values error'); + return res.status(400).json({ + status: 'error', + errorType: 'bad_data', + error: e instanceof Error ? e.message : String(e), + }); + } +}); + +export default router; diff --git a/packages/api/src/routers/external-api/v2/sources.ts b/packages/api/src/routers/external-api/v2/sources.ts index 7c05cb0ec0..0903dccf0e 100644 --- a/packages/api/src/routers/external-api/v2/sources.ts +++ b/packages/api/src/routers/external-api/v2/sources.ts @@ -79,6 +79,8 @@ function formatExternalSource(source: SourceDocument) { return source.toJSON({ getters: true }); case SourceKind.Session: return source.toJSON({ getters: true }); + case SourceKind.Promql: + return source.toJSON({ getters: true }); default: source satisfies never; return {}; diff --git a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts index 3a67c0f7a2..e49dd6a7e6 100644 --- a/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts +++ b/packages/api/src/routers/external-api/v2/utils/__tests__/dashboards.test.ts @@ -380,4 +380,24 @@ describe('convertToExternalDashboard orphan-ref heal', () => { expect(ext.tiles[0].containerId).toBe('real'); expect(ext.tiles[0].tabId).toBe('errors'); }); + + it('drops PromQL tiles from external response (no schema variant yet)', () => { + const doc = makeDoc({ + tiles: [ + makeTile({ + id: 'promql-tile', + config: { + configType: 'promql', + promqlExpression: 'up', + connection: 'conn-1', + displayType: DisplayType.Line, + name: 'My PromQL tile', + } as any, + }), + makeTile({ id: 'normal-tile' }), + ], + }); + const ext = convertToExternalDashboard(doc); + expect(ext.tiles.map(t => t.id)).toEqual(['normal-tile']); + }); }); diff --git a/packages/api/src/routers/external-api/v2/utils/dashboards.ts b/packages/api/src/routers/external-api/v2/utils/dashboards.ts index 2d5a2af6e8..25c59b35f2 100644 --- a/packages/api/src/routers/external-api/v2/utils/dashboards.ts +++ b/packages/api/src/routers/external-api/v2/utils/dashboards.ts @@ -5,6 +5,7 @@ import { } from '@hyperdx/common-utils/dist/dashboardValidation'; import { isHeatmapCompatibleSource, + isPromqlSavedChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { @@ -213,6 +214,11 @@ const convertToExternalTileChartConfig = ( return undefined; } + // PromQL configs are not yet supported in the external API + if (isPromqlSavedChartConfig(config)) { + return undefined; + } + const sourceId = config.source?.toString() ?? ''; const stringValueOrDefault = ( @@ -369,6 +375,18 @@ function convertTileToExternalChart( containerById: Map, dashboardId: string, ): ExternalDashboardTileWithId | undefined { + // PromQL tiles have no external schema representation yet. Dropping them on + // read (and letting the caller filter undefined) is safer than falling + // through to defaultTileConfig β€” that would silently overwrite the PromQL + // config with an empty Line tile on a GET β†’ PUT round-trip. + if (isPromqlSavedChartConfig(tile.config)) { + logger.warn( + { dashboardId, tileId: tile.id }, + 'Skipping PromQL tile in external API response (not yet supported)', + ); + return undefined; + } + // Returned in case of a failure converting the saved chart config const defaultTileConfig: ExternalDashboardTileConfig = isRawSqlSavedChartConfig(tile.config) diff --git a/packages/api/src/tasks/checkAlerts/index.ts b/packages/api/src/tasks/checkAlerts/index.ts index 5d0649462a..031b770e45 100644 --- a/packages/api/src/tasks/checkAlerts/index.ts +++ b/packages/api/src/tasks/checkAlerts/index.ts @@ -27,6 +27,7 @@ import { timeBucketByGranularity } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderChartConfig, isBuilderSavedChartConfig, + isPromqlSavedChartConfig, isRawSqlChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; @@ -569,6 +570,11 @@ const getChartConfigFromAlert = ( return undefined; } + // PromQL tiles don't support alerts yet + if (isPromqlSavedChartConfig(tile.config)) { + return undefined; + } + const { source } = details; if (!source) { logger.error( diff --git a/packages/app/.env.development b/packages/app/.env.development index a673fdd0d9..7fd59cdfde 100644 --- a/packages/app/.env.development +++ b/packages/app/.env.development @@ -11,4 +11,5 @@ NEXT_PUBLIC_OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:${HDX_DEV_OTEL_HTTP_PO # Per-signal OTLP endpoints (override the base endpoint for individual signals) # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_TRACES_ENDPOINT= # NEXT_PUBLIC_OTEL_EXPORTER_OTLP_METRICS_ENDPOINT= -# NEXT_PUBLIC_OTEL_EXPORTER_OTLP_LOGS_ENDPOINT= \ No newline at end of file +# NEXT_PUBLIC_OTEL_EXPORTER_OTLP_LOGS_ENDPOINT= +NEXT_PUBLIC_ENABLE_PROMQL=true diff --git a/packages/app/package.json b/packages/app/package.json index 6c4bf6c11a..931a238491 100644 --- a/packages/app/package.json +++ b/packages/app/package.json @@ -47,6 +47,7 @@ "@mantine/hooks": "^9.0.0", "@mantine/notifications": "^9.0.0", "@mantine/spotlight": "^9.0.0", + "@prometheus-io/codemirror-promql": "^0.311.3", "@tabler/icons-react": "^3.39.0", "@tanstack/react-query": "^5.56.2", "@tanstack/react-query-devtools": "^5.56.2", diff --git a/packages/app/src/DBDashboardPage.tsx b/packages/app/src/DBDashboardPage.tsx index c911a30321..690535da6f 100644 --- a/packages/app/src/DBDashboardPage.tsx +++ b/packages/app/src/DBDashboardPage.tsx @@ -29,6 +29,7 @@ import { displayTypeRequiresSource, isBuilderChartConfig, isBuilderSavedChartConfig, + isPromqlSavedChartConfig, isRawSqlChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; @@ -463,6 +464,19 @@ const Tile = forwardRef( !chart.config.source; useEffect(() => { + if (isPromqlSavedChartConfig(chart.config)) { + if (source != null) { + setQueriedConfig({ + ...chart.config, + from: source.from, + connection: source.connection, + dateRange, + granularity, + }); + } + return; + } + if (isRawSqlSavedChartConfig(chart.config)) { // Some raw SQL charts don't have a source if (!chart.config.source) { diff --git a/packages/app/src/api.ts b/packages/app/src/api.ts index dc76cf1fc3..f67574b18d 100644 --- a/packages/app/src/api.ts +++ b/packages/app/src/api.ts @@ -550,3 +550,86 @@ const api = { }, }; export default api; + +// -------------------------- +// Prometheus API +// -------------------------- +type PrometheusMetric = Record; +type PrometheusMatrixResult = { + metric: PrometheusMetric; + values: [number, string][]; +}; +type PrometheusQueryRangeResponse = { + status: 'success' | 'error'; + data?: { + resultType: 'matrix'; + result: PrometheusMatrixResult[]; + }; + error?: string; +}; +type PrometheusLabelValuesResponse = { + status: 'success' | 'error'; + data?: string[]; + error?: string; +}; + +async function prometheusFetch( + path: string, + searchParams: Record, +): Promise { + try { + return await server.post(path, { searchParams }).json(); + } catch (e: any) { + // ky throws HTTPError on non-2xx β€” read the response body for the real error + if (e?.response) { + try { + const body = await e.response.json(); + if (body?.error) { + throw new Error(body.error); + } + } catch (parseErr) { + if (parseErr instanceof Error && parseErr.message !== e.message) { + throw parseErr; + } + } + } + throw e; + } +} + +export const prometheusApi = { + queryRange: (params: { + query: string; + start: number; + end: number; + step: string; + connectionId: string; + database: string; + table: string; + }): Promise => + prometheusFetch('v1/prometheus/query_range', { + query: params.query, + start: String(params.start), + end: String(params.end), + step: params.step, + connectionId: params.connectionId, + database: params.database, + table: params.table, + }), + + labelValues: (params: { + label: string; + connectionId: string; + database?: string; + table?: string; + }): Promise => + server + .get(`v1/prometheus/label/${params.label}/values`, { + searchParams: { + connectionId: params.connectionId, + ...(params.database ? { database: params.database } : {}), + ...(params.table ? { table: params.table } : {}), + }, + }) + .json(), +}; diff --git a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx index 13fe0092e3..7d3b989c77 100644 --- a/packages/app/src/components/ChartDisplaySettingsDrawer.tsx +++ b/packages/app/src/components/ChartDisplaySettingsDrawer.tsx @@ -41,7 +41,7 @@ interface ChartDisplaySettingsDrawerProps { defaultNumberFormat?: NumberFormat; displayType: DisplayType; /** 'sql' for raw SQL chart configs; anything else is treated as a builder config. */ - configType?: 'sql' | 'builder'; + configType?: 'sql' | 'builder' | 'promql'; previousDateRange?: [Date, Date]; onChange: (settings: ChartConfigDisplaySettings) => void; onClose: () => void; diff --git a/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx new file mode 100644 index 0000000000..1b6a7b346d --- /dev/null +++ b/packages/app/src/components/ChartEditor/PromqlChartEditor.tsx @@ -0,0 +1,71 @@ +import { Control, useController, useWatch } from 'react-hook-form'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; +import { Box, Button, Flex, Stack, Text } from '@mantine/core'; + +import PromQLEditor from '@/components/PromQLEditor/PromQLEditor'; +import { SourceSelectControlled } from '@/components/SourceSelect'; +import { usePromqlMetricNames } from '@/hooks/usePromqlMetadata'; +import { useSource } from '@/source'; + +import { ChartEditorFormState } from './types'; + +export default function PromqlChartEditor({ + control, + onSubmit, + onOpenDisplaySettings, +}: { + control: Control; + onSubmit: (suppressErrorNotification?: boolean) => void; + onOpenDisplaySettings: () => void; +}) { + const { field: expressionField } = useController({ + control, + name: 'promqlExpression', + }); + + const sourceId = useWatch({ control, name: 'source' }); + const { data: source } = useSource({ id: sourceId }); + const connectionId = source?.connection; + const { data: metricNames } = usePromqlMetricNames( + connectionId, + source?.from.databaseName, + source?.from.tableName, + ); + + return ( + + + + Data Source + + + + + + PromQL Expression + + onSubmit()} + placeholder="rate(http_requests_total{service='api'}[5m])" + metricNames={metricNames} + /> + + + + + + ); +} diff --git a/packages/app/src/components/ChartEditor/types.ts b/packages/app/src/components/ChartEditor/types.ts index 8bc14b785a..4e22b22d3c 100644 --- a/packages/app/src/components/ChartEditor/types.ts +++ b/packages/app/src/components/ChartEditor/types.ts @@ -1,5 +1,6 @@ import { BuilderSavedChartConfig, + PromqlSavedChartConfig, RawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/types'; @@ -14,22 +15,23 @@ export type SavedChartConfigWithSelectArray = Omit< /** * A type that flattens the SavedChartConfig union so that the form can include - * properties from both BuilderChartConfig and RawSqlSavedChartConfig without - * type errors. + * properties from both BuilderChartConfig, RawSqlSavedChartConfig, and + * PromqlSavedChartConfig without type errors. * - * All fields are optional since the form may be in either builder or raw SQL - * mode at any given time. `configType?: 'sql'` is the discriminator. + * All fields are optional since the form may be in builder, raw SQL, or PromQL + * mode at any given time. `configType` is the discriminator. * * Additionally, 'series' is added as a separate field that is always an array, * to work around the fact that useFieldArray only works with fields which are *always* * arrays. `series` stores the array `select` data for the form. **/ export type ChartEditorFormState = Partial & - Partial> & { + Partial> & + Partial> & { alert?: BuilderSavedChartConfig['alert'] & { id?: string; createdBy?: AlertWithCreatedBy['createdBy']; }; series: SavedChartConfigWithSelectArray['select']; - configType?: 'sql' | 'builder'; + configType?: 'sql' | 'builder' | 'promql'; }; diff --git a/packages/app/src/components/ChartEditor/utils.ts b/packages/app/src/components/ChartEditor/utils.ts index 39a68c8bc8..54907b8874 100644 --- a/packages/app/src/components/ChartEditor/utils.ts +++ b/packages/app/src/components/ChartEditor/utils.ts @@ -3,6 +3,7 @@ import { Path, UseFormSetError } from 'react-hook-form'; import { validateRawSqlForAlert } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderSavedChartConfig, + isPromqlSavedChartConfig, isRawSqlSavedChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { @@ -14,6 +15,8 @@ import { isMetricSource, isRangeThresholdType, isTraceSource, + PromqlChartConfig, + PromqlSavedChartConfig, RawSqlChartConfig, RawSqlSavedChartConfig, SavedChartConfig, @@ -70,6 +73,28 @@ export function convertFormStateToSavedChartConfig( form: ChartEditorFormState, source: TSource | undefined, ): SavedChartConfig | undefined { + if (form.configType === 'promql') { + const promqlConfig: PromqlSavedChartConfig = { + configType: 'promql', + ...pick(form, [ + 'name', + 'displayType', + 'numberFormat', + 'granularity', + 'compareToPreviousPeriod', + 'fillNulls', + 'alignDateRangeToGranularity', + 'alert', + 'step', + ]), + promqlExpression: form.promqlExpression ?? '', + connection: form.connection ?? '', + source: form.source || undefined, + }; + + return promqlConfig; + } + if (form.configType === 'sql' && isRawSqlDisplayType(form.displayType)) { const rawSqlConfig: RawSqlSavedChartConfig = { configType: 'sql', @@ -115,6 +140,27 @@ export function convertFormStateToChartConfig( dateRange: ChartConfigWithDateRange['dateRange'], source: TSource | undefined, ): ChartConfigWithDateRange | undefined { + if (form.configType === 'promql') { + const promqlConfig: PromqlChartConfig = { + configType: 'promql', + ...pick(form, [ + 'displayType', + 'numberFormat', + 'granularity', + 'compareToPreviousPeriod', + 'fillNulls', + 'alignDateRangeToGranularity', + 'step', + ]), + promqlExpression: form.promqlExpression ?? '', + connection: source?.connection ?? form.connection ?? '', + source: form.source || undefined, + from: source?.from, + }; + + return { ...promqlConfig, dateRange }; + } + if (form.configType === 'sql' && isRawSqlDisplayType(form.displayType)) { const rawSqlConfig: RawSqlChartConfig = { configType: 'sql', @@ -186,7 +232,11 @@ export function convertSavedChartConfigToFormState( ): ChartEditorFormState { return { ...config, - configType: isRawSqlSavedChartConfig(config) ? 'sql' : 'builder', + configType: isPromqlSavedChartConfig(config) + ? 'promql' + : isRawSqlSavedChartConfig(config) + ? 'sql' + : 'builder', series: isBuilderSavedChartConfig(config) && Array.isArray(config.select) ? config.select.map(s => ({ diff --git a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx index 1b5409dc32..dea9438aad 100644 --- a/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx +++ b/packages/app/src/components/DBEditTimeChartForm/EditTimeChartForm.tsx @@ -45,6 +45,7 @@ import { getPreviousDateRange } from '@/ChartUtils'; import ChartDisplaySettingsDrawer, { ChartConfigDisplaySettings, } from '@/components/ChartDisplaySettingsDrawer'; +import PromqlChartEditor from '@/components/ChartEditor/PromqlChartEditor'; import RawSqlChartEditor from '@/components/ChartEditor/RawSqlChartEditor'; import { ChartEditorFormState, @@ -65,6 +66,7 @@ import HeatmapSettingsDrawer, { import { InputControlled } from '@/components/InputControlled'; import SaveToDashboardModal from '@/components/SaveToDashboardModal'; import { getStoredLanguage } from '@/components/SearchInput/SearchWhereInput'; +import { IS_PROMQL_ENABLED } from '@/config'; import HDXMarkdownChart from '@/HDXMarkdownChart'; import { getDurationMsExpression, @@ -188,6 +190,7 @@ export default function EditTimeChartForm({ const chartConfigAlert = chartConfig.alert; const isRawSqlInput = configType === 'sql' && isRawSqlDisplayType(displayType); + const isPromqlInput = configType === 'promql'; const { data: tableSource } = useSource({ id: sourceId }); const databaseName = tableSource?.from.databaseName; @@ -208,10 +211,11 @@ export default function EditTimeChartForm({ } }, [configType, displayType, previousDisplayType, setValue]); - const showGeneratedSql = TABS_WITH_GENERATED_SQL.has(activeTab); + const showGeneratedSql = + TABS_WITH_GENERATED_SQL.has(activeTab) && !isPromqlInput; const showSampleEvents = - tableSource?.kind !== SourceKind.Metric && !isRawSqlInput; + tableSource?.kind !== SourceKind.Metric && !isRawSqlInput && !isPromqlInput; const [ alignDateRangeToGranularity, @@ -666,6 +670,9 @@ export default function EditTimeChartForm({ data={[ { label: 'Builder', value: 'builder' }, { label: 'SQL', value: 'sql' }, + ...(IS_PROMQL_ENABLED + ? [{ label: 'PromQL', value: 'promql' }] + : []), ]} /> )} @@ -694,6 +701,12 @@ export default function EditTimeChartForm({ /> + ) : isPromqlInput ? ( + ) : isRawSqlInput ? ( { if (!queriedConfig) return false; + if (isPromqlChartConfig(queriedConfig)) { + return !!(queriedConfig.promqlExpression && queriedConfig.connection); + } if (isRawSqlChartConfig(queriedConfig)) { return !!(queriedConfig.sqlTemplate && queriedConfig.connection); } @@ -64,7 +69,11 @@ export function seriesToFilters(select: SelectList): Filter[] { const filters: Filter[] = select .map(({ aggCondition, aggConditionLanguage }) => { - if (aggConditionLanguage != null && aggCondition != null) { + if ( + aggConditionLanguage != null && + aggConditionLanguage !== 'promql' && + aggCondition != null + ) { return { type: aggConditionLanguage, condition: aggCondition, @@ -216,7 +225,7 @@ export function buildChartConfigForExplanations({ } : undefined; - if (!config || isRawSqlChartConfig(config)) { + if (!config || isRawSqlChartConfig(config) || isPromqlChartConfig(config)) { return undefined; } @@ -227,14 +236,16 @@ export function buildChartConfigForExplanations({ // other at runtime, so the SQL preview transforms `config` itself into // both queries on render and the MV indicator is suppressed for this // tab. Returning `config` unchanged is intentional. + const builderConfig = config as BuilderChartConfigWithDateRange; + if (activeTab === 'time') { - return convertToTimeChartConfig(config); + return convertToTimeChartConfig(builderConfig); } else if (activeTab === 'number') { - return convertToNumberChartConfig(config); + return convertToNumberChartConfig(builderConfig); } else if (activeTab === 'table') { - return convertToTableChartConfig(config); + return convertToTableChartConfig(builderConfig); } else if (activeTab === 'pie') { - return convertToPieChartConfig(config); + return convertToPieChartConfig(builderConfig); } else if (activeTab === 'heatmap') { return config; } diff --git a/packages/app/src/components/DBNumberChart.tsx b/packages/app/src/components/DBNumberChart.tsx index a1f9d8e189..f63049c157 100644 --- a/packages/app/src/components/DBNumberChart.tsx +++ b/packages/app/src/components/DBNumberChart.tsx @@ -7,10 +7,7 @@ import { isBuilderChartConfig, isRawSqlChartConfig, } from '@hyperdx/common-utils/dist/guards'; -import { - BuilderChartConfigWithDateRange, - RawSqlConfigWithDateRange, -} from '@hyperdx/common-utils/dist/types'; +import { ChartConfigWithDateRange } from '@hyperdx/common-utils/dist/types'; import { Flex, Text } from '@mantine/core'; import { @@ -38,7 +35,7 @@ export default function DBNumberChart({ showMVOptimizationIndicator = true, errorVariant, }: { - config: BuilderChartConfigWithDateRange | RawSqlConfigWithDateRange; + config: ChartConfigWithDateRange; queryKeyPrefix?: string; enabled?: boolean; title?: React.ReactNode; diff --git a/packages/app/src/components/DBPieChart.tsx b/packages/app/src/components/DBPieChart.tsx index 7177d9d3c9..11c0fd8702 100644 --- a/packages/app/src/components/DBPieChart.tsx +++ b/packages/app/src/components/DBPieChart.tsx @@ -1,10 +1,7 @@ import { memo, useMemo } from 'react'; import { Cell, Pie, PieChart, ResponsiveContainer, Tooltip } from 'recharts'; import { isBuilderChartConfig } from '@hyperdx/common-utils/dist/guards'; -import { - BuilderChartConfigWithOptTimestamp, - RawSqlConfigWithDateRange, -} from '@hyperdx/common-utils/dist/types'; +import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; import { Box, Flex, ScrollArea, Text } from '@mantine/core'; import { @@ -109,7 +106,7 @@ export const DBPieChart = ({ toolbarSuffix, errorVariant, }: { - config: BuilderChartConfigWithOptTimestamp | RawSqlConfigWithDateRange; + config: ChartConfigWithOptTimestamp; title?: React.ReactNode; enabled?: boolean; queryKeyPrefix?: string; diff --git a/packages/app/src/components/DBRowDataPanel.tsx b/packages/app/src/components/DBRowDataPanel.tsx index 0ef8525091..2a38f2d105 100644 --- a/packages/app/src/components/DBRowDataPanel.tsx +++ b/packages/app/src/components/DBRowDataPanel.tsx @@ -114,7 +114,8 @@ export function useRowData({ }, ] : []), - ...(source.resourceAttributesExpression + ...('resourceAttributesExpression' in source && + source.resourceAttributesExpression ? [ { valueExpression: source.resourceAttributesExpression, diff --git a/packages/app/src/components/DBRowOverviewPanel.tsx b/packages/app/src/components/DBRowOverviewPanel.tsx index 1ef782c34d..ab372d031b 100644 --- a/packages/app/src/components/DBRowOverviewPanel.tsx +++ b/packages/app/src/components/DBRowOverviewPanel.tsx @@ -107,7 +107,7 @@ export function RowOverviewPanel({ ); const _generateSearchUrl = useCallback( - (query?: string, queryLanguage?: 'sql' | 'lucene') => { + (query?: string, queryLanguage?: 'sql' | 'lucene' | 'promql') => { return ( generateSearchUrl?.({ where: query, @@ -313,13 +313,16 @@ export function RowOverviewPanel({ ? { onPropertyAddClick, sqlExpression: + 'resourceAttributesExpression' in source && source.resourceAttributesExpression && jsonColumns?.includes( source.resourceAttributesExpression, ) ? // If resource attributes is a JSON column, we need to cast the key to a string so we can run where X in Y queries `toString(${source.resourceAttributesExpression}.${key})` - : `${source.resourceAttributesExpression}['${key}']`, + : 'resourceAttributesExpression' in source + ? `${source.resourceAttributesExpression}['${key}']` + : '', } : { onPropertyAddClick: undefined, @@ -329,7 +332,7 @@ export function RowOverviewPanel({ generateSearchUrl ? _generateSearchUrl : undefined } displayedKey={key} - name={`${source.resourceAttributesExpression}.${key}`} + name={`${'resourceAttributesExpression' in source ? source.resourceAttributesExpression : ''}.${key}`} value={value as string} key={key} /> diff --git a/packages/app/src/components/DBRowSidePanel.tsx b/packages/app/src/components/DBRowSidePanel.tsx index 9efff30ed3..1bd6dcde7d 100644 --- a/packages/app/src/components/DBRowSidePanel.tsx +++ b/packages/app/src/components/DBRowSidePanel.tsx @@ -170,6 +170,8 @@ const DBRowSidePanel = ({ source.resourceAttributesExpression ) { return true; + } else if (source.kind === SourceKind.Promql) { + return false; } return false; }, [source]); @@ -289,7 +291,11 @@ const DBRowSidePanel = ({ const hasK8sContext = useMemo(() => { try { - if (!source?.resourceAttributesExpression || !normalizedRow) { + if ( + !('resourceAttributesExpression' in source) || + !source?.resourceAttributesExpression || + !normalizedRow + ) { return false; } diff --git a/packages/app/src/components/DBTableChart.tsx b/packages/app/src/components/DBTableChart.tsx index 3862424c20..44d1f466bb 100644 --- a/packages/app/src/components/DBTableChart.tsx +++ b/packages/app/src/components/DBTableChart.tsx @@ -4,6 +4,7 @@ import { ClickHouseQueryError } from '@hyperdx/common-utils/dist/clickhouse'; import { isRatioChartConfig } from '@hyperdx/common-utils/dist/core/renderChartConfig'; import { isBuilderChartConfig, + isPromqlChartConfig, isRawSqlChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { ChartConfigWithOptTimestamp } from '@hyperdx/common-utils/dist/types'; @@ -76,6 +77,7 @@ export default function DBTableChart({ const queriedConfig = useMemo(() => { if (isRawSqlChartConfig(config)) return config; + if (isPromqlChartConfig(config)) return config; const _config = convertToTableChartConfig(config); @@ -103,7 +105,7 @@ export default function DBTableChart({ // Returns an array of aliases, so we can check if something is using an alias const aliasMap = useMemo(() => { - if (isRawSqlChartConfig(config)) { + if (isRawSqlChartConfig(config) || isPromqlChartConfig(config)) { return []; } diff --git a/packages/app/src/components/DBTimeChart.tsx b/packages/app/src/components/DBTimeChart.tsx index 3f6a5725b1..d0d4dfa9e1 100644 --- a/packages/app/src/components/DBTimeChart.tsx +++ b/packages/app/src/components/DBTimeChart.tsx @@ -7,6 +7,7 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderChartConfig, + isPromqlChartConfig, isRawSqlChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { @@ -483,7 +484,8 @@ function DBTimeChartComponent({ if ( clickedActiveLabelDate == null || source == null || - isRawSqlChartConfig(config) + isRawSqlChartConfig(config) || + isPromqlChartConfig(config) ) { return null; } diff --git a/packages/app/src/components/PromQLEditor/PromQLEditor.tsx b/packages/app/src/components/PromQLEditor/PromQLEditor.tsx new file mode 100644 index 0000000000..a99cb4ad09 --- /dev/null +++ b/packages/app/src/components/PromQLEditor/PromQLEditor.tsx @@ -0,0 +1,177 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import cx from 'classnames'; +import { + acceptCompletion, + autocompletion, + Completion, + startCompletion, +} from '@codemirror/autocomplete'; +import { Paper, useMantineColorScheme } from '@mantine/core'; +import { PromQLExtension } from '@prometheus-io/codemirror-promql'; +import CodeMirror, { + Compartment, + EditorView, + keymap, + Prec, + ReactCodeMirrorRef, +} from '@uiw/react-codemirror'; + +import { + createCodeMirrorStyleTheme, + DEFAULT_CODE_MIRROR_BASIC_SETUP, +} from '@/components/SQLEditor/utils'; + +import styles from '../SQLEditor/SQLInlineEditor.module.scss'; + +const promqlExtension = new PromQLExtension(); + +type PromQLEditorProps = { + value: string; + onChange: (value: string) => void; + placeholder?: string; + onSubmit?: () => void; + metricNames?: string[]; +}; + +const MAX_EDITOR_HEIGHT = '150px'; + +export default function PromQLEditor({ + value, + onChange, + placeholder, + onSubmit, + metricNames, +}: PromQLEditorProps) { + const { colorScheme } = useMantineColorScheme(); + const ref = useRef(null); + const compartmentRef = useRef(new Compartment()); + const [isFocused, setIsFocused] = useState(false); + + const metricCompletions = useMemo( + () => + (metricNames ?? []).map(name => ({ + label: name, + type: 'variable' as const, + })), + [metricNames], + ); + + const updateAutocomplete = useCallback( + (viewRef: EditorView) => { + if (metricCompletions.length > 0) { + viewRef.dispatch({ + effects: compartmentRef.current.reconfigure( + autocompletion({ + override: [ + context => { + const word = context.matchBefore(/[\w.:]+/); + if (!word && !context.explicit) return null; + return { + from: word?.from ?? context.pos, + options: metricCompletions, + }; + }, + ], + }), + ), + }); + } + }, + [metricCompletions], + ); + + useEffect(() => { + if (ref.current?.view) { + updateAutocomplete(ref.current.view); + } + }, [updateAutocomplete]); + + const cmExtensions = useMemo( + () => [ + createCodeMirrorStyleTheme(MAX_EDITOR_HEIGHT), + EditorView.lineWrapping, + + // PromQL syntax highlighting + promqlExtension.asExtension(), + + // Metric name autocomplete (via compartment for hot-swapping) + // eslint-disable-next-line react-hooks/refs + compartmentRef.current.of([]), + + // Enter to submit, Shift+Enter for newline + Prec.highest( + keymap.of([ + { + key: 'Enter', + run: () => { + onSubmit?.(); + return true; + }, + }, + { + key: 'Shift-Enter', + run: () => false, + }, + ]), + ), + keymap.of([ + { + key: 'Tab', + run: acceptCompletion, + }, + ]), + ], + [onSubmit], + ); + + const onClickCodeMirror = useCallback(() => { + if (ref?.current?.view) { + startCompletion(ref.current.view); + } + }, []); + + const isExpanded = isFocused; + const baseHeight = 36; + + return ( +
+ {isExpanded && + ); +} diff --git a/packages/app/src/components/Sources/SourceForm.tsx b/packages/app/src/components/Sources/SourceForm.tsx index 273eef386b..7544b14bc1 100644 --- a/packages/app/src/components/Sources/SourceForm.tsx +++ b/packages/app/src/components/Sources/SourceForm.tsx @@ -54,7 +54,11 @@ import { import { SourceSelectControlled } from '@/components/SourceSelect'; import { SQLInlineEditorControlled } from '@/components/SQLEditor/SQLInlineEditor'; -import { IS_METRICS_ENABLED, IS_SESSIONS_ENABLED } from '@/config'; +import { + IS_METRICS_ENABLED, + IS_PROMQL_ENABLED, + IS_SESSIONS_ENABLED, +} from '@/config'; import { useConnections } from '@/connection'; import { useExplainQuery } from '@/hooks/useExplainQuery'; import { useMetadataWithSettings } from '@/hooks/useMetadata'; @@ -147,6 +151,8 @@ function setCorrelationFieldValue( return { ...source, [field]: value }; } return source; + case SourceKind.Promql: + return source; } } @@ -199,6 +205,7 @@ const CORRELATION_FIELD_MAP: Record< { targetKind: SourceKind.Log, targetField: 'metricSourceId' }, ], }, + [SourceKind.Promql]: {}, }; function FormRow({ @@ -1897,6 +1904,19 @@ function MetricTableModelForm({ control, setValue }: TableModelProps) { ); } +function PromqlTableModelForm({ + control: _control, + setValue, +}: TableModelProps) { + useEffect(() => { + setValue('timestampValueExpression' as any, 'timestamp'); + }, [setValue]); + + // PromQL sources use the standard database + table fields from BaseSourceSchema. + // No additional fields needed β€” the table should point to the TimeSeries engine table. + return null; +} + function TableModelForm({ control, setValue, @@ -1915,6 +1935,8 @@ function TableModelForm({ return ; case SourceKind.Metric: return ; + case SourceKind.Promql: + return ; } } @@ -2380,6 +2402,9 @@ export function TableSourceForm({ {IS_SESSIONS_ENABLED && ( )} + {IS_PROMQL_ENABLED && ( + + )} )} diff --git a/packages/app/src/components/sourceSelectUtils.tsx b/packages/app/src/components/sourceSelectUtils.tsx index 0333a4c4c0..fb9265cbea 100644 --- a/packages/app/src/components/sourceSelectUtils.tsx +++ b/packages/app/src/components/sourceSelectUtils.tsx @@ -12,6 +12,7 @@ export const SOURCE_KIND_ICONS: Record = { [SourceKind.Trace]: , [SourceKind.Session]: , [SourceKind.Metric]: , + [SourceKind.Promql]: , }; export function useSourceKindMap(sources: TSource[] | undefined) { diff --git a/packages/app/src/config.ts b/packages/app/src/config.ts index 90db80e949..af8737bd8b 100644 --- a/packages/app/src/config.ts +++ b/packages/app/src/config.ts @@ -64,3 +64,4 @@ export const IS_K8S_DASHBOARD_ENABLED = true; export const IS_METRICS_ENABLED = true; export const IS_MTVIEWS_ENABLED = false; export const IS_SESSIONS_ENABLED = true; +export const IS_PROMQL_ENABLED = env('NEXT_PUBLIC_ENABLE_PROMQL') === 'true'; diff --git a/packages/app/src/hooks/useChartConfig.tsx b/packages/app/src/hooks/useChartConfig.tsx index 3f28c498a1..26c1b1fd36 100644 --- a/packages/app/src/hooks/useChartConfig.tsx +++ b/packages/app/src/hooks/useChartConfig.tsx @@ -14,6 +14,7 @@ import { import { convertDateRangeToGranularityString } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderChartConfig, + isPromqlChartConfig, isRawSqlChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { format } from '@hyperdx/common-utils/dist/sqlFormatter'; @@ -29,6 +30,7 @@ import { UseQueryOptions, } from '@tanstack/react-query'; +import { prometheusApi } from '@/api'; import { toStartOfInterval } from '@/ChartUtils'; import { useClickhouseClient } from '@/clickhouse'; import { IS_MTVIEWS_ENABLED } from '@/config'; @@ -71,7 +73,7 @@ const shouldUseChunking = ( granularity: string; } => { // Avoid chunking for raw SQL charts since they can include arbitrary window functions, etc. - if (isRawSqlChartConfig(config)) return false; + if (isRawSqlChartConfig(config) || isPromqlChartConfig(config)) return false; // Granularity is required for chunking, otherwise we could break other group-bys. if (!isUsingGranularity(config)) return false; @@ -294,6 +296,94 @@ export function useQueriedChartConfig( // TODO: Replace this with `streamedQuery` when it is no longer experimental. Use 'replace' refetch mode. // https://tanstack.com/query/latest/docs/reference/streamedQuery queryFn: async context => { + // PromQL queries go through the Prometheus API route, not ClickHouse proxy + if (isPromqlChartConfig(config) && config.dateRange) { + const [startDate, endDate] = config.dateRange; + const startSec = startDate.getTime() / 1000; + const endSec = endDate.getTime() / 1000; + + // Convert HyperDX granularity ("5 minute") to Prometheus step ("300s") + let stepStr = '60s'; + if (config.granularity && config.granularity !== 'auto') { + const granToSec: Record = { + '15 second': 15, + '30 second': 30, + '1 minute': 60, + '5 minute': 300, + '10 minute': 600, + '15 minute': 900, + '30 minute': 1800, + '1 hour': 3600, + '2 hour': 7200, + '6 hour': 21600, + '12 hour': 43200, + '1 day': 86400, + }; + stepStr = `${granToSec[config.granularity] ?? 60}s`; + } + + const resp = await prometheusApi.queryRange({ + query: config.promqlExpression, + start: startSec, + end: endSec, + step: stepStr, + connectionId: config.connection, + database: config.from?.databaseName ?? 'default', + table: config.from?.tableName ?? 'otel_metrics_ts', + }); + + if (resp.status !== 'success' || !resp.data) { + throw new Error(resp.error ?? 'PromQL query failed'); + } + + // Transform Prometheus matrix response into chart-compatible format. + // Use Grafana-style legends: only show labels that differ across series. + const allSeries = resp.data.result; + + // Find labels that have more than one distinct value across all series + const labelValueSets = new Map>(); + for (const s of allSeries) { + for (const [k, v] of Object.entries(s.metric)) { + if (k === '__name__') continue; + if (!labelValueSets.has(k)) labelValueSets.set(k, new Set()); + labelValueSets.get(k)!.add(v); + } + } + const distinguishingKeys = new Set(); + for (const [k, vs] of labelValueSets) { + if (vs.size > 1) distinguishingKeys.add(k); + } + + const data: Record[] = []; + for (const series of allSeries) { + const metricName = series.metric.__name__ ?? ''; + const labels = Object.entries(series.metric) + .filter(([k]) => k !== '__name__' && distinguishingKeys.has(k)) + .map(([k, v]) => `${k}="${v}"`) + .join(', '); + const seriesName = labels ? `${metricName}{${labels}}` : metricName; + + for (const [ts, val] of series.values) { + data.push({ + __hdx_time_bucket: new Date(ts * 1000).toISOString(), + value: parseFloat(val), + series_name: seriesName, + }); + } + } + + return { + data, + meta: [ + { name: '__hdx_time_bucket', type: 'DateTime64(3)' }, + { name: 'value', type: 'Float64' }, + { name: 'series_name', type: 'String' }, + ], + rows: data.length, + isComplete: true, + }; + } + const optimizedConfig = mvOptimizationData?.optimizedConfig ?? config; const query = queryClient .getQueryCache() @@ -386,10 +476,19 @@ export function useRenderedSqlChartConfig( metadata, source?.querySettings, ); - return format(parameterizedQueryToSql(query)); + const sql = parameterizedQueryToSql(query); + // sql-formatter can't handle prometheusQuery() / CTE syntax in PromQL queries + if (isPromqlChartConfig(config)) { + return sql; + } + return format(sql); }, ...options, - enabled: enabled && !isLoadingMVOptimization && !isSourceLoading, + enabled: + enabled && + !isLoadingMVOptimization && + !isSourceLoading && + !isPromqlChartConfig(config), }); return { @@ -433,6 +532,20 @@ export function useAliasMapFromChartConfig( return {}; } + // PromQL queries use prometheusQuery() which node-sql-parser can't parse. + // Return a fixed alias map since the column names are known. + // Check configType directly since the TS type may not include PromQL here. + if ( + 'configType' in config && + (config as { configType: string }).configType === 'promql' + ) { + return { + __hdx_time_bucket: '__hdx_time_bucket', + value: 'value', + series_name: 'series_name', + }; + } + const query = await renderChartConfig( config, metadata, diff --git a/packages/app/src/hooks/useDashboardFilterValues.tsx b/packages/app/src/hooks/useDashboardFilterValues.tsx index d83e342f11..43b375dff8 100644 --- a/packages/app/src/hooks/useDashboardFilterValues.tsx +++ b/packages/app/src/hooks/useDashboardFilterValues.tsx @@ -26,7 +26,7 @@ type FilterSourceKey = { sourceId: string; metricType?: string; where: string; - whereLanguage: 'sql' | 'lucene'; + whereLanguage: 'sql' | 'lucene' | 'promql'; }; const filterToKey = (filter: DashboardFilter): string => diff --git a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx index 964c3e4680..13b890d9a2 100644 --- a/packages/app/src/hooks/useOffsetPaginatedQuery.tsx +++ b/packages/app/src/hooks/useOffsetPaginatedQuery.tsx @@ -19,6 +19,7 @@ import { } from '@hyperdx/common-utils/dist/core/utils'; import { isBuilderChartConfig, + isPromqlChartConfig, isRawSqlChartConfig, } from '@hyperdx/common-utils/dist/guards'; import { @@ -115,8 +116,12 @@ function getNextPageParam( config: ChartConfigWithOptTimestamp, windowDurationsSeconds: number[], ): TPageParam | undefined { - // Pagination is not supported for raw SQL tables since they may not be ordered at all. - if (lastPage == null || isRawSqlChartConfig(config)) { + // Pagination is not supported for raw SQL or PromQL tables since they may not be ordered at all. + if ( + lastPage == null || + isRawSqlChartConfig(config) || + isPromqlChartConfig(config) + ) { return undefined; } diff --git a/packages/app/src/hooks/usePromqlMetadata.ts b/packages/app/src/hooks/usePromqlMetadata.ts new file mode 100644 index 0000000000..aa4d057604 --- /dev/null +++ b/packages/app/src/hooks/usePromqlMetadata.ts @@ -0,0 +1,25 @@ +import { useQuery } from '@tanstack/react-query'; + +import { prometheusApi } from '@/api'; + +export function usePromqlMetricNames( + connectionId: string | undefined, + database?: string, + table?: string, +) { + return useQuery({ + queryKey: ['promql-metric-names', connectionId, database, table], + queryFn: async () => { + if (!connectionId) return []; + const resp = await prometheusApi.labelValues({ + label: '__name__', + connectionId, + database, + table, + }); + return resp.data ?? []; + }, + enabled: !!connectionId, + staleTime: 60_000, + }); +} diff --git a/packages/app/src/sessions.ts b/packages/app/src/sessions.ts index d1c8374d3c..f17d58cb8b 100644 --- a/packages/app/src/sessions.ts +++ b/packages/app/src/sessions.ts @@ -156,7 +156,9 @@ export function useSessions( ...(where && { filters: [ { - type: whereLanguage ?? 'lucene', + type: + (whereLanguage === 'promql' ? 'lucene' : whereLanguage) ?? + 'lucene', condition: where, }, ], diff --git a/packages/app/src/source.ts b/packages/app/src/source.ts index d299bd3755..d47df406c6 100644 --- a/packages/app/src/source.ts +++ b/packages/app/src/source.ts @@ -22,6 +22,7 @@ import { SourceSchema, TLogSource, TMetricSource, + TPromqlSource, TSessionSource, TSource, TSourceNoId, @@ -264,7 +265,8 @@ type InferredSourceConfig = | TStrippedSource | TStrippedSource | TStrippedSource - | TStrippedSource; + | TStrippedSource + | TStrippedSource; export async function inferTableSourceConfig({ databaseName, @@ -308,6 +310,10 @@ export async function inferTableSourceConfig({ kind, }; + if (kind === SourceKind.Promql) { + return baseConfig as TStrippedSource; + } + if (kind === SourceKind.Session) { const isSessionSchema = hasAllColumns(columns, Object.values(SESSION_TABLE_EXPRESSIONS)) || diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index d8986dae1a..645382a5a6 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -2759,4 +2759,29 @@ describe('renderChartConfig', () => { expect(actual).not.toContain('SampleRate'); }); }); + + describe('PromQL chart config', () => { + it('should return empty SQL (PromQL is executed via Prometheus API)', async () => { + const promqlConfig: ChartConfigWithOptDateRange = { + configType: 'promql' as const, + promqlExpression: 'rate(http_requests_total[5m])', + connection: 'test-connection', + displayType: DisplayType.Line, + dateRange: [ + new Date('2025-01-01T00:00:00Z'), + new Date('2025-01-01T01:00:00Z'), + ], + }; + + const generatedSql = await renderChartConfig( + promqlConfig, + mockMetadata, + undefined, + ); + + // PromQL configs return empty SQL β€” queries go through the Prometheus API route + expect(generatedSql.sql).toBe(''); + expect(generatedSql.params).toEqual({}); + }); + }); }); diff --git a/packages/common-utils/src/clickhouse/index.ts b/packages/common-utils/src/clickhouse/index.ts index 1a8b21283c..dfb28d0a6e 100644 --- a/packages/common-utils/src/clickhouse/index.ts +++ b/packages/common-utils/src/clickhouse/index.ts @@ -823,7 +823,7 @@ export function chSqlToAliasMap( chSql: ChSql | undefined, ): Record { const aliasMap: Record = {}; - if (chSql == null) { + if (chSql == null || !chSql.sql) { return aliasMap; } diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 6d0df87a96..f79db8adfb 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -15,7 +15,11 @@ import { parseToStartOfFunction, splitAndTrimWithBracket, } from '@/core/utils'; -import { isBuilderChartConfig, isRawSqlChartConfig } from '@/guards'; +import { + isBuilderChartConfig, + isPromqlChartConfig, + isRawSqlChartConfig, +} from '@/guards'; import { replaceMacros } from '@/macros'; import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser'; import { QUERY_PARAMS_BY_DISPLAY_TYPE } from '@/rawSqlParams'; @@ -32,6 +36,7 @@ import { DateRange, DisplayType, MetricsDataType, + PromqlChartConfig, QuerySettings, RawSqlChartConfig, SearchCondition, @@ -155,8 +160,12 @@ export const splitChartConfigs = ( return _configs; } - if (isRawSqlChartConfig(config) || isBuilderChartConfig(config)) { - return [config]; // narrowed to BuilderChartConfig or RawSqlChartConfig, assignable to RawSqlChartConfigEx + if ( + isRawSqlChartConfig(config) || + isPromqlChartConfig(config) || + isBuilderChartConfig(config) + ) { + return [config]; } throw new Error(`Unexpected chart config type: ${JSON.stringify(config)}`); @@ -824,7 +833,7 @@ async function renderWhereExpressionStr({ metadata: Metadata; from: BuilderChartConfigWithDateRange['from']; implicitColumnExpression?: string; - useTextIndexForImplicitColumn?: ChartConfigWithOptDateRangeEx['useTextIndexForImplicitColumn']; + useTextIndexForImplicitColumn?: BuilderChartConfigWithDateRange['useTextIndexForImplicitColumn']; connectionId: string; with?: BuilderChartConfigWithDateRange['with']; }): Promise { @@ -1099,9 +1108,14 @@ type RawSqlChartConfigEx = RawSqlChartConfig & Partial & InternalChartFields; +type PromqlChartConfigEx = PromqlChartConfig & + Partial & + InternalChartFields; + export type ChartConfigWithOptDateRangeEx = | BuilderChartConfigWithOptDateRangeEx - | RawSqlChartConfigEx; + | RawSqlChartConfigEx + | PromqlChartConfigEx; async function renderWith( chartConfig: BuilderChartConfigWithOptDateRangeEx, @@ -1757,6 +1771,11 @@ export async function renderChartConfig( metadata: Metadata, querySettings: QuerySettings | undefined, ): Promise { + if (isPromqlChartConfig(rawChartConfig)) { + // PromQL queries are executed server-side via the Prometheus API route, + // not via SQL generation. Return empty SQL as a no-op. + return { sql: '', params: {} }; + } if (isRawSqlChartConfig(rawChartConfig)) { return renderRawSqlChartConfig(rawChartConfig, metadata); } diff --git a/packages/common-utils/src/guards.ts b/packages/common-utils/src/guards.ts index e95042431c..059be3473b 100644 --- a/packages/common-utils/src/guards.ts +++ b/packages/common-utils/src/guards.ts @@ -4,6 +4,8 @@ import { ChartConfig, ChartConfigWithOptDateRange, DisplayType, + PromqlChartConfig, + PromqlSavedChartConfig, RawSqlChartConfig, RawSqlSavedChartConfig, SavedChartConfig, @@ -41,10 +43,16 @@ export function isRawSqlChartConfig( return 'configType' in chartConfig && chartConfig.configType === 'sql'; } +export function isPromqlChartConfig( + chartConfig: ChartConfig | ChartConfigWithOptDateRange, +): chartConfig is PromqlChartConfig { + return 'configType' in chartConfig && chartConfig.configType === 'promql'; +} + export function isBuilderChartConfig( chartConfig: ChartConfig | ChartConfigWithOptDateRange, ): chartConfig is BuilderChartConfig { - return !isRawSqlChartConfig(chartConfig); + return !isRawSqlChartConfig(chartConfig) && !isPromqlChartConfig(chartConfig); } export function isRawSqlSavedChartConfig( @@ -53,10 +61,19 @@ export function isRawSqlSavedChartConfig( return 'configType' in chartConfig && chartConfig.configType === 'sql'; } +export function isPromqlSavedChartConfig( + chartConfig: SavedChartConfig, +): chartConfig is PromqlSavedChartConfig { + return 'configType' in chartConfig && chartConfig.configType === 'promql'; +} + export function isBuilderSavedChartConfig( chartConfig: SavedChartConfig, ): chartConfig is BuilderSavedChartConfig { - return !isRawSqlSavedChartConfig(chartConfig); + return ( + !isRawSqlSavedChartConfig(chartConfig) && + !isPromqlSavedChartConfig(chartConfig) + ); } /** diff --git a/packages/common-utils/src/types.ts b/packages/common-utils/src/types.ts index d684f85fe2..fd29b0a0c2 100644 --- a/packages/common-utils/src/types.ts +++ b/packages/common-utils/src/types.ts @@ -148,7 +148,7 @@ export const SQLIntervalSchema = z .regex(/^\d+ (second|minute|hour|day)$/); export const SearchConditionSchema = z.string(); export const SearchConditionLanguageSchema = z - .enum(['sql', 'lucene']) + .enum(['sql', 'lucene', 'promql']) .optional(); export const AggregateFunctionSchema = z.enum([ 'avg', @@ -886,9 +886,29 @@ const RawSqlChartConfigSchema = RawSqlBaseChartConfigSchema.extend({ export type RawSqlChartConfig = z.infer; +/** Base schema for PromQL chart configs (persisted fields) */ +const PromqlBaseChartConfigSchema = SharedChartSettingsSchema.extend({ + configType: z.literal('promql'), + promqlExpression: z.string(), + connection: z.string(), + source: z.string().optional(), + step: z.string().optional(), +}); + +/** Schema describing PromQL chart configs with runtime-only fields */ +const PromqlChartConfigSchema = PromqlBaseChartConfigSchema.extend({ + filters: z.array(FilterSchema).optional(), + from: z + .object({ databaseName: z.string(), tableName: z.string() }) + .optional(), +}); + +export type PromqlChartConfig = z.infer; + export const ChartConfigSchema = z.union([ BuilderChartConfigSchema, RawSqlChartConfigSchema, + PromqlChartConfigSchema, ]); export type ChartConfig = z.infer; @@ -902,6 +922,7 @@ export type DateRange = { export type ChartConfigWithDateRange = ChartConfig & DateRange; export type BuilderChartConfigWithDateRange = BuilderChartConfig & DateRange; export type RawSqlConfigWithDateRange = RawSqlChartConfig & DateRange; +export type PromqlConfigWithDateRange = PromqlChartConfig & DateRange; export type BuilderChartConfigWithOptTimestamp = Omit< BuilderChartConfigWithDateRange, @@ -912,7 +933,8 @@ export type BuilderChartConfigWithOptTimestamp = Omit< export type ChartConfigWithOptTimestamp = | BuilderChartConfigWithOptTimestamp - | RawSqlConfigWithDateRange; + | RawSqlConfigWithDateRange + | PromqlConfigWithDateRange; // For non-time-based searches (ex. grab 1 row) export type BuilderChartConfigWithOptDateRange = Omit< @@ -924,7 +946,8 @@ export type BuilderChartConfigWithOptDateRange = Omit< export type ChartConfigWithOptDateRange = | BuilderChartConfigWithOptDateRange - | (RawSqlChartConfig & Partial); + | (RawSqlChartConfig & Partial) + | (PromqlChartConfig & Partial); // When making changes here, consider if they need to be made to the external API // schema as well (packages/api/src/utils/zod.ts). @@ -971,15 +994,33 @@ const RawSqlSavedChartConfigSchema = ]), }); +const PromqlSavedChartConfigWithoutAlertSchema = + PromqlBaseChartConfigSchema.extend({ + name: z.string().optional(), + }); + +const PromqlSavedChartConfigSchema = + PromqlSavedChartConfigWithoutAlertSchema.extend({ + alert: z.union([ + AlertBaseSchema.optional(), + ChartAlertBaseSchema.optional(), + ]), + }); + export const SavedChartConfigSchema = z.union([ BuilderSavedChartConfigSchema, RawSqlSavedChartConfigSchema, + PromqlSavedChartConfigSchema, ]); export type RawSqlSavedChartConfig = z.infer< typeof RawSqlSavedChartConfigSchema >; +export type PromqlSavedChartConfig = z.infer< + typeof PromqlSavedChartConfigSchema +>; + export type SavedChartConfig = z.infer; export const TileSchema = z.object({ @@ -1000,6 +1041,7 @@ export const TileTemplateSchema = TileSchema.extend({ config: z.union([ BuilderSavedChartConfigWithoutAlertSchema, RawSqlSavedChartConfigWithoutAlertSchema, + PromqlSavedChartConfigWithoutAlertSchema, ]), }); @@ -1157,6 +1199,7 @@ export const ConnectionSchema = z.object({ .regex(/^[a-z0-9_]+$/i) .optional() .nullable(), + prometheusEndpoint: z.string().url().optional(), }); export type Connection = z.infer; @@ -1207,6 +1250,7 @@ export enum SourceKind { Trace = 'trace', Session = 'session', Metric = 'metric', + Promql = 'promql', } // -------------------------- @@ -1401,12 +1445,18 @@ export const MetricSourceSchema = BaseSourceSchema.extend({ logSourceId: z.string().optional(), }); +// PromQL source form schema +export const PromqlSourceSchema = BaseSourceSchema.extend({ + kind: z.literal(SourceKind.Promql), +}); + // Union of all source form schemas for validation export const SourceSchema = z.discriminatedUnion('kind', [ LogSourceSchema, TraceSourceSchema, SessionSourceSchema, MetricSourceSchema, + PromqlSourceSchema, ]); export type TSource = z.infer; @@ -1415,6 +1465,7 @@ export const SourceSchemaNoId = z.discriminatedUnion('kind', [ TraceSourceSchema.omit({ id: true }), SessionSourceSchema.omit({ id: true }), MetricSourceSchema.omit({ id: true }), + PromqlSourceSchema.omit({ id: true }), ]); export type TSourceNoId = z.infer; @@ -1423,6 +1474,7 @@ export type TLogSource = Extract; export type TTraceSource = Extract; export type TSessionSource = Extract; export type TMetricSource = Extract; +export type TPromqlSource = Extract; // Type guards for narrowing TSource by kind export function isLogSource(source: TSource): source is TLogSource { @@ -1437,6 +1489,9 @@ export function isSessionSource(source: TSource): source is TSessionSource { export function isMetricSource(source: TSource): source is TMetricSource { return source.kind === SourceKind.Metric; } +export function isPromqlSource(source: TSource): source is TPromqlSource { + return source.kind === SourceKind.Promql; +} export function isSearchableSource(source: TSource): boolean { return isLogSource(source) || isTraceSource(source); } diff --git a/packages/otel-collector/builder-config.yaml b/packages/otel-collector/builder-config.yaml index ff3fbe5f88..5b8e0c8f94 100644 --- a/packages/otel-collector/builder-config.yaml +++ b/packages/otel-collector/builder-config.yaml @@ -119,6 +119,9 @@ exporters: - gomod: github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter v__OTEL_COLLECTOR_VERSION__ + - gomod: + github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter + v__OTEL_COLLECTOR_VERSION__ connectors: # Core diff --git a/packages/otel-collector/cmd/migrate/main.go b/packages/otel-collector/cmd/migrate/main.go index 3f09d48b03..cb0877bf2e 100644 --- a/packages/otel-collector/cmd/migrate/main.go +++ b/packages/otel-collector/cmd/migrate/main.go @@ -410,6 +410,18 @@ func removeCompatLogsSchema(tempDir string) error { return nil } +// removePromqlSchema removes the experimental TimeSeries-engine schema from +// the temp directory so it is only created when PromQL support is opted into +// via ENABLE_PROMQL=true. Keeps the experimental engine and otel_metrics_ts +// table out of deployments that have not enabled the feature. +func removePromqlSchema(tempDir string) error { + promqlPath := filepath.Join(tempDir, "00008_otel_metrics_timeseries.sql") + if err := os.Remove(promqlPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove promql schema: %w", err) + } + return nil +} + // swapTracesSchemaForCompat replaces the full-text-search traces schema with // the compatibility variant (bloom_filter indexes, no items columns) in the // processed temp directory. It removes 00005_otel_traces.sql and renames @@ -527,6 +539,13 @@ func main() { } } + if os.Getenv("ENABLE_PROMQL") != "true" { + log.Printf("ENABLE_PROMQL not set, skipping PromQL TimeSeries schema") + if err := removePromqlSchema(tempDir); err != nil { + log.Fatalf("Failed to remove promql schema: %v", err) + } + } + // List SQL files sqlFiles, err := listSQLFiles(tempDir) if err != nil { diff --git a/packages/otel-collector/cmd/migrate/main_test.go b/packages/otel-collector/cmd/migrate/main_test.go index 39a9a5ae3f..023bf77f85 100644 --- a/packages/otel-collector/cmd/migrate/main_test.go +++ b/packages/otel-collector/cmd/migrate/main_test.go @@ -827,6 +827,57 @@ func TestSwapLogsSchemaForCompat(t *testing.T) { // removeCompatLogsSchema // --------------------------------------------------------------------------- +func TestRemovePromqlSchema(t *testing.T) { + t.Run("removes existing promql schema file", func(t *testing.T) { + dir := t.TempDir() + promqlPath := filepath.Join(dir, "00008_otel_metrics_timeseries.sql") + + if err := os.WriteFile(promqlPath, []byte("PROMQL SCHEMA"), 0644); err != nil { + t.Fatal(err) + } + + if err := removePromqlSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(promqlPath); !os.IsNotExist(err) { + t.Error("promql schema file should have been removed") + } + }) + + t.Run("no error when promql schema file does not exist", func(t *testing.T) { + dir := t.TempDir() + + if err := removePromqlSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + }) + + t.Run("preserves other files", func(t *testing.T) { + dir := t.TempDir() + otherPath := filepath.Join(dir, "00001_other.sql") + promqlPath := filepath.Join(dir, "00008_otel_metrics_timeseries.sql") + + if err := os.WriteFile(otherPath, []byte("OTHER"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(promqlPath, []byte("PROMQL"), 0644); err != nil { + t.Fatal(err) + } + + if err := removePromqlSchema(dir); err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if _, err := os.Stat(otherPath); err != nil { + t.Error("other file should still exist") + } + if _, err := os.Stat(promqlPath); !os.IsNotExist(err) { + t.Error("promql schema file should have been removed") + } + }) +} + func TestRemoveCompatLogsSchema(t *testing.T) { t.Run("removes existing compat file", func(t *testing.T) { dir := t.TempDir() diff --git a/yarn.lock b/yarn.lock index 724a14c4e4..e9a000f5ea 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4438,6 +4438,7 @@ __metadata: "@mantine/spotlight": "npm:^9.0.0" "@next/eslint-plugin-next": "npm:^16.0.10" "@playwright/test": "npm:^1.57.0" + "@prometheus-io/codemirror-promql": "npm:^0.311.3" "@storybook/addon-docs": "npm:^10.1.4" "@storybook/addon-links": "npm:^10.1.4" "@storybook/addon-styling-webpack": "npm:^3.0.0" @@ -8066,6 +8067,33 @@ __metadata: languageName: node linkType: hard +"@prometheus-io/codemirror-promql@npm:^0.311.3": + version: 0.311.3 + resolution: "@prometheus-io/codemirror-promql@npm:0.311.3" + dependencies: + "@prometheus-io/lezer-promql": "npm:0.311.3" + lru-cache: "npm:^11.2.7" + peerDependencies: + "@codemirror/autocomplete": ^6.4.0 + "@codemirror/language": ^6.3.0 + "@codemirror/lint": ^6.0.0 + "@codemirror/state": ^6.1.1 + "@codemirror/view": ^6.4.0 + "@lezer/common": ^1.0.1 + checksum: 10c0/3c0ef6e5aa061fe67c32764c07cdfe26c171f0196e64120e56029d56aadd1f53b28a25615a6861dfaef18e71d8af429ac6a368f4b2a5e54ae184567b51c80f04 + languageName: node + linkType: hard + +"@prometheus-io/lezer-promql@npm:0.311.3": + version: 0.311.3 + resolution: "@prometheus-io/lezer-promql@npm:0.311.3" + peerDependencies: + "@lezer/highlight": ^1.1.2 + "@lezer/lr": ^1.2.3 + checksum: 10c0/4f0cd6b16418e1c2103178a30db7160e7a3615195087ae24cf730dd878abcd4d0640a3b1a1a89823aad0c3b05586ebb216f2957d02e4a1595fc53831c34379b7 + languageName: node + linkType: hard + "@protobufjs/aspromise@npm:^1.1.1, @protobufjs/aspromise@npm:^1.1.2": version: 1.1.2 resolution: "@protobufjs/aspromise@npm:1.1.2" @@ -20409,6 +20437,13 @@ __metadata: languageName: node linkType: hard +"lru-cache@npm:^11.2.7": + version: 11.3.5 + resolution: "lru-cache@npm:11.3.5" + checksum: 10c0/5b54ef7b88afb4bd25b7a778f1b2b1cde32d9770913e530da34ab203cf0442413bcaa6e372800cbab9562557a4480e4d8bf32e3a368bb5a91b12218eca085c66 + languageName: node + linkType: hard + "lru-cache@npm:^4.0.1": version: 4.1.5 resolution: "lru-cache@npm:4.1.5"