diff --git a/.direnv/flake-profile b/.direnv/flake-profile new file mode 120000 index 0000000000..c7ae5b709f --- /dev/null +++ b/.direnv/flake-profile @@ -0,0 +1 @@ +flake-profile-2-link \ No newline at end of file diff --git a/.direnv/flake-profile-2-link b/.direnv/flake-profile-2-link new file mode 120000 index 0000000000..140a682073 --- /dev/null +++ b/.direnv/flake-profile-2-link @@ -0,0 +1 @@ +/nix/store/pknafnv7zx67x4misiglnnvbj3sgm30r-nix-shell-env \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2d5e84f164..a22457059a 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ frontend/build/ .idea/ *.swp *.swo +.envrc # OS .DS_Store diff --git a/Cargo.lock b/Cargo.lock index a2ade3bdd1..7ec3b84257 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3951,6 +3951,7 @@ dependencies = [ "rivet-envoy-protocol", "rivet-error", "rivet-metrics", + "rivet-outbound-guard", "rivet-pools", "rivet-runner-protocol", "rivet-runtime", @@ -3976,6 +3977,7 @@ dependencies = [ "utoipa", "uuid", "vbare", + "webhook", "xxhash-rust", ] @@ -4112,12 +4114,15 @@ dependencies = [ "rivet-config", "rivet-envoy-protocol", "rivet-metrics", + "rivet-outbound-guard", + "rivet-pools", "rivet-runtime", "rivet-types", "tokio", "tracing", "universaldb", "universalpubsub", + "url", "vbare", ] @@ -5106,6 +5111,7 @@ dependencies = [ "tracing", "urlencoding", "utoipa", + "webhook", ] [[package]] @@ -5693,6 +5699,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "rivet-outbound-guard" +version = "2.3.7" +dependencies = [ + "anyhow", + "ipnet", + "reqwest 0.12.22", + "rivet-config", + "thiserror 1.0.69", + "tokio", + "tracing", + "url", +] + [[package]] name = "rivet-perf" version = "2.3.7" @@ -5719,6 +5739,7 @@ dependencies = [ "rivet-async-nats", "rivet-config", "rivet-metrics", + "rivet-outbound-guard", "rivet-util", "rustls", "serde", @@ -6077,6 +6098,7 @@ dependencies = [ "pegboard", "rivet-config", "tracing", + "webhook", ] [[package]] @@ -8722,6 +8744,32 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webhook" +version = "2.3.7" +dependencies = [ + "anyhow", + "chrono", + "epoxy", + "futures-util", + "gasoline", + "namespace", + "reqwest 0.12.22", + "rivet-data", + "rivet-error", + "rivet-outbound-guard", + "rivet-pools", + "rivet-util", + "serde", + "serde_bare", + "serde_json", + "tracing", + "universaldb", + "url", + "uuid", + "vbare", +] + [[package]] name = "webpki-root-certs" version = "1.0.7" diff --git a/Cargo.toml b/Cargo.toml index a9048c159f..26f0b4f26b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ members = [ "engine/packages/metrics", "engine/packages/metrics-server", "engine/packages/namespace", + "engine/packages/outbound-guard", "engine/packages/pegboard", "engine/packages/pegboard-envoy", "engine/packages/pegboard-gateway", @@ -56,6 +57,7 @@ members = [ "engine/packages/universalpubsub", "engine/packages/util", "engine/packages/util-id", + "engine/packages/webhook", "engine/packages/util-serde", "engine/packages/workflow-worker", "engine/sdks/rust/api-full", @@ -500,6 +502,9 @@ members = [ [workspace.dependencies.namespace] path = "engine/packages/namespace" + [workspace.dependencies.rivet-outbound-guard] + path = "engine/packages/outbound-guard" + [workspace.dependencies.pegboard] path = "engine/packages/pegboard" @@ -573,6 +578,9 @@ members = [ [workspace.dependencies.rivet-util-id] path = "engine/packages/util-id" + [workspace.dependencies.webhook] + path = "engine/packages/webhook" + [workspace.dependencies.rivet-util-serde] path = "engine/packages/util-serde" version = "=2.3.7" diff --git a/docs/content/docs/debugging.mdx b/docs/content/docs/debugging.mdx index 93cfe96e1b..432ec7deaf 100644 --- a/docs/content/docs/debugging.mdx +++ b/docs/content/docs/debugging.mdx @@ -199,7 +199,7 @@ Returns the configured provider settings per datacenter and the latest pool erro } ``` -`runner_pool_error` mirrors actor scheduling errors such as `serverless_http_error`, `serverless_connection_error`, and `serverless_stream_ended_early`. +`runner_pool_error` mirrors actor scheduling errors such as `serverless_http_error`, `serverless_connection_error`, `serverless_destination_blocked`, and `serverless_stream_ended_early`. ### Check Serverless Provider Health diff --git a/docs/content/docs/troubleshooting.mdx b/docs/content/docs/troubleshooting.mdx index f06c30fe14..75ba34704d 100644 --- a/docs/content/docs/troubleshooting.mdx +++ b/docs/content/docs/troubleshooting.mdx @@ -97,6 +97,14 @@ Rivet was unable to connect to your serverless endpoint. Check that: - Your server is publicly reachable from the internet. - There are no DNS or firewall issues blocking the connection. +### `serverless_destination_blocked` + +The configured serverless URL points at a destination Rivet is not allowed to reach. Rivet dials serverless endpoints from inside its own network, so by default it refuses any destination that is not publicly routable: private ranges, link-local and cloud metadata addresses, carrier-grade NAT, and other reserved ranges. Check that: + +- Your endpoint URL is publicly reachable, and is not an internal hostname or address. +- The URL scheme is `http` or `https`, and the URL does not embed credentials. +- If you are self-hosting and intentionally point runners at an address on your own network, set `outbound.allow_private_networks` to `true` in your engine config, or list the specific range in `outbound.allow_cidrs`. + ### `serverless_stream_ended_early` The connection to your serverless endpoint was terminated before the actor finished. This usually means your serverless function hit its execution time limit. Ensure that your Rivet provider's request lifespan is configured to match the max duration of your serverless platform. diff --git a/engine/artifacts/config-schema.json b/engine/artifacts/config-schema.json index 9147b7a7d1..23533f876c 100644 --- a/engine/artifacts/config-schema.json +++ b/engine/artifacts/config-schema.json @@ -106,6 +106,17 @@ } ] }, + "outbound": { + "default": null, + "anyOf": [ + { + "$ref": "#/definitions/Outbound" + }, + { + "type": "null" + } + ] + }, "pegboard": { "default": null, "anyOf": [ @@ -803,6 +814,63 @@ }, "additionalProperties": false }, + "Outbound": { + "description": "Policy for outbound HTTP requests to user-configured destinations, such as serverless runner URLs.\n\nThese requests originate from inside the trusted engine network, so without restrictions a caller who can configure a runner can reach internal-only services. The defaults deny every non-globally-routable destination except loopback.", + "type": "object", + "properties": { + "allow_cidrs": { + "description": "Additional CIDRs that are always permitted, evaluated after `deny_cidrs`.\n\nUse this to reach a specific internal service without opening up the whole private range.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "allow_insecure_scheme": { + "description": "Allow plaintext `http://` destinations. When disabled only `https://` is permitted.", + "type": [ + "boolean", + "null" + ] + }, + "allow_loopback": { + "description": "Allow destinations that resolve to loopback addresses (127.0.0.0/8, ::1).\n\nEnabled by default so local development against `http://localhost:...` works without configuration.", + "type": [ + "boolean", + "null" + ] + }, + "allow_private_networks": { + "description": "Allow destinations that resolve to private, link-local, shared (CGNAT), or otherwise non-globally-routable addresses.\n\nSelf-hosted deployments that point runners at addresses on their own network, such as a Docker Compose service name, need this enabled.", + "type": [ + "boolean", + "null" + ] + }, + "deny_cidrs": { + "description": "Additional CIDRs that are always denied. Takes precedence over every allow rule.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "max_redirects": { + "description": "Maximum number of redirects to follow. Every hop is re-checked against this policy.", + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 0.0 + } + }, + "additionalProperties": false + }, "Pegboard": { "type": "object", "properties": { diff --git a/engine/artifacts/errors/webhook.conflict.json b/engine/artifacts/errors/webhook.conflict.json new file mode 100644 index 0000000000..f0ad50310a --- /dev/null +++ b/engine/artifacts/errors/webhook.conflict.json @@ -0,0 +1,5 @@ +{ + "code": "conflict", + "group": "webhook", + "message": "Webhook config changed concurrently." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.delivery_failed.json b/engine/artifacts/errors/webhook.delivery_failed.json new file mode 100644 index 0000000000..7454fcc450 --- /dev/null +++ b/engine/artifacts/errors/webhook.delivery_failed.json @@ -0,0 +1,5 @@ +{ + "code": "delivery_failed", + "group": "webhook", + "message": "Webhook delivery failed." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.delivery_not_found.json b/engine/artifacts/errors/webhook.delivery_not_found.json new file mode 100644 index 0000000000..0217d3cb8f --- /dev/null +++ b/engine/artifacts/errors/webhook.delivery_not_found.json @@ -0,0 +1,5 @@ +{ + "code": "delivery_not_found", + "group": "webhook", + "message": "Webhook delivery not found." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.delivery_not_retryable.json b/engine/artifacts/errors/webhook.delivery_not_retryable.json new file mode 100644 index 0000000000..b5f02f7b7f --- /dev/null +++ b/engine/artifacts/errors/webhook.delivery_not_retryable.json @@ -0,0 +1,5 @@ +{ + "code": "delivery_not_retryable", + "group": "webhook", + "message": "Webhook delivery is not in a retryable state." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.destination_blocked.json b/engine/artifacts/errors/webhook.destination_blocked.json new file mode 100644 index 0000000000..e9fa286998 --- /dev/null +++ b/engine/artifacts/errors/webhook.destination_blocked.json @@ -0,0 +1,5 @@ +{ + "code": "destination_blocked", + "group": "webhook", + "message": "Webhook destination is not allowed." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.event_type_not_allowed.json b/engine/artifacts/errors/webhook.event_type_not_allowed.json new file mode 100644 index 0000000000..c7d19f6278 --- /dev/null +++ b/engine/artifacts/errors/webhook.event_type_not_allowed.json @@ -0,0 +1,5 @@ +{ + "code": "event_type_not_allowed", + "group": "webhook", + "message": "Event type cannot be subscribed to by a webhook." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.invalid.json b/engine/artifacts/errors/webhook.invalid.json new file mode 100644 index 0000000000..3c00e6e1d5 --- /dev/null +++ b/engine/artifacts/errors/webhook.invalid.json @@ -0,0 +1,5 @@ +{ + "code": "invalid", + "group": "webhook", + "message": "Invalid webhook config." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.not_found.json b/engine/artifacts/errors/webhook.not_found.json new file mode 100644 index 0000000000..6b843aecf7 --- /dev/null +++ b/engine/artifacts/errors/webhook.not_found.json @@ -0,0 +1,5 @@ +{ + "code": "not_found", + "group": "webhook", + "message": "Webhook not found." +} \ No newline at end of file diff --git a/engine/artifacts/openapi.json b/engine/artifacts/openapi.json index e54b6132c2..15fe9ec92f 100644 --- a/engine/artifacts/openapi.json +++ b/engine/artifacts/openapi.json @@ -1103,6 +1103,246 @@ } ] } + }, + "/webhooks": { + "get": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_list", + "parameters": [ + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksListResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + } + }, + "/webhooks/{webhook_name}": { + "put": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_upsert", + "parameters": [ + { + "name": "webhook_name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksUpsertRequestBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksUpsertResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + }, + "delete": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_delete", + "parameters": [ + { + "name": "webhook_name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksDeleteResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + } + }, + "/webhooks/{webhook_name}/deliveries/{delivery_id}/retry": { + "post": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_retry_delivery", + "parameters": [ + { + "name": "webhook_name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "delivery_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksRetryDeliveryResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + } + }, + "/webhooks/{webhook_name}/events": { + "get": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_events", + "parameters": [ + { + "name": "webhook_name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0 + } + }, + { + "name": "cursor", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksEventsResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + } } }, "components": { @@ -2290,6 +2530,133 @@ } }, "additionalProperties": false + }, + "WebhookConfig": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "propertyNames": { + "type": "string" + } + }, + "subscriptions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEventType" + } + }, + "url": { + "type": "string" + } + }, + "additionalProperties": false + }, + "WebhookEvent": { + "type": "object", + "required": [ + "id", + "create_ts", + "status", + "event_type", + "attempt_count" + ], + "properties": { + "attempt_count": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "create_ts": { + "type": "integer", + "format": "int64" + }, + "event_type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_error": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + } + }, + "additionalProperties": false + }, + "WebhookEventType": { + "type": "string", + "enum": [ + "runner_pool.error", + "runner_pool.healthy" + ] + }, + "WebhooksDeleteResponse": { + "type": "object", + "additionalProperties": false + }, + "WebhooksEventsResponse": { + "type": "object", + "required": [ + "events", + "pagination" + ], + "properties": { + "events": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + } + }, + "pagination": { + "$ref": "#/components/schemas/Pagination" + } + }, + "additionalProperties": false + }, + "WebhooksListResponse": { + "type": "object", + "required": [ + "webhooks", + "pagination" + ], + "properties": { + "pagination": { + "$ref": "#/components/schemas/Pagination" + }, + "webhooks": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/WebhookConfig" + }, + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "WebhooksRetryDeliveryResponse": { + "type": "object", + "additionalProperties": false + }, + "WebhooksUpsertRequestBody": { + "$ref": "#/components/schemas/WebhookConfig" + }, + "WebhooksUpsertResponse": { + "type": "object", + "additionalProperties": false } }, "securitySchemes": { diff --git a/engine/packages/api-public/Cargo.toml b/engine/packages/api-public/Cargo.toml index b5289c97bc..d82f5596c6 100644 --- a/engine/packages/api-public/Cargo.toml +++ b/engine/packages/api-public/Cargo.toml @@ -37,6 +37,7 @@ tower-http.workspace = true tracing.workspace = true urlencoding.workspace = true utoipa.workspace = true +webhook.workspace = true [build-dependencies] anyhow.workspace = true diff --git a/engine/packages/api-public/src/lib.rs b/engine/packages/api-public/src/lib.rs index c7a2c33706..0638a41d9e 100644 --- a/engine/packages/api-public/src/lib.rs +++ b/engine/packages/api-public/src/lib.rs @@ -10,5 +10,6 @@ pub mod router; pub mod runner_configs; pub mod runners; pub mod ui; +pub mod webhooks; pub use router::router; diff --git a/engine/packages/api-public/src/router.rs b/engine/packages/api-public/src/router.rs index d2a79fb99b..ce80749b72 100644 --- a/engine/packages/api-public/src/router.rs +++ b/engine/packages/api-public/src/router.rs @@ -10,6 +10,7 @@ use utoipa::OpenApi; use crate::{ actors, ctx, datacenters, envoys, health, metadata, namespaces, runner_configs, runners, ui, + webhooks, }; #[derive(OpenApi)] @@ -33,6 +34,11 @@ use crate::{ runner_configs::delete::delete, runner_configs::serverless_health_check::serverless_health_check, runner_configs::refresh_metadata::refresh_metadata, + webhooks::list, + webhooks::upsert, + webhooks::delete, + webhooks::retry_delivery, + webhooks::events, datacenters::list, health::fanout, metadata::get, @@ -81,6 +87,24 @@ pub async fn router( "/runner-configs/{runner_name}/refresh-metadata", axum::routing::post(runner_configs::refresh_metadata), ) + // MARK: Webhooks + .route("/webhooks", axum::routing::get(webhooks::list)) + .route( + "/webhooks/{webhook_name}", + axum::routing::put(webhooks::upsert), + ) + .route( + "/webhooks/{webhook_name}", + axum::routing::delete(webhooks::delete), + ) + .route( + "/webhooks/{webhook_name}/deliveries/{delivery_id}/retry", + axum::routing::post(webhooks::retry_delivery), + ) + .route( + "/webhooks/{webhook_name}/events", + axum::routing::get(webhooks::events), + ) // MARK: Actors .route("/actors", axum::routing::get(actors::list::list)) .route("/actors", axum::routing::post(actors::create::create)) diff --git a/engine/packages/api-public/src/webhooks.rs b/engine/packages/api-public/src/webhooks.rs new file mode 100644 index 0000000000..45f18c1788 --- /dev/null +++ b/engine/packages/api-public/src/webhooks.rs @@ -0,0 +1,517 @@ +use std::collections::HashMap; + +use anyhow::Result; +use axum::response::{IntoResponse, Response}; +use rivet_api_builder::{ + ApiError, + errors::ApiBadRequest, + extract::{Extension, Json, Path, Query}, +}; +use rivet_api_types::pagination::Pagination; +use serde::{Deserialize, Serialize}; +use utoipa::{IntoParams, ToSchema}; + +use crate::ctx::ApiCtx; + +// Config for a single webhook, keyed by an arbitrary name within a namespace. `subscriptions` +// names the event types to deliver; only webhook-safe types are accepted (see +// `webhook::types::WebhookEventType`). +#[derive(Debug, Deserialize, Serialize, Clone, ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WebhookConfig { + pub url: String, + #[serde(default)] + pub headers: HashMap, + #[serde(default)] + pub subscriptions: Vec, +} + +// Mirrors `webhook::types::WebhookEventType` for the public API. Only webhook-safe variants are +// exposed; high-throughput event types are not subscribable and so have no API representation. +#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, ToSchema)] +pub enum WebhookEventType { + #[serde(rename = "runner_pool.error")] + RunnerPoolError, + #[serde(rename = "runner_pool.healthy")] + RunnerPoolHealthy, +} + +impl From for webhook::types::WebhookEventType { + fn from(value: WebhookEventType) -> Self { + match value { + WebhookEventType::RunnerPoolError => webhook::types::WebhookEventType::RunnerPoolError, + WebhookEventType::RunnerPoolHealthy => { + webhook::types::WebhookEventType::RunnerPoolHealthy + } + } + } +} + +impl WebhookEventType { + // `None` for event types that have no API representation because they cannot be subscribed + // to. Upsert validation rejects those, so a stored config should never contain one. + fn from_internal(value: webhook::types::WebhookEventType) -> Option { + match value { + webhook::types::WebhookEventType::RunnerPoolError => { + Some(WebhookEventType::RunnerPoolError) + } + webhook::types::WebhookEventType::RunnerPoolHealthy => { + Some(WebhookEventType::RunnerPoolHealthy) + } + webhook::types::WebhookEventType::ActorHttpRequest => None, + } + } +} + +// MARK: List + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct ListQuery { + pub namespace: String, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksListResponse)] +pub struct ListResponse { + pub webhooks: HashMap, + pub pagination: Pagination, +} + +#[utoipa::path( + get, + operation_id = "webhooks_list", + path = "/webhooks", + params(ListQuery), + responses( + (status = 200, body = ListResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn list(Extension(ctx): Extension, Query(query): Query) -> Response { + match list_inner(ctx, query).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +#[tracing::instrument(skip_all)] +async fn list_inner(ctx: ApiCtx, query: ListQuery) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(namespace::ops::resolve_for_name_global::Input { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| namespace::errors::Namespace::NotFound.build())?; + + let webhooks = ctx + .op(webhook::ops::list::Input { + namespace_id: namespace.namespace_id, + }) + .await?; + + Ok(ListResponse { + webhooks: webhooks + .into_iter() + .map(|w| { + ( + w.name, + WebhookConfig { + url: w.config.url, + headers: w.config.headers, + subscriptions: w + .config + .subscriptions + .into_iter() + .filter_map(WebhookEventType::from_internal) + .collect(), + }, + ) + }) + .collect(), + pagination: Pagination { cursor: None }, + }) +} + +// MARK: Upsert + +#[derive(Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct UpsertPath { + pub webhook_name: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct UpsertQuery { + pub namespace: String, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksUpsertRequestBody)] +pub struct UpsertRequest(pub WebhookConfig); + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksUpsertResponse)] +pub struct UpsertResponse {} + +#[utoipa::path( + put, + operation_id = "webhooks_upsert", + path = "/webhooks/{webhook_name}", + params( + ("webhook_name" = String, Path), + UpsertQuery, + ), + request_body(content = UpsertRequest, content_type = "application/json"), + responses( + (status = 200, body = UpsertResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn upsert( + Extension(ctx): Extension, + Path(path): Path, + Query(query): Query, + Json(body): Json, +) -> Response { + match upsert_inner(ctx, path, query, body).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +#[tracing::instrument(skip_all)] +async fn upsert_inner( + ctx: ApiCtx, + path: UpsertPath, + query: UpsertQuery, + body: UpsertRequest, +) -> Result { + ctx.auth().await?; + + // Resolve and validate namespace + let namespace = ctx + .op(namespace::ops::resolve_for_name_global::Input { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| namespace::errors::Namespace::NotFound.build())?; + + // Upsert operation + ctx.op(webhook::ops::upsert::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + config: webhook::types::WebhookConfig { + url: body.0.url, + headers: body.0.headers, + subscriptions: body.0.subscriptions.into_iter().map(Into::into).collect(), + }, + }) + .await?; + + // The config is durable in epoxy and the webhook workflow is dispatched or signaled by + // the op above. Delivering triggered events over HTTP still needs to be built (see webhook + // spec). + + Ok(UpsertResponse {}) +} + +// MARK: Delete + +#[derive(Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct DeletePath { + pub webhook_name: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct DeleteQuery { + pub namespace: String, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksDeleteResponse)] +pub struct DeleteResponse {} + +#[utoipa::path( + delete, + operation_id = "webhooks_delete", + path = "/webhooks/{webhook_name}", + params( + ("webhook_name" = String, Path), + DeleteQuery, + ), + responses( + (status = 200, body = DeleteResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn delete( + Extension(ctx): Extension, + Path(path): Path, + Query(query): Query, +) -> Response { + match delete_inner(ctx, path, query).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +#[tracing::instrument(skip_all)] +async fn delete_inner(ctx: ApiCtx, path: DeletePath, query: DeleteQuery) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(namespace::ops::resolve_for_name_global::Input { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| namespace::errors::Namespace::NotFound.build())?; + + ctx.op(webhook::ops::delete::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + }) + .await?; + + Ok(DeleteResponse {}) +} + +// MARK: Retry delivery + +#[derive(Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct RetryDeliveryPath { + pub webhook_name: String, + pub delivery_id: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct RetryDeliveryQuery { + pub namespace: String, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksRetryDeliveryResponse)] +pub struct RetryDeliveryResponse {} + +#[utoipa::path( + post, + operation_id = "webhooks_retry_delivery", + path = "/webhooks/{webhook_name}/deliveries/{delivery_id}/retry", + params( + ("webhook_name" = String, Path), + ("delivery_id" = String, Path), + RetryDeliveryQuery, + ), + responses( + (status = 200, body = RetryDeliveryResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn retry_delivery( + Extension(ctx): Extension, + Path(path): Path, + Query(query): Query, +) -> Response { + match retry_delivery_inner(ctx, path, query).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +#[tracing::instrument(skip_all)] +async fn retry_delivery_inner( + ctx: ApiCtx, + path: RetryDeliveryPath, + query: RetryDeliveryQuery, +) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(namespace::ops::resolve_for_name_global::Input { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| namespace::errors::Namespace::NotFound.build())?; + + ctx.op(webhook::ops::retry::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + delivery_id: path.delivery_id.clone(), + }) + .await?; + + Ok(RetryDeliveryResponse {}) +} + +// MARK: Events + +#[derive(Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct EventsPath { + pub webhook_name: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct EventsQuery { + pub namespace: String, + pub limit: Option, + pub cursor: Option, +} + +// One delivery in a webhook's event history. `id` is the delivery id, which is also the +// CloudEvents `id` sent to the destination and what the retry endpoint takes. +#[derive(Debug, Deserialize, Serialize, Clone, ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WebhookEvent { + pub id: String, + pub create_ts: i64, + pub status: String, + pub event_type: String, + pub attempt_count: u32, + pub last_error: Option, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksEventsResponse)] +pub struct EventsResponse { + pub events: Vec, + pub pagination: Pagination, +} + +#[utoipa::path( + get, + operation_id = "webhooks_events", + path = "/webhooks/{webhook_name}/events", + params( + ("webhook_name" = String, Path), + EventsQuery, + ), + responses( + (status = 200, body = EventsResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn events( + Extension(ctx): Extension, + Path(path): Path, + Query(query): Query, +) -> Response { + match events_inner(ctx, path, query).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +const DEFAULT_EVENTS_LIMIT: usize = 20; + +#[tracing::instrument(skip_all)] +async fn events_inner(ctx: ApiCtx, path: EventsPath, query: EventsQuery) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(namespace::ops::resolve_for_name_global::Input { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| namespace::errors::Namespace::NotFound.build())?; + + // Distinguish "this webhook has no history" from "this webhook does not exist", which would + // otherwise both be an empty list. Note this means a deleted webhook's history is no longer + // readable, since delete clears the config. + if ctx + .op(webhook::ops::get::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + }) + .await? + .is_none() + { + return Err(webhook::errors::Webhook::NotFound.build()); + } + + let mut deliveries = ctx + .op(webhook::ops::list_deliveries::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + }) + .await?; + + // Most recent first. `delivery_id` breaks ties deterministically since two deliveries can + // share a `created_at` millisecond. + deliveries.sort_by(|a, b| { + b.record + .created_at + .cmp(&a.record.created_at) + .then_with(|| b.delivery_id.cmp(&a.delivery_id)) + }); + + // The cursor is the `(created_at, delivery_id)` of the last item on the previous page; + // resume strictly after it in the same sorted order. + if let Some(cursor) = query.cursor { + let (created_at, delivery_id) = cursor + .split_once(':') + .and_then(|(ts, id)| ts.parse::().ok().map(|ts| (ts, id.to_string()))) + .ok_or_else(|| { + ApiBadRequest { + reason: "cursor must be formatted as `{create_ts}:{delivery_id}`".to_string(), + } + .build() + })?; + + deliveries.retain(|d| { + (d.record.created_at, d.delivery_id.as_str()) < (created_at, delivery_id.as_str()) + }); + } + + let limit = query.limit.unwrap_or(DEFAULT_EVENTS_LIMIT); + let has_more = deliveries.len() > limit; + deliveries.truncate(limit); + + let cursor = has_more + .then(|| deliveries.last()) + .flatten() + .map(|last| format!("{}:{}", last.record.created_at, last.delivery_id)); + + Ok(EventsResponse { + events: deliveries + .into_iter() + .map(|d| WebhookEvent { + id: d.delivery_id, + create_ts: d.record.created_at, + status: match d.record.status { + webhook::types::DeliveryStatus::Pending => "pending".to_string(), + webhook::types::DeliveryStatus::Succeeded => "succeeded".to_string(), + webhook::types::DeliveryStatus::Failed => "failed".to_string(), + }, + event_type: d.record.event_type.as_str().to_string(), + attempt_count: d.record.attempt_count, + last_error: d.record.last_error, + }) + .collect(), + pagination: Pagination { cursor }, + }) +} diff --git a/engine/packages/config/src/config/mod.rs b/engine/packages/config/src/config/mod.rs index 5ee03c6d2a..babed64296 100644 --- a/engine/packages/config/src/config/mod.rs +++ b/engine/packages/config/src/config/mod.rs @@ -11,6 +11,7 @@ pub mod db; pub mod guard; pub mod logs; pub mod metrics; +pub mod outbound; pub mod pegboard; pub mod pubsub; pub mod pyroscope; @@ -27,6 +28,7 @@ pub use db::Database; pub use guard::*; pub use logs::*; pub use metrics::*; +pub use outbound::*; pub use pegboard::*; pub use pubsub::PubSub; pub use pyroscope::*; @@ -110,6 +112,9 @@ pub struct Root { #[serde(default)] pub pyroscope: Option, + + #[serde(default)] + pub outbound: Option, } impl Default for Root { @@ -130,6 +135,7 @@ impl Default for Root { sqlite: None, metrics: Default::default(), pyroscope: None, + outbound: None, } } } @@ -145,6 +151,11 @@ impl Root { self.api_peer.as_ref().unwrap_or(&DEFAULT) } + pub fn outbound(&self) -> &Outbound { + static DEFAULT: LazyLock = LazyLock::new(Outbound::default); + self.outbound.as_ref().unwrap_or(&DEFAULT) + } + pub fn pegboard(&self) -> &Pegboard { static DEFAULT: LazyLock = LazyLock::new(Pegboard::default); self.pegboard.as_ref().unwrap_or(&DEFAULT) diff --git a/engine/packages/config/src/config/outbound.rs b/engine/packages/config/src/config/outbound.rs new file mode 100644 index 0000000000..d0aa65b35f --- /dev/null +++ b/engine/packages/config/src/config/outbound.rs @@ -0,0 +1,60 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Policy for outbound HTTP requests to user-configured destinations, such as serverless runner +/// URLs. +/// +/// These requests originate from inside the trusted engine network, so without restrictions a +/// caller who can configure a runner can reach internal-only services. The defaults deny every +/// non-globally-routable destination except loopback. +#[derive(Debug, Serialize, Deserialize, Clone, Default, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Outbound { + /// Allow destinations that resolve to loopback addresses (127.0.0.0/8, ::1). + /// + /// Enabled by default so local development against `http://localhost:...` works without + /// configuration. + pub allow_loopback: Option, + /// Allow destinations that resolve to private, link-local, shared (CGNAT), or otherwise + /// non-globally-routable addresses. + /// + /// Self-hosted deployments that point runners at addresses on their own network, such as a + /// Docker Compose service name, need this enabled. + pub allow_private_networks: Option, + /// Allow plaintext `http://` destinations. When disabled only `https://` is permitted. + pub allow_insecure_scheme: Option, + /// Additional CIDRs that are always permitted, evaluated after `deny_cidrs`. + /// + /// Use this to reach a specific internal service without opening up the whole private range. + pub allow_cidrs: Option>, + /// Additional CIDRs that are always denied. Takes precedence over every allow rule. + pub deny_cidrs: Option>, + /// Maximum number of redirects to follow. Every hop is re-checked against this policy. + pub max_redirects: Option, +} + +impl Outbound { + pub fn allow_loopback(&self) -> bool { + self.allow_loopback.unwrap_or(true) + } + + pub fn allow_private_networks(&self) -> bool { + self.allow_private_networks.unwrap_or(false) + } + + pub fn allow_insecure_scheme(&self) -> bool { + self.allow_insecure_scheme.unwrap_or(true) + } + + pub fn allow_cidrs(&self) -> &[String] { + self.allow_cidrs.as_deref().unwrap_or(&[]) + } + + pub fn deny_cidrs(&self) -> &[String] { + self.deny_cidrs.as_deref().unwrap_or(&[]) + } + + pub fn max_redirects(&self) -> usize { + self.max_redirects.unwrap_or(4) + } +} diff --git a/engine/packages/outbound-guard/Cargo.toml b/engine/packages/outbound-guard/Cargo.toml new file mode 100644 index 0000000000..a47d61ea49 --- /dev/null +++ b/engine/packages/outbound-guard/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rivet-outbound-guard" +publish = false +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +ipnet.workspace = true +reqwest.workspace = true +rivet-config.workspace = true +thiserror.workspace = true +tokio.workspace = true +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/engine/packages/outbound-guard/src/client.rs b/engine/packages/outbound-guard/src/client.rs new file mode 100644 index 0000000000..8dc44cedf8 --- /dev/null +++ b/engine/packages/outbound-guard/src/client.rs @@ -0,0 +1,82 @@ +use std::net::SocketAddr; +use std::sync::Arc; + +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; + +use crate::policy::{BlockReason, Policy}; + +/// A DNS resolver that drops every address the policy disallows. +/// +/// Placing the check here rather than before the request closes the DNS rebinding window: reqwest +/// connects to exactly the addresses this returns, and it runs for every redirect hop as well as +/// the initial request. +#[derive(Debug)] +pub struct GuardedResolver { + policy: Arc, +} + +impl GuardedResolver { + pub fn new(policy: Arc) -> Self { + GuardedResolver { policy } + } +} + +impl Resolve for GuardedResolver { + fn resolve(&self, name: Name) -> Resolving { + let policy = self.policy.clone(); + let host = name.as_str().to_string(); + + Box::pin(async move { + // The port is discarded by the connector, which substitutes the real one. + let resolved = tokio::net::lookup_host((host.as_str(), 0)) + .await + .map_err(|err| { + tracing::debug!(%host, ?err, "failed to resolve outbound host"); + Box::new(BlockReason::ResolutionFailed { host: host.clone() }) + as Box + })? + .map(|addr| addr.ip()) + .collect::>(); + + let addrs = policy + .filter_addrs(&host, resolved)? + .into_iter() + .map(|ip| SocketAddr::new(ip, 0)) + .collect::>(); + + Ok(Box::new(addrs.into_iter()) as Addrs) + }) + } +} + +/// Build the redirect policy a guarded client must be configured with. +/// +/// Every hop is re-checked, so a destination cannot bounce the engine somewhere it was not +/// allowed to reach directly. +pub fn redirect_policy(policy: Arc) -> reqwest::redirect::Policy { + let max_redirects = policy.max_redirects(); + + reqwest::redirect::Policy::custom(move |attempt| { + if attempt.previous().len() >= max_redirects { + return attempt.error(BlockReason::TooManyRedirects { max: max_redirects }); + } + + match policy.check_url(attempt.url()) { + Ok(()) => attempt.follow(), + Err(reason) => { + tracing::debug!(url = %attempt.url(), %reason, "blocked outbound redirect"); + attempt.error(reason) + } + } + }) +} + +/// Recover the [`BlockReason`] that caused a request to fail, if the policy is what stopped it. +/// +/// The reason is buried in the source chain of a `reqwest::Error`, so callers that want to tell +/// "the destination is not allowed" apart from "the destination is down" have to walk it. +pub fn block_reason(err: &anyhow::Error) -> Option { + err.chain() + .find_map(|err| err.downcast_ref::()) + .cloned() +} diff --git a/engine/packages/outbound-guard/src/lib.rs b/engine/packages/outbound-guard/src/lib.rs new file mode 100644 index 0000000000..4076d4299f --- /dev/null +++ b/engine/packages/outbound-guard/src/lib.rs @@ -0,0 +1,14 @@ +//! Destination policy for outbound HTTP requests to user-configured URLs. +//! +//! Serverless runner URLs are supplied by whoever can write a runner config, and the engine dials +//! them from inside its own trusted network. This crate is the trust boundary for those requests: +//! it decides which destinations are reachable, and supplies the resolver and redirect policy that +//! enforce that decision at connect time. +//! +//! Clients are built in `rivet_pools::reqwest`, which owns every `reqwest::Client` in the process. + +mod client; +mod policy; + +pub use client::{GuardedResolver, block_reason, redirect_policy}; +pub use policy::{AddressClass, BlockReason, Policy}; diff --git a/engine/packages/outbound-guard/src/policy.rs b/engine/packages/outbound-guard/src/policy.rs new file mode 100644 index 0000000000..a0d255c9b7 --- /dev/null +++ b/engine/packages/outbound-guard/src/policy.rs @@ -0,0 +1,375 @@ +use std::fmt; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use anyhow::{Context, Result}; +use ipnet::IpNet; +use url::Url; + +/// Why a destination was rejected. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum BlockReason { + #[error("url is not a valid absolute url")] + InvalidUrl, + #[error("scheme {scheme:?} is not allowed, expected http or https")] + UnsupportedScheme { scheme: String }, + #[error("plaintext http is not allowed, use https")] + InsecureScheme, + #[error("url must not contain embedded credentials")] + EmbeddedCredentials, + #[error("url has no host")] + MissingHost, + #[error("address {addr} is not an allowed destination ({class})")] + BlockedAddress { addr: IpAddr, class: AddressClass }, + #[error("address {addr} is explicitly denied")] + DeniedAddress { addr: IpAddr }, + #[error("host {host:?} resolved to no allowed addresses")] + NoAllowedAddresses { host: String }, + #[error("failed to resolve host {host:?}")] + ResolutionFailed { host: String }, + #[error("exceeded the maximum of {max} redirects")] + TooManyRedirects { max: usize }, +} + +/// The reason an address is not globally routable. +/// +/// Every class except [`AddressClass::Loopback`] is governed by +/// `outbound.allow_private_networks`. An address that matches no class is globally +/// routable and always allowed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AddressClass { + /// `0.0.0.0` or `::`, which the kernel routes to a local interface. + Unspecified, + /// `127.0.0.0/8` or `::1`. + Loopback, + /// `169.254.0.0/16` or `fe80::/10`. Covers the cloud metadata endpoint. + LinkLocal, + /// `10/8`, `172.16/12`, `192.168/16`. + Private, + /// `255.255.255.255`. + Broadcast, + /// `224.0.0.0/4` or `ff00::/8`. + Multicast, + /// `192.0.2.0/24`, `198.51.100.0/24`, `203.0.113.0/24`, or `2001:db8::/32`. + Documentation, + /// `100.64.0.0/10`, the carrier-grade NAT range. + Shared, + /// `192.0.0.0/24`, reserved for IETF protocol assignments. + ProtocolAssignments, + /// `198.18.0.0/15`, reserved for network benchmarking. + Benchmarking, + /// `240.0.0.0/4`. + Reserved, + /// `fc00::/7`, the IPv6 equivalent of the private ranges. + UniqueLocal, + /// `100::/64`, which is discarded rather than routed. + Discard, + /// An IPv6 address carrying an IPv4 destination that is itself globally routable. + /// + /// These are held to the same rule as the private classes because they are an easy way to + /// smuggle a destination past a filter that only understands one address family. + Ipv4Embedded, +} + +impl fmt::Display for AddressClass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + AddressClass::Unspecified => "unspecified", + AddressClass::Loopback => "loopback", + AddressClass::LinkLocal => "link-local", + AddressClass::Private => "private", + AddressClass::Broadcast => "broadcast", + AddressClass::Multicast => "multicast", + AddressClass::Documentation => "documentation", + AddressClass::Shared => "shared", + AddressClass::ProtocolAssignments => "protocol-assignments", + AddressClass::Benchmarking => "benchmarking", + AddressClass::Reserved => "reserved", + AddressClass::UniqueLocal => "unique-local", + AddressClass::Discard => "discard", + AddressClass::Ipv4Embedded => "ipv4-embedded", + }; + + f.write_str(name) + } +} + +impl AddressClass { + /// Classify an address, or `None` if it is globally routable. + pub fn of(addr: IpAddr) -> Option { + match addr { + IpAddr::V4(v4) => AddressClass::of_ipv4(v4), + IpAddr::V6(v6) => AddressClass::of_ipv6(v6), + } + } + + fn of_ipv4(addr: Ipv4Addr) -> Option { + let [a, b, c, _] = addr.octets(); + + if addr.is_unspecified() { + return Some(AddressClass::Unspecified); + } + if addr.is_loopback() { + return Some(AddressClass::Loopback); + } + if addr.is_link_local() { + return Some(AddressClass::LinkLocal); + } + if addr.is_private() { + return Some(AddressClass::Private); + } + if addr.is_broadcast() { + return Some(AddressClass::Broadcast); + } + if addr.is_multicast() { + return Some(AddressClass::Multicast); + } + if addr.is_documentation() { + return Some(AddressClass::Documentation); + } + if a == 100 && (64..128).contains(&b) { + return Some(AddressClass::Shared); + } + if a == 192 && b == 0 && c == 0 { + return Some(AddressClass::ProtocolAssignments); + } + if a == 198 && (b == 18 || b == 19) { + return Some(AddressClass::Benchmarking); + } + if a >= 240 { + return Some(AddressClass::Reserved); + } + + None + } + + fn of_ipv6(addr: Ipv6Addr) -> Option { + // An IPv4 address wearing an IPv6 costume routes to the embedded IPv4 destination, so + // classify it as that address rather than trusting the outer form. + if let Some(v4) = unwrap_embedded_ipv4(addr) { + return AddressClass::of_ipv4(v4).or(Some(AddressClass::Ipv4Embedded)); + } + + let segments = addr.segments(); + + if addr.is_unspecified() { + return Some(AddressClass::Unspecified); + } + if addr.is_loopback() { + return Some(AddressClass::Loopback); + } + if addr.is_multicast() { + return Some(AddressClass::Multicast); + } + if segments[0] & 0xfe00 == 0xfc00 { + return Some(AddressClass::UniqueLocal); + } + if segments[0] & 0xffc0 == 0xfe80 { + return Some(AddressClass::LinkLocal); + } + if segments[0] == 0x2001 && segments[1] == 0x0db8 { + return Some(AddressClass::Documentation); + } + if segments[0] == 0x0100 && segments[1..4] == [0, 0, 0] { + return Some(AddressClass::Discard); + } + + None + } +} + +/// Extract the IPv4 destination an IPv6 address actually routes to, if any. +/// +/// Covers IPv4-mapped (`::ffff:0:0/96`), IPv4-compatible (`::/96`), and the well-known NAT64 +/// prefix (`64:ff9b::/96`). +fn unwrap_embedded_ipv4(addr: Ipv6Addr) -> Option { + if let Some(v4) = addr.to_ipv4_mapped() { + return Some(v4); + } + + let segments = addr.segments(); + let tail = Ipv4Addr::new( + (segments[6] >> 8) as u8, + (segments[6] & 0xff) as u8, + (segments[7] >> 8) as u8, + (segments[7] & 0xff) as u8, + ); + + if segments[0] == 0x0064 && segments[1] == 0xff9b && segments[2..6] == [0, 0, 0, 0] { + return Some(tail); + } + + // IPv4-compatible addresses are deprecated but still routed. `::` and `::1` have their own + // classes, so skip anything in the lowest /104. + if segments[0..6] == [0, 0, 0, 0, 0, 0] && segments[6] != 0 { + return Some(tail); + } + + None +} + +/// Which destinations the engine may reach on behalf of a user-supplied URL. +#[derive(Debug, Clone)] +pub struct Policy { + allow_loopback: bool, + allow_private_networks: bool, + allow_insecure_scheme: bool, + allow_cidrs: Vec, + deny_cidrs: Vec, + max_redirects: usize, +} + +impl Policy { + pub fn from_config(config: &rivet_config::Config) -> Result { + let outbound = config.outbound(); + + Ok(Policy { + allow_loopback: outbound.allow_loopback(), + allow_private_networks: outbound.allow_private_networks(), + allow_insecure_scheme: outbound.allow_insecure_scheme(), + allow_cidrs: parse_cidrs(outbound.allow_cidrs(), "outbound.allow_cidrs")?, + deny_cidrs: parse_cidrs(outbound.deny_cidrs(), "outbound.deny_cidrs")?, + max_redirects: outbound.max_redirects(), + }) + } + + pub fn max_redirects(&self) -> usize { + self.max_redirects + } + + /// Check everything about a destination that can be known without resolving DNS. + /// + /// This is the gate that runs when a runner config is written, so a bad URL is rejected before + /// it is ever stored. It is also re-run on every redirect hop. + pub fn check_url(&self, url: &Url) -> Result<(), BlockReason> { + match url.scheme() { + "https" => {} + "http" => { + if !self.allow_insecure_scheme { + return Err(BlockReason::InsecureScheme); + } + } + scheme => { + return Err(BlockReason::UnsupportedScheme { + scheme: scheme.to_string(), + }); + } + } + + // Credentials in the URL would be replayed to whatever the destination redirects to. + if !url.username().is_empty() || url.password().is_some() { + return Err(BlockReason::EmbeddedCredentials); + } + + let Some(host) = url.host() else { + return Err(BlockReason::MissingHost); + }; + + // A hostname is checked once it resolves, at connect time. An address literal skips the + // resolver entirely, so it has to be checked here. + match host { + url::Host::Ipv4(addr) => self.check_addr(IpAddr::V4(addr)), + url::Host::Ipv6(addr) => self.check_addr(IpAddr::V6(addr)), + url::Host::Domain(_) => Ok(()), + } + } + + /// Check a single resolved address. + pub fn check_addr(&self, addr: IpAddr) -> Result<(), BlockReason> { + // An IPv6 address that carries an IPv4 destination has to match CIDR rules written for + // either form, so both are tested. + let mut forms = vec![addr]; + if let IpAddr::V6(v6) = addr { + if let Some(v4) = unwrap_embedded_ipv4(v6) { + forms.push(IpAddr::V4(v4)); + } + } + + // An explicit deny always wins, including over the allow list. + if forms + .iter() + .any(|form| self.deny_cidrs.iter().any(|net| net.contains(form))) + { + return Err(BlockReason::DeniedAddress { addr }); + } + + if forms + .iter() + .any(|form| self.allow_cidrs.iter().any(|net| net.contains(form))) + { + return Ok(()); + } + + let Some(class) = AddressClass::of(addr) else { + return Ok(()); + }; + + let allowed = match class { + AddressClass::Loopback => self.allow_loopback, + AddressClass::Unspecified + | AddressClass::LinkLocal + | AddressClass::Private + | AddressClass::Broadcast + | AddressClass::Multicast + | AddressClass::Documentation + | AddressClass::Shared + | AddressClass::ProtocolAssignments + | AddressClass::Benchmarking + | AddressClass::Reserved + | AddressClass::UniqueLocal + | AddressClass::Discard + | AddressClass::Ipv4Embedded => self.allow_private_networks, + }; + + if allowed { + Ok(()) + } else { + Err(BlockReason::BlockedAddress { addr, class }) + } + } + + /// Filter a resolver's answer down to the addresses this policy permits. + /// + /// Dropping individual addresses rather than rejecting the whole answer keeps a dual-stack + /// host reachable when only one of its families is allowed, and still guarantees the + /// connection can only land on an address that passed. + pub fn filter_addrs( + &self, + host: &str, + addrs: impl IntoIterator, + ) -> Result, BlockReason> { + let mut allowed = Vec::new(); + + for addr in addrs { + match self.check_addr(addr) { + Ok(()) => allowed.push(addr), + Err(reason) => { + tracing::debug!(%host, %addr, %reason, "dropped disallowed resolved address"); + } + } + } + + if allowed.is_empty() { + Err(BlockReason::NoAllowedAddresses { + host: host.to_string(), + }) + } else { + Ok(allowed) + } + } +} + +fn parse_cidrs(raw: &[String], label: &str) -> Result> { + raw.iter() + .map(|entry| { + let entry = entry.trim(); + // Accept a bare address as a single-host CIDR so operators do not have to write /32. + if let Ok(addr) = entry.parse::() { + return Ok(IpNet::from(addr)); + } + + entry + .parse::() + .with_context(|| format!("invalid cidr in {label}: {entry:?}")) + }) + .collect() +} diff --git a/engine/packages/outbound-guard/tests/policy.rs b/engine/packages/outbound-guard/tests/policy.rs new file mode 100644 index 0000000000..dd4b19b368 --- /dev/null +++ b/engine/packages/outbound-guard/tests/policy.rs @@ -0,0 +1,240 @@ +use std::net::IpAddr; + +use rivet_config::config::{Outbound, Root}; +use rivet_outbound_guard::{AddressClass, BlockReason, Policy}; +use url::Url; + +fn policy(outbound: Outbound) -> Policy { + let config = rivet_config::Config::from_root(Root { + outbound: Some(outbound), + ..Default::default() + }); + + Policy::from_config(&config).expect("policy should build") +} + +fn default_policy() -> Policy { + policy(Outbound::default()) +} + +fn check(policy: &Policy, url: &str) -> Result<(), BlockReason> { + policy.check_url(&Url::parse(url).expect("test url should parse")) +} + +fn addr(raw: &str) -> IpAddr { + raw.parse().expect("test address should parse") +} + +#[test] +fn allows_public_destinations() { + let policy = default_policy(); + + check(&policy, "https://runner.example.com/start").expect("public host should be allowed"); + check(&policy, "https://8.8.8.8/start").expect("public literal should be allowed"); + policy + .check_addr(addr("2606:4700::1")) + .expect("public v6 should be allowed"); +} + +#[test] +fn allows_loopback_by_default() { + let policy = default_policy(); + + check(&policy, "http://127.0.0.1:6420/start").expect("loopback v4 should be allowed"); + check(&policy, "http://[::1]:6420/start").expect("loopback v6 should be allowed"); +} + +#[test] +fn denies_loopback_when_disabled() { + let policy = policy(Outbound { + allow_loopback: Some(false), + ..Default::default() + }); + + assert_eq!( + check(&policy, "http://127.0.0.1:6420/start"), + Err(BlockReason::BlockedAddress { + addr: addr("127.0.0.1"), + class: AddressClass::Loopback, + }), + ); +} + +#[test] +fn denies_private_ranges_by_default() { + let policy = default_policy(); + + for (raw, class) in [ + ("10.0.0.5", AddressClass::Private), + ("172.16.4.1", AddressClass::Private), + ("192.168.1.1", AddressClass::Private), + ("169.254.169.254", AddressClass::LinkLocal), + ("100.64.0.1", AddressClass::Shared), + ("198.18.0.1", AddressClass::Benchmarking), + ("0.0.0.0", AddressClass::Unspecified), + ("240.0.0.1", AddressClass::Reserved), + ("fd00::1", AddressClass::UniqueLocal), + ("fe80::1", AddressClass::LinkLocal), + ] { + assert_eq!( + policy.check_addr(addr(raw)), + Err(BlockReason::BlockedAddress { + addr: addr(raw), + class, + }), + "{raw} should be blocked", + ); + } +} + +#[test] +fn denies_ipv6_wrapped_ipv4_metadata_endpoint() { + let policy = default_policy(); + + // The same destination reached through three different IPv6 encodings. + for raw in ["::ffff:169.254.169.254", "64:ff9b::169.254.169.254"] { + assert_eq!( + policy.check_addr(addr(raw)), + Err(BlockReason::BlockedAddress { + addr: addr(raw), + class: AddressClass::LinkLocal, + }), + "{raw} should be blocked", + ); + } +} + +#[test] +fn allows_private_ranges_when_enabled() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + ..Default::default() + }); + + policy + .check_addr(addr("10.0.0.5")) + .expect("private should be allowed when enabled"); + policy + .check_addr(addr("169.254.169.254")) + .expect("link-local should be allowed when enabled"); +} + +#[test] +fn allow_cidrs_open_a_single_destination() { + let policy = policy(Outbound { + allow_cidrs: Some(vec!["10.1.2.0/24".to_string(), "192.168.5.9".to_string()]), + ..Default::default() + }); + + policy + .check_addr(addr("10.1.2.7")) + .expect("allow-listed cidr should be allowed"); + policy + .check_addr(addr("192.168.5.9")) + .expect("bare address should be read as a single-host cidr"); + assert!( + policy.check_addr(addr("10.1.3.7")).is_err(), + "address outside the allow-listed cidr should stay blocked", + ); +} + +#[test] +fn deny_cidrs_win_over_allow_rules() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + allow_cidrs: Some(vec!["169.254.0.0/16".to_string()]), + deny_cidrs: Some(vec!["169.254.169.254".to_string()]), + ..Default::default() + }); + + policy + .check_addr(addr("169.254.1.1")) + .expect("rest of the range should still be reachable"); + assert_eq!( + policy.check_addr(addr("169.254.169.254")), + Err(BlockReason::DeniedAddress { + addr: addr("169.254.169.254"), + }), + ); +} + +#[test] +fn deny_cidrs_catch_the_ipv6_wrapped_form() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + deny_cidrs: Some(vec!["169.254.169.254".to_string()]), + ..Default::default() + }); + + assert_eq!( + policy.check_addr(addr("::ffff:169.254.169.254")), + Err(BlockReason::DeniedAddress { + addr: addr("::ffff:169.254.169.254"), + }), + ); +} + +#[test] +fn rejects_non_http_schemes() { + let policy = default_policy(); + + assert_eq!( + check(&policy, "file:///etc/passwd"), + Err(BlockReason::UnsupportedScheme { + scheme: "file".to_string(), + }), + ); + assert_eq!( + check(&policy, "gopher://example.com/"), + Err(BlockReason::UnsupportedScheme { + scheme: "gopher".to_string(), + }), + ); +} + +#[test] +fn rejects_plaintext_http_when_disabled() { + let policy = policy(Outbound { + allow_insecure_scheme: Some(false), + ..Default::default() + }); + + assert_eq!( + check(&policy, "http://runner.example.com/"), + Err(BlockReason::InsecureScheme), + ); + check(&policy, "https://runner.example.com/").expect("https should still be allowed"); +} + +#[test] +fn rejects_embedded_credentials() { + let policy = default_policy(); + + assert_eq!( + check(&policy, "https://user:pass@runner.example.com/"), + Err(BlockReason::EmbeddedCredentials), + ); +} + +#[test] +fn filter_addrs_keeps_only_allowed_answers() { + let policy = default_policy(); + + let allowed = policy + .filter_addrs( + "rebind.example.com", + [addr("10.0.0.1"), addr("93.184.216.34")], + ) + .expect("a dual answer with one public address should still connect"); + assert_eq!(allowed, vec![addr("93.184.216.34")]); + + assert_eq!( + policy.filter_addrs( + "rebind.example.com", + [addr("10.0.0.1"), addr("192.168.0.1")] + ), + Err(BlockReason::NoAllowedAddresses { + host: "rebind.example.com".to_string(), + }), + ); +} diff --git a/engine/packages/pegboard-outbound/Cargo.toml b/engine/packages/pegboard-outbound/Cargo.toml index e48c0ba430..6095f44b01 100644 --- a/engine/packages/pegboard-outbound/Cargo.toml +++ b/engine/packages/pegboard-outbound/Cargo.toml @@ -18,10 +18,13 @@ reqwest.workspace = true rivet-config.workspace = true rivet-envoy-protocol.workspace = true rivet-metrics.workspace = true +rivet-outbound-guard.workspace = true +rivet-pools.workspace = true rivet-runtime.workspace = true rivet-types.workspace = true tokio.workspace = true tracing.workspace = true universaldb.workspace = true universalpubsub.workspace = true +url.workspace = true vbare.workspace = true diff --git a/engine/packages/pegboard-outbound/src/lib.rs b/engine/packages/pegboard-outbound/src/lib.rs index 00697ea7b0..4a1341fd7c 100644 --- a/engine/packages/pegboard-outbound/src/lib.rs +++ b/engine/packages/pegboard-outbound/src/lib.rs @@ -327,6 +327,7 @@ fn error_label(error: &RunnerPoolError) -> &'static str { match error { RunnerPoolError::ServerlessHttpError { .. } => "http_error", RunnerPoolError::ServerlessConnectionError { .. } => "connection_error", + RunnerPoolError::ServerlessDestinationBlocked { .. } => "destination_blocked", RunnerPoolError::ServerlessStreamEndedEarly => "stream_ended_early", RunnerPoolError::ServerlessInvalidSsePayload { .. } => "invalid_payload", RunnerPoolError::Downgrade => "downgrade", @@ -345,6 +346,7 @@ fn status_label(error: &RunnerPoolError) -> &'static str { _ => "other", }, RunnerPoolError::ServerlessConnectionError { .. } + | RunnerPoolError::ServerlessDestinationBlocked { .. } | RunnerPoolError::ServerlessStreamEndedEarly | RunnerPoolError::ServerlessInvalidSsePayload { .. } | RunnerPoolError::Downgrade @@ -362,6 +364,7 @@ fn error_result_label(error: &RunnerPoolError) -> &'static str { _ => "error_http_other", }, RunnerPoolError::ServerlessConnectionError { .. } => "error_connection", + RunnerPoolError::ServerlessDestinationBlocked { .. } => "error_destination_blocked", RunnerPoolError::ServerlessStreamEndedEarly => "error_stream_ended", RunnerPoolError::ServerlessInvalidSsePayload { .. } => "error_invalid_payload", RunnerPoolError::Downgrade => "error_downgrade", @@ -441,7 +444,37 @@ async fn serverless_outbound_req( let endpoint_url = format!("{}/start", url.trim_end_matches('/')); - let client = rivet_pools::reqwest::client_no_timeout().await?; + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here, which also re-gates configs stored before this + // policy existed. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + let block_reason = match url::Url::parse(&endpoint_url) { + Ok(parsed_url) => policy.check_url(&parsed_url).err(), + Err(_) => Some(rivet_outbound_guard::BlockReason::InvalidUrl), + }; + if let Some(reason) = block_reason { + tracing::warn!( + ?namespace_id, + %pool_name, + %reason, + "serverless url is not an allowed destination, dropping outbound req" + ); + + report_error( + ctx, + namespace_id, + pool_name, + RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + ) + .await; + + return Ok(()); + } + + let client = rivet_pools::reqwest::guarded_client_no_timeout(ctx.config()).await?; let req = client .post(endpoint_url.clone()) .body(payload) @@ -550,13 +583,20 @@ async fn serverless_outbound_req( Err(err) => { let wrapped_err = anyhow::Error::from(err); - let error = RunnerPoolError::ServerlessConnectionError { - // Print entire error chain - message: wrapped_err - .chain() - .map(|err| err.to_string()) - .collect::>() - .join("\n"), + // A hostname that only resolves to disallowed addresses is rejected by the + // resolver, which the pre-flight check above cannot see. + let error = match rivet_outbound_guard::block_reason(&wrapped_err) { + Some(reason) => RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + None => RunnerPoolError::ServerlessConnectionError { + // Print entire error chain + message: wrapped_err + .chain() + .map(|err| err.to_string()) + .collect::>() + .join("\n"), + }, }; report_error(ctx, namespace_id, &pool_name, error.clone()).await; observe_req_duration( diff --git a/engine/packages/pegboard/Cargo.toml b/engine/packages/pegboard/Cargo.toml index 6e9701246f..7f47ada9b1 100644 --- a/engine/packages/pegboard/Cargo.toml +++ b/engine/packages/pegboard/Cargo.toml @@ -30,6 +30,7 @@ rivet-data.workspace = true rivet-envoy-protocol.workspace = true rivet-error.workspace = true rivet-metrics.workspace = true +rivet-outbound-guard.workspace = true rivet-pools.workspace = true rivet-runner-protocol.workspace = true rivet-runtime.workspace = true @@ -52,6 +53,7 @@ url.workspace = true utoipa.workspace = true uuid.workspace = true vbare.workspace = true +webhook.workspace = true [dev-dependencies] portpicker.workspace = true diff --git a/engine/packages/pegboard/src/ops/runner_config/upsert.rs b/engine/packages/pegboard/src/ops/runner_config/upsert.rs index e5d2c3c4b3..ad3865612e 100644 --- a/engine/packages/pegboard/src/ops/runner_config/upsert.rs +++ b/engine/packages/pegboard/src/ops/runner_config/upsert.rs @@ -34,9 +34,23 @@ pub async fn pegboard_runner_config_upsert(ctx: &OperationCtx, input: &Input) -> slots_per_runner, .. } => { - if let Err(err) = url::Url::parse(url) { + let parsed_url = match url::Url::parse(url) { + Ok(parsed_url) => parsed_url, + Err(err) => { + return Err(errors::RunnerConfig::Invalid { + reason: format!("invalid serverless url: {err}"), + } + .build()); + } + }; + + // Reject destinations the engine is not allowed to reach before the config is stored. + // Requests are checked again when they connect, which catches configs written before + // this gate existed and hosts whose DNS answer changes afterwards. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { return Err(errors::RunnerConfig::Invalid { - reason: format!("invalid serverless url: {err}"), + reason: format!("invalid serverless url: {reason}"), } .build()); } diff --git a/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs b/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs index 2d25ba3c70..da4dab6cd8 100644 --- a/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs +++ b/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs @@ -21,6 +21,7 @@ pub struct Input { #[derive(Clone, Debug, PartialEq, Eq)] pub enum ServerlessMetadataError { InvalidRequest {}, + DestinationBlocked { reason: String }, RequestFailed {}, RequestTimedOut {}, NonSuccessStatus { status_code: u16, body: String }, @@ -59,6 +60,14 @@ impl From for ServerlessMetadataErrorEnvelope { details: None, metadata: serde_json::json!({ "kind": "invalid_request" }), }, + ServerlessMetadataError::DestinationBlocked { reason } => Self { + message: "serverless endpoint is not an allowed destination".to_string(), + details: Some(reason.clone()), + metadata: serde_json::json!({ + "kind": "destination_blocked", + "reason": reason, + }), + }, ServerlessMetadataError::RequestFailed {} => Self { message: "failed to reach serverless endpoint".to_string(), details: None, @@ -158,8 +167,19 @@ pub async fn pegboard_serverless_metadata_fetch( let metadata_url = format!("{}/metadata", trimmed_url.trim_end_matches('/')); - if reqwest::Url::parse(&metadata_url).is_err() { + let Ok(parsed_url) = reqwest::Url::parse(&metadata_url) else { return Ok(Err(ServerlessMetadataError::InvalidRequest {})); + }; + + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here. This op also backs the health check endpoint, + // where an unchecked URL is a probe for whatever the engine can reach. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { + return Ok(Err(ServerlessMetadataError::DestinationBlocked { + reason: reason.to_string(), + })); } let mut header_map = ReqwestHeaderMap::new(); @@ -177,7 +197,7 @@ pub async fn pegboard_serverless_metadata_fetch( header_map.insert(header_name, header_value); } - let client = match rivet_pools::reqwest::client().await { + let client = match rivet_pools::reqwest::guarded_client(ctx.config()).await { Ok(c) => c, Err(_) => return Ok(Err(ServerlessMetadataError::RequestFailed {})), }; @@ -193,7 +213,18 @@ pub async fn pegboard_serverless_metadata_fetch( { Ok(r) => r, Err(err) => { - return Ok(Err(if err.is_timeout() { + let is_timeout = err.is_timeout(); + let err = anyhow::Error::from(err); + + // A hostname that only resolves to disallowed addresses is rejected by the resolver, + // which the pre-flight check above cannot see. + if let Some(reason) = rivet_outbound_guard::block_reason(&err) { + return Ok(Err(ServerlessMetadataError::DestinationBlocked { + reason: reason.to_string(), + })); + } + + return Ok(Err(if is_timeout { ServerlessMetadataError::RequestTimedOut {} } else { ServerlessMetadataError::RequestFailed {} diff --git a/engine/packages/pegboard/src/workflows/runner_pool_error_tracker.rs b/engine/packages/pegboard/src/workflows/runner_pool_error_tracker.rs index d758645d92..36e82e1b80 100644 --- a/engine/packages/pegboard/src/workflows/runner_pool_error_tracker.rs +++ b/engine/packages/pegboard/src/workflows/runner_pool_error_tracker.rs @@ -2,10 +2,74 @@ use std::time::Duration; use gas::prelude::*; use rivet_types::actor::RunnerPoolError; +use webhook::types::{WebhookEvent, WebhookEventType}; const SIGNAL_DEBOUNCE: Duration = Duration::from_millis(250); const SIGNAL_BATCH_SIZE: usize = 1024; +// Cap on fields that carry bytes straight from the user's serverless endpoint. These are +// unvetted upstream output, and a webhook forwards them to a third party and stores them in the +// delivery record, which is a single UDB value bound by FoundationDB's 100KB limit. Truncating +// keeps a large error page from breaking delivery recording and bounds what gets relayed. +const MAX_RAW_FIELD_BYTES: usize = 4096; + +fn truncate_raw(value: &str) -> String { + if value.len() <= MAX_RAW_FIELD_BYTES { + return value.to_string(); + } + + // Step back to a char boundary so the truncated string stays valid UTF-8. + let mut end = MAX_RAW_FIELD_BYTES; + while end > 0 && !value.is_char_boundary(end) { + end -= 1; + } + + format!("{}... (truncated)", &value[..end]) +} + +// Bounds the passthrough fields before an error is sent outside the engine. Engine-authored +// strings such as `message` and `reason` are left alone; only fields echoing the user's endpoint +// verbatim are truncated. +fn truncate_error_for_webhook(error: &RunnerPoolError) -> RunnerPoolError { + match error { + RunnerPoolError::ServerlessHttpError { status_code, body } => { + RunnerPoolError::ServerlessHttpError { + status_code: *status_code, + body: truncate_raw(body), + } + } + RunnerPoolError::ServerlessInvalidSsePayload { + message, + raw_payload, + } => RunnerPoolError::ServerlessInvalidSsePayload { + message: message.clone(), + raw_payload: raw_payload.as_deref().map(truncate_raw), + }, + RunnerPoolError::ServerlessStreamEndedEarly => RunnerPoolError::ServerlessStreamEndedEarly, + RunnerPoolError::ServerlessConnectionError { message } => { + RunnerPoolError::ServerlessConnectionError { + message: message.clone(), + } + } + RunnerPoolError::ServerlessDestinationBlocked { reason } => { + RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.clone(), + } + } + RunnerPoolError::Downgrade => RunnerPoolError::Downgrade, + RunnerPoolError::InternalError => RunnerPoolError::InternalError, + } +} + +// CloudEvents `data` for a runner pool health transition. +#[derive(Debug, Serialize)] +struct RunnerPoolEventPayload<'a> { + namespace_id: Id, + runner_name: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, +} + #[derive(Debug, Serialize, Deserialize, Clone)] pub struct Input { pub namespace_id: Id, @@ -42,12 +106,16 @@ pub async fn pegboard_runner_pool_error_tracker( ctx.activity(InitStateInput {}).await?; + let namespace_id = input.namespace_id; + let runner_name = input.runner_name.clone(); + // Batch receive signals with debounce. This allows us to (a) not require polling if the pool // is idle and has no signals and (b) avoid a hot loop by debouncing signal processing. ctx.lupe() // Txn sizes can quickly get large in this workflow, need to commit loop more often .commit_interval(1) - .run(|ctx, _| { + .run(move |ctx, _| { + let runner_name = runner_name.clone(); Box::pin(async move { // Sleep until we receive a signal let signals_a = ctx.v(2).listen_n::
(SIGNAL_BATCH_SIZE).await?; @@ -72,11 +140,60 @@ pub async fn pegboard_runner_pool_error_tracker( .collect(); // Process signals - let shutdown = ctx - .activity(ProcessSignalsInput { - signals: signals_inner, - }) - .await?; + let ProcessSignalsOutput { + shutdown, + transitions, + } = ctx.activity(ProcessSignalsInput { + signals: signals_inner, + }) + .await?; + + for transition in transitions { + let (event_type, error) = match transition { + HealthTransition::Errored(error) => ( + WebhookEventType::RunnerPoolError, + Some(truncate_error_for_webhook(&error)), + ), + HealthTransition::Recovered => (WebhookEventType::RunnerPoolHealthy, None), + }; + + let webhook_names = ctx + .activity(ListSubscribedWebhooksInput { + namespace_id, + event_type, + }) + .await?; + + if webhook_names.is_empty() { + continue; + } + + let data = serde_json::to_value(RunnerPoolEventPayload { + namespace_id, + runner_name: &runner_name, + error, + })?; + + for webhook_name in webhook_names { + // `graceful_not_found` because a webhook can be deleted between the + // listing above and this signal. + ctx.signal(webhook::workflows::webhook::Trigger { + event: WebhookEvent { + event_type, + // CloudEvents `subject`: which runner pool within the namespace + // this event is about. + subject: Some(runner_name.clone()), + data: data.clone(), + }, + }) + .to_workflow::() + .tag("namespace_id", namespace_id) + .tag("name", webhook_name) + .graceful_not_found() + .send() + .await?; + } + } if shutdown { Ok(Loop::Break(())) @@ -105,11 +222,34 @@ pub struct ProcessSignalsInput { pub signals: Vec, } -/// Returns `true` if shutdown signal received. +/// A change in the pool's active error state. Edge-triggered: emitted only when the state +/// actually flips, not on every reported error or success. +#[derive(Debug, Serialize, Deserialize)] +pub enum HealthTransition { + /// The pool went from clean to having an active error. + Errored(RunnerPoolError), + /// The pool's active error cleared after enough consecutive successes. + Recovered, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct ProcessSignalsOutput { + /// `true` if a shutdown signal was received. + pub shutdown: bool, + /// Health transitions observed while processing this batch. Returned rather than acted on + /// here because this activity's result is replayed from history on workflow replay, so the + /// workflow body is the only place that can durably send one webhook signal per transition. + pub transitions: Vec, +} + #[activity(ProcessSignals)] -pub async fn process_signals(ctx: &ActivityCtx, input: &ProcessSignalsInput) -> Result { +pub async fn process_signals( + ctx: &ActivityCtx, + input: &ProcessSignalsInput, +) -> Result { let mut state = ctx.state::()?; let now = util::timestamp::now(); + let mut transitions = Vec::new(); for signal in &input.signals { match signal { @@ -121,6 +261,9 @@ pub async fn process_signals(ctx: &ActivityCtx, input: &ProcessSignalsInput) -> was_clean, "runner pool error tracker received error" ); + if was_clean { + transitions.push(HealthTransition::Errored(report.error.clone())); + } state.active_error = Some(ActiveError { timestamp: now, error: report.error.clone(), @@ -142,17 +285,51 @@ pub async fn process_signals(ctx: &ActivityCtx, input: &ProcessSignalsInput) -> consecutive_successes = state.consecutive_successes, "runner pool error tracker cleared active error" ); + transitions.push(HealthTransition::Recovered); } state.active_error = None; } } MainInner::Shutdown(_) => { - return Ok(true); + return Ok(ProcessSignalsOutput { + shutdown: true, + transitions, + }); } } } - Ok(false) + Ok(ProcessSignalsOutput { + shutdown: false, + transitions, + }) +} + +#[derive(Debug, Serialize, Deserialize, Hash)] +pub struct ListSubscribedWebhooksInput { + pub namespace_id: Id, + pub event_type: WebhookEventType, +} + +/// Names of the namespace's webhooks that are subscribed to `event_type`. The webhook workflow +/// re-checks its own subscription when the trigger arrives, so this filter is an optimization +/// that avoids signaling webhooks that would just drop the event. +#[activity(ListSubscribedWebhooks)] +pub async fn list_subscribed_webhooks( + ctx: &ActivityCtx, + input: &ListSubscribedWebhooksInput, +) -> Result> { + let webhooks = ctx + .op(webhook::ops::list::Input { + namespace_id: input.namespace_id, + }) + .await?; + + Ok(webhooks + .into_iter() + .filter(|webhook| webhook.config.subscriptions.contains(&input.event_type)) + .map(|webhook| webhook.name) + .collect()) } #[derive(Debug, Clone, Hash)] diff --git a/engine/packages/pegboard/src/workflows/serverless/conn.rs b/engine/packages/pegboard/src/workflows/serverless/conn.rs index 0064fab984..d621ba4976 100644 --- a/engine/packages/pegboard/src/workflows/serverless/conn.rs +++ b/engine/packages/pegboard/src/workflows/serverless/conn.rs @@ -299,9 +299,22 @@ async fn outbound_req_inner( let endpoint_url = format!("{}/start", url.trim_end_matches('/')); + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here, which also re-gates configs stored before this + // policy existed. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + let block_reason = match url::Url::parse(&endpoint_url) { + Ok(parsed_url) => policy.check_url(&parsed_url).err(), + Err(_) => Some(rivet_outbound_guard::BlockReason::InvalidUrl), + }; + if let Some(reason) = block_reason { + return Ok(blocked_destination(ctx, input, reason).await); + } + tracing::debug!(%endpoint_url, "sending outbound req"); - let client = rivet_pools::reqwest::client_no_timeout().await?; + let client = rivet_pools::reqwest::guarded_client_no_timeout(ctx.config()).await?; let req = client.get(endpoint_url).headers(headers); let conn_started = Instant::now(); @@ -411,11 +424,11 @@ async fn outbound_req_inner( _ => { let wrapped_err = anyhow::Error::from(err); - report_error( - ctx, - input.namespace_id, - &input.runner_name, - RunnerPoolError::ServerlessConnectionError { + let error = match rivet_outbound_guard::block_reason(&wrapped_err) { + Some(reason) => RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + None => RunnerPoolError::ServerlessConnectionError { // Print entire error chain message: wrapped_err .chain() @@ -423,8 +436,9 @@ async fn outbound_req_inner( .collect::>() .join("\n"), }, - ) - .await; + }; + + report_error(ctx, input.namespace_id, &input.runner_name, error).await; return Err(wrapped_err); } @@ -484,6 +498,34 @@ async fn outbound_req_inner( Ok(OutboundReqOutput::Draining { drain_sent: true }) } +/// Report a destination the policy refuses to dial and stop the connection loop. +/// +/// Retrying cannot help: the config has to change before this URL becomes reachable. +async fn blocked_destination( + ctx: &ActivityCtx, + input: &OutboundReqInput, + reason: rivet_outbound_guard::BlockReason, +) -> OutboundReqOutput { + tracing::warn!( + namespace_id = %input.namespace_id, + runner_name = %input.runner_name, + %reason, + "serverless url is not an allowed destination, ending outbound req" + ); + + report_error( + ctx, + input.namespace_id, + &input.runner_name, + RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + ) + .await; + + OutboundReqOutput::Draining { drain_sent: false } +} + /// Reads from the adjacent serverless runner wf which is keeping track of signals while this workflow runs /// outbound requests. #[tracing::instrument(skip_all)] diff --git a/engine/packages/pools/Cargo.toml b/engine/packages/pools/Cargo.toml index 2dcf4390b7..00fe13a97d 100644 --- a/engine/packages/pools/Cargo.toml +++ b/engine/packages/pools/Cargo.toml @@ -17,6 +17,7 @@ hyper-util.workspace = true lazy_static.workspace = true reqwest.workspace = true rivet-config.workspace = true +rivet-outbound-guard.workspace = true rivet-metrics.workspace = true rivet-util.workspace = true rustls.workspace = true diff --git a/engine/packages/pools/src/pools.rs b/engine/packages/pools/src/pools.rs index e2a10d1ac9..0c3acf8c91 100644 --- a/engine/packages/pools/src/pools.rs +++ b/engine/packages/pools/src/pools.rs @@ -46,7 +46,8 @@ impl Pools { // Initialize here to avoid cold starts elsewhere crate::reqwest::client().await?; - crate::reqwest::client_no_timeout().await?; + crate::reqwest::guarded_client(pool.config()).await?; + crate::reqwest::guarded_client_no_timeout(pool.config()).await?; Ok(pool) } diff --git a/engine/packages/pools/src/reqwest.rs b/engine/packages/pools/src/reqwest.rs index 779c08e219..24179f2bca 100644 --- a/engine/packages/pools/src/reqwest.rs +++ b/engine/packages/pools/src/reqwest.rs @@ -1,10 +1,21 @@ +use std::sync::Arc; + +use anyhow::{Context, Result}; use reqwest::Client; +use rivet_outbound_guard::{GuardedResolver, Policy}; use tokio::sync::OnceCell; static CLIENT: OnceCell = OnceCell::const_new(); -static CLIENT_NO_TIMEOUT: OnceCell = OnceCell::const_new(); +static GUARDED_CLIENT: OnceCell = OnceCell::const_new(); +static GUARDED_CLIENT_NO_TIMEOUT: OnceCell = OnceCell::const_new(); +static OUTBOUND_POLICY: OnceCell> = OnceCell::const_new(); static CLIENT_USER_AGENT: &str = concat!("RivetEngine/", env!("CARGO_PKG_VERSION")); +/// Client for trusted destinations inside the engine network, such as peer datacenters and epoxy +/// replicas. +/// +/// Never use this for a URL that came from user configuration. Those go through +/// [`guarded_client`], which restricts what the request can reach. pub async fn client() -> Result { CLIENT .get_or_try_init(|| async { @@ -17,9 +28,53 @@ pub async fn client() -> Result { .cloned() } -pub async fn client_no_timeout() -> Result { - CLIENT_NO_TIMEOUT - .get_or_try_init(|| async { Client::builder().user_agent(CLIENT_USER_AGENT).build() }) +/// Client for destinations that come from user configuration, such as serverless runner URLs. +/// +/// The `outbound` security policy is enforced at DNS resolution time and on every redirect, so +/// these requests cannot be steered at services only reachable from inside the engine network. +pub async fn guarded_client(config: &rivet_config::Config) -> Result { + GUARDED_CLIENT + .get_or_try_init(|| async { + build_guarded_client(config, Some(std::time::Duration::from_secs(30))).await + }) + .await + .cloned() +} + +/// Same as [`guarded_client`] but without a request timeout, for long-lived streaming requests +/// such as the serverless SSE connection. +pub async fn guarded_client_no_timeout(config: &rivet_config::Config) -> Result { + GUARDED_CLIENT_NO_TIMEOUT + .get_or_try_init(|| async { build_guarded_client(config, None).await }) + .await + .cloned() +} + +async fn build_guarded_client( + config: &rivet_config::Config, + timeout: Option, +) -> Result { + let policy = outbound_policy(config).await?; + + let mut builder = Client::builder() + .user_agent(CLIENT_USER_AGENT) + .dns_resolver(Arc::new(GuardedResolver::new(policy.clone()))) + .redirect(rivet_outbound_guard::redirect_policy(policy)); + + if let Some(timeout) = timeout { + builder = builder.timeout(timeout); + } + + builder.build().context("failed building guarded client") +} + +/// The destination policy applied to every request to a user-configured URL. +/// +/// Callers use this to reject a URL at the point it is submitted, before it is ever stored. The +/// guarded clients apply the same policy again when they connect. +pub async fn outbound_policy(config: &rivet_config::Config) -> Result> { + OUTBOUND_POLICY + .get_or_try_init(|| async { Policy::from_config(config).map(Arc::new) }) .await .cloned() } diff --git a/engine/packages/types/src/actor/error.rs b/engine/packages/types/src/actor/error.rs index c8a5e94095..158fddc036 100644 --- a/engine/packages/types/src/actor/error.rs +++ b/engine/packages/types/src/actor/error.rs @@ -15,6 +15,9 @@ pub enum RunnerPoolError { /// Serverless: SSE connection or network error ServerlessConnectionError { message: String }, + /// Serverless: the configured URL is not a destination the engine is allowed to reach + ServerlessDestinationBlocked { reason: String }, + /// Serverless: Runner sent invalid payload ServerlessInvalidSsePayload { message: String, diff --git a/engine/packages/universaldb/src/utils/keys.rs b/engine/packages/universaldb/src/utils/keys.rs index e8c61ebe13..a6c4fb8300 100644 --- a/engine/packages/universaldb/src/utils/keys.rs +++ b/engine/packages/universaldb/src/utils/keys.rs @@ -158,4 +158,6 @@ define_keys! { (130, GENERATION, "generation"), (131, ENVOY_HASH_IDX, "envoy_hash_idx"), (132, VIRTUAL_NODES, "virtual_nodes"), + (133, WEBHOOK, "webhook"), + (134, DELIVERY, "delivery"), } diff --git a/engine/packages/webhook/Cargo.toml b/engine/packages/webhook/Cargo.toml new file mode 100644 index 0000000000..0528958807 --- /dev/null +++ b/engine/packages/webhook/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "webhook" +publish = false +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +epoxy.workspace = true +futures-util.workspace = true +gas.workspace = true +namespace.workspace = true +reqwest.workspace = true +rivet-data.workspace = true +rivet-error.workspace = true +rivet-outbound-guard.workspace = true +rivet-pools.workspace = true +rivet-util.workspace = true +serde.workspace = true +serde_bare.workspace = true +serde_json.workspace = true +tracing.workspace = true +universaldb.workspace = true +url.workspace = true +uuid.workspace = true +vbare.workspace = true diff --git a/engine/packages/webhook/src/errors.rs b/engine/packages/webhook/src/errors.rs new file mode 100644 index 0000000000..45f0ce18ea --- /dev/null +++ b/engine/packages/webhook/src/errors.rs @@ -0,0 +1,51 @@ +use rivet_error::*; +use serde::{Deserialize, Serialize}; + +#[derive(RivetError, Debug, Deserialize, Serialize)] +#[error("webhook")] +pub enum Webhook { + #[error( + "invalid", + "Invalid webhook config.", + "Invalid webhook config: {reason}" + )] + Invalid { reason: String }, + #[error( + "conflict", + "Webhook config changed concurrently.", + "Webhook config was modified concurrently, please retry." + )] + Conflict, + #[error( + "delivery_failed", + "Webhook delivery failed.", + "Webhook delivery failed with status {status}." + )] + DeliveryFailed { status: u16 }, + #[error( + "destination_blocked", + "Webhook destination is not allowed.", + "Webhook destination is not allowed: {reason}" + )] + DestinationBlocked { reason: String }, + #[error("not_found", "Webhook not found.", "Webhook not found.")] + NotFound, + #[error( + "delivery_not_found", + "Webhook delivery not found.", + "Webhook delivery not found." + )] + DeliveryNotFound, + #[error( + "delivery_not_retryable", + "Webhook delivery is not in a retryable state.", + "Webhook delivery is not in a retryable state; only failed deliveries can be retried." + )] + DeliveryNotRetryable, + #[error( + "event_type_not_allowed", + "Event type cannot be subscribed to by a webhook.", + "Event type {event_type} cannot be subscribed to by a webhook because it is too high-throughput." + )] + EventTypeNotAllowed { event_type: String }, +} diff --git a/engine/packages/webhook/src/keys.rs b/engine/packages/webhook/src/keys.rs new file mode 100644 index 0000000000..c87b3fb55f --- /dev/null +++ b/engine/packages/webhook/src/keys.rs @@ -0,0 +1,241 @@ +use anyhow::Result; +use gas::prelude::*; +use universaldb::prelude::*; +use vbare::OwnedVersionedData; + +use crate::types::{DeliveryRecord, WebhookConfig}; + +fn serialize_config(value: WebhookConfig) -> Result> { + rivet_data::versioned::WebhookConfigData::wrap_latest(value.into()) + .serialize_with_embedded_version(rivet_data::WEBHOOK_CONFIG_VERSION) +} + +fn deserialize_config(raw: &[u8]) -> Result { + Ok(rivet_data::versioned::WebhookConfigData::deserialize_with_embedded_version(raw)?.into()) +} + +fn serialize_delivery(value: DeliveryRecord) -> Result> { + rivet_data::versioned::WebhookDeliveryData::wrap_latest(value.into()) + .serialize_with_embedded_version(rivet_data::WEBHOOK_DELIVERY_VERSION) +} + +fn deserialize_delivery(raw: &[u8]) -> Result { + Ok(rivet_data::versioned::WebhookDeliveryData::deserialize_with_embedded_version(raw)?.into()) +} + +// Durable, replicated copy proposed through epoxy. Slow to write and not meant to be read +// frequently; the local `DataKey` below is what backs listing/reads within a datacenter. +#[derive(Debug)] +pub struct GlobalDataKey { + pub namespace_id: Id, + pub name: String, +} + +impl GlobalDataKey { + pub fn new(namespace_id: Id, name: String) -> Self { + GlobalDataKey { namespace_id, name } + } +} + +impl FormalKey for GlobalDataKey { + type Value = WebhookConfig; + + fn deserialize(&self, raw: &[u8]) -> Result { + deserialize_config(raw) + } + + fn serialize(&self, value: Self::Value) -> Result> { + serialize_config(value) + } +} + +impl TuplePack for GlobalDataKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = (WEBHOOK, CONFIG, GLOBAL, DATA, self.namespace_id, &self.name); + t.pack(w, tuple_depth) + } +} + +impl<'de> TupleUnpack<'de> for GlobalDataKey { + fn unpack(input: &[u8], tuple_depth: TupleDepth) -> PackResult<(&[u8], Self)> { + let (input, (_, _, _, _, namespace_id, name)) = + <(usize, usize, usize, usize, Id, String)>::unpack(input, tuple_depth)?; + + let v = GlobalDataKey { namespace_id, name }; + + Ok((input, v)) + } +} + +// Local-only mirror of `GlobalDataKey`, written directly to this datacenter's UDB after every +// epoxy propose succeeds. Listing reads scan this instead of epoxy. +#[derive(Debug)] +pub struct DataKey { + pub namespace_id: Id, + pub name: String, +} + +impl DataKey { + pub fn new(namespace_id: Id, name: String) -> Self { + DataKey { namespace_id, name } + } + + pub fn subspace(namespace_id: Id) -> DataSubspaceKey { + DataSubspaceKey::new(namespace_id) + } +} + +impl FormalKey for DataKey { + type Value = WebhookConfig; + + fn deserialize(&self, raw: &[u8]) -> Result { + deserialize_config(raw) + } + + fn serialize(&self, value: Self::Value) -> Result> { + serialize_config(value) + } +} + +impl TuplePack for DataKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = (WEBHOOK, CONFIG, DATA, self.namespace_id, &self.name); + t.pack(w, tuple_depth) + } +} + +impl<'de> TupleUnpack<'de> for DataKey { + fn unpack(input: &[u8], tuple_depth: TupleDepth) -> PackResult<(&[u8], Self)> { + let (input, (_, _, _, namespace_id, name)) = + <(usize, usize, usize, Id, String)>::unpack(input, tuple_depth)?; + + let v = DataKey { namespace_id, name }; + + Ok((input, v)) + } +} + +// Subspace of all webhook `DataKey`s for a namespace, used to list webhook names. +#[derive(Debug)] +pub struct DataSubspaceKey { + pub namespace_id: Id, +} + +impl DataSubspaceKey { + pub fn new(namespace_id: Id) -> Self { + DataSubspaceKey { namespace_id } + } +} + +impl TuplePack for DataSubspaceKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = (WEBHOOK, CONFIG, DATA, self.namespace_id); + t.pack(w, tuple_depth) + } +} + +// Local-only record of a single delivery (one triggered event, identified by delivery id, and +// every attempt made to deliver it), written by the webhook workflow. Not replicated through +// epoxy: unlike config, a delivery only ever matters to the datacenter that ran it, since the +// workflow that owns a delivery lives in exactly one datacenter. +#[derive(Debug)] +pub struct DeliveryKey { + pub namespace_id: Id, + pub name: String, + pub delivery_id: String, +} + +impl DeliveryKey { + pub fn new(namespace_id: Id, name: String, delivery_id: String) -> Self { + DeliveryKey { + namespace_id, + name, + delivery_id, + } + } + + pub fn subspace(namespace_id: Id, name: String) -> DeliverySubspaceKey { + DeliverySubspaceKey::new(namespace_id, name) + } +} + +impl FormalKey for DeliveryKey { + type Value = DeliveryRecord; + + fn deserialize(&self, raw: &[u8]) -> Result { + deserialize_delivery(raw) + } + + fn serialize(&self, value: Self::Value) -> Result> { + serialize_delivery(value) + } +} + +impl TuplePack for DeliveryKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = ( + WEBHOOK, + DELIVERY, + DATA, + self.namespace_id, + &self.name, + &self.delivery_id, + ); + t.pack(w, tuple_depth) + } +} + +impl<'de> TupleUnpack<'de> for DeliveryKey { + fn unpack(input: &[u8], tuple_depth: TupleDepth) -> PackResult<(&[u8], Self)> { + let (input, (_, _, _, namespace_id, name, delivery_id)) = + <(usize, usize, usize, Id, String, String)>::unpack(input, tuple_depth)?; + + let v = DeliveryKey { + namespace_id, + name, + delivery_id, + }; + + Ok((input, v)) + } +} + +// Subspace of all `DeliveryKey`s for a single webhook, used to list its delivery history. +#[derive(Debug)] +pub struct DeliverySubspaceKey { + pub namespace_id: Id, + pub name: String, +} + +impl DeliverySubspaceKey { + pub fn new(namespace_id: Id, name: String) -> Self { + DeliverySubspaceKey { namespace_id, name } + } +} + +impl TuplePack for DeliverySubspaceKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = (WEBHOOK, DELIVERY, DATA, self.namespace_id, &self.name); + t.pack(w, tuple_depth) + } +} diff --git a/engine/packages/webhook/src/lib.rs b/engine/packages/webhook/src/lib.rs new file mode 100644 index 0000000000..c93634f6f5 --- /dev/null +++ b/engine/packages/webhook/src/lib.rs @@ -0,0 +1,16 @@ +use gas::prelude::*; + +pub mod errors; +pub mod keys; +pub mod ops; +pub mod types; +pub mod workflows; + +pub fn registry() -> WorkflowResult { + use workflows::*; + + let mut registry = Registry::new(); + registry.register_workflow::()?; + + Ok(registry) +} diff --git a/engine/packages/webhook/src/ops/delete.rs b/engine/packages/webhook/src/ops/delete.rs new file mode 100644 index 0000000000..2951ff3d2f --- /dev/null +++ b/engine/packages/webhook/src/ops/delete.rs @@ -0,0 +1,86 @@ +use epoxy::ops::propose::{ + CheckAndSetCommand, Command, CommandKind, ConsensusFailedReason, Proposal, ProposalResult, +}; +use gas::prelude::*; + +use crate::{errors, keys, workflows}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, +} + +// Proposes the epoxy clear (setting the value to `None` is how a key is deleted through epoxy), +// clears the local UDB mirror, then signals the webhook workflow to exit. `graceful_not_found` +// tolerates the workflow already being gone (e.g. a repeat delete), which also makes a repeat +// delete a no-op rather than an error. +// +// `expect_one_of` is always `vec![None]` because epoxy v2 does not implement value-conditional +// compare-and-swap; it accepts only that value. Concurrency is still detected, just at a +// different granularity: consensus decides one value per round, and a proposal that loses the +// round comes back as `ExpectedValueDoesNotMatch`, surfaced here as `Conflict`. +#[operation] +pub async fn webhook_config_delete(ctx: &OperationCtx, input: &Input) -> Result<()> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + + let global_key = keys::GlobalDataKey::new(namespace_id, name.clone()); + + let propose_res = ctx + .op(epoxy::ops::propose::Input { + proposal: Proposal { + commands: vec![Command { + kind: CommandKind::CheckAndSetCommand(CheckAndSetCommand { + key: namespace::keys::subspace().pack(&global_key), + expect_one_of: vec![None], + new_value: None, + }), + }], + }, + purge_cache: true, + mutable: true, + target_replicas: None, + }) + .await?; + + match propose_res { + ProposalResult::Committed => {} + ProposalResult::ConsensusFailed { reason } => match reason { + ConsensusFailedReason::ExpectedValueDoesNotMatch { .. } => { + // Another proposer's value won this round, so the delete did not take effect. + return Err(errors::Webhook::Conflict.build()); + } + ConsensusFailedReason::PreparePhaseConsensusFailed => { + bail!("epoxy propose failed: prepare phase consensus failed"); + } + ConsensusFailedReason::AcceptPhaseConsensusFailed => { + bail!("epoxy propose failed: accept phase consensus failed"); + } + ConsensusFailedReason::StaleBallot => { + bail!("epoxy propose failed: stale ballot"); + } + }, + } + + ctx.udb()? + .txn("webhook_config_delete", |tx| { + let name = name.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + tx.delete(&keys::DataKey::new(namespace_id, name)); + Ok(()) + } + }) + .await?; + + ctx.signal(workflows::webhook::Destroy {}) + .to_workflow::() + .tag("namespace_id", input.namespace_id) + .tag("name", input.name.clone()) + .graceful_not_found() + .send() + .await?; + + Ok(()) +} diff --git a/engine/packages/webhook/src/ops/get.rs b/engine/packages/webhook/src/ops/get.rs new file mode 100644 index 0000000000..f97a951622 --- /dev/null +++ b/engine/packages/webhook/src/ops/get.rs @@ -0,0 +1,33 @@ +use gas::prelude::*; +use universaldb::utils::IsolationLevel::*; + +use crate::{keys, types::WebhookConfig}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, +} + +// Point read of a single webhook config from the local UDB mirror written by `upsert`, for +// callers that only need to know whether one webhook exists. Use `list` when you need all of a +// namespace's webhooks. +#[operation] +pub async fn webhook_config_get( + ctx: &OperationCtx, + input: &Input, +) -> Result> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + + ctx.udb()? + .txn("webhook_config_get", move |tx| { + let name = name.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + tx.read_opt(&keys::DataKey::new(namespace_id, name), Serializable) + .await + } + }) + .await +} diff --git a/engine/packages/webhook/src/ops/list.rs b/engine/packages/webhook/src/ops/list.rs new file mode 100644 index 0000000000..027b79b40f --- /dev/null +++ b/engine/packages/webhook/src/ops/list.rs @@ -0,0 +1,57 @@ +use futures_util::{StreamExt, TryStreamExt}; +use gas::prelude::*; +use universaldb::options::StreamingMode; +use universaldb::utils::IsolationLevel::*; + +use crate::{keys, types::WebhookConfig}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, +} + +#[derive(Debug)] +pub struct Webhook { + pub name: String, + pub config: WebhookConfig, +} + +// Reads from the local UDB mirror written by `upsert`, not epoxy directly (see +// `webhook_config_upsert` for why). +#[operation] +pub async fn webhook_config_list(ctx: &OperationCtx, input: &Input) -> Result> { + let webhooks = ctx + .udb()? + .txn("webhook_config_list", |tx| async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + + let (start, end) = namespace::keys::subspace() + .subspace(&keys::DataKey::subspace(input.namespace_id)) + .range(); + + tx.get_ranges_keyvalues( + universaldb::RangeOption { + mode: StreamingMode::WantAll, + ..(start, end).into() + }, + Serializable, + ) + .map(|res| { + let tx = tx.clone(); + async move { + let entry = res?; + let (key, config) = tx.read_entry::(&entry)?; + Ok(Webhook { + name: key.name, + config, + }) + } + }) + .buffer_unordered(16) + .try_collect::>() + .await + }) + .await?; + + Ok(webhooks) +} diff --git a/engine/packages/webhook/src/ops/list_deliveries.rs b/engine/packages/webhook/src/ops/list_deliveries.rs new file mode 100644 index 0000000000..d20c2c7a11 --- /dev/null +++ b/engine/packages/webhook/src/ops/list_deliveries.rs @@ -0,0 +1,67 @@ +use futures_util::{StreamExt, TryStreamExt}; +use gas::prelude::*; +use universaldb::options::StreamingMode; +use universaldb::utils::IsolationLevel::*; + +use crate::{keys, types::DeliveryRecord}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, +} + +#[derive(Debug)] +pub struct Delivery { + pub delivery_id: String, + pub record: DeliveryRecord, +} + +// Reads every delivery recorded for one webhook from the local UDB mirror written by +// `record_delivery` (see `workflows::webhook`). Unordered; callers sort by `created_at` for +// chronological event history. A full scan of the webhook's delivery subspace, which is fine +// given deliveries are meant to stay low-throughput (see the event-type allowlist in the webhook +// spec) rather than a place to paginate over via range bounds. +#[operation] +pub async fn webhook_delivery_list(ctx: &OperationCtx, input: &Input) -> Result> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + + let deliveries = ctx + .udb()? + .txn("webhook_delivery_list", move |tx| { + let name = name.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + + let (start, end) = namespace::keys::subspace() + .subspace(&keys::DeliveryKey::subspace(namespace_id, name)) + .range(); + + tx.get_ranges_keyvalues( + universaldb::RangeOption { + mode: StreamingMode::WantAll, + ..(start, end).into() + }, + Serializable, + ) + .map(|res| { + let tx = tx.clone(); + async move { + let entry = res?; + let (key, record) = tx.read_entry::(&entry)?; + Ok(Delivery { + delivery_id: key.delivery_id, + record, + }) + } + }) + .buffer_unordered(16) + .try_collect::>() + .await + } + }) + .await?; + + Ok(deliveries) +} diff --git a/engine/packages/webhook/src/ops/mod.rs b/engine/packages/webhook/src/ops/mod.rs new file mode 100644 index 0000000000..42a0d49e49 --- /dev/null +++ b/engine/packages/webhook/src/ops/mod.rs @@ -0,0 +1,6 @@ +pub mod delete; +pub mod get; +pub mod list; +pub mod list_deliveries; +pub mod retry; +pub mod upsert; diff --git a/engine/packages/webhook/src/ops/retry.rs b/engine/packages/webhook/src/ops/retry.rs new file mode 100644 index 0000000000..0ba79a089c --- /dev/null +++ b/engine/packages/webhook/src/ops/retry.rs @@ -0,0 +1,62 @@ +use gas::prelude::*; +use universaldb::utils::IsolationLevel::*; + +use crate::{errors, keys, types::DeliveryStatus, workflows}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, + pub delivery_id: String, +} + +// Validates the delivery exists and is in a failed state, then signals the webhook workflow to +// retry it. The workflow re-checks the same state on its side before actually redelivering, since +// this read and the signal are not part of the same transaction. +#[operation] +pub async fn webhook_delivery_retry(ctx: &OperationCtx, input: &Input) -> Result<()> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + let delivery_id = input.delivery_id.clone(); + + let record = ctx + .udb()? + .txn("webhook_delivery_retry_read", { + let name = name.clone(); + let delivery_id = delivery_id.clone(); + move |tx| { + let name = name.clone(); + let delivery_id = delivery_id.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + tx.read_opt( + &keys::DeliveryKey::new(namespace_id, name, delivery_id), + Serializable, + ) + .await + } + } + }) + .await?; + + let record = record.ok_or_else(|| errors::Webhook::DeliveryNotFound.build())?; + + if !matches!(record.status, DeliveryStatus::Failed) { + return Err(errors::Webhook::DeliveryNotRetryable.build()); + } + + let signal_res = ctx + .signal(workflows::webhook::Retry { delivery_id }) + .to_workflow::() + .tag("namespace_id", namespace_id) + .tag("name", name) + .graceful_not_found() + .send() + .await?; + + if signal_res.is_none() { + return Err(errors::Webhook::NotFound.build()); + } + + Ok(()) +} diff --git a/engine/packages/webhook/src/ops/upsert.rs b/engine/packages/webhook/src/ops/upsert.rs new file mode 100644 index 0000000000..61edfc0fda --- /dev/null +++ b/engine/packages/webhook/src/ops/upsert.rs @@ -0,0 +1,60 @@ +use gas::prelude::*; + +use crate::{types::WebhookConfig, workflows}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, + pub config: WebhookConfig, +} + +// Signals the existing webhook workflow to update its config, or dispatches it for the first +// time if it doesn't exist yet, then waits for the workflow to report success or failure. The +// workflow itself does the validation and the epoxy/UDB write (see `workflows::webhook`), +// mirroring `namespace.rs`'s dispatch-then-wait pattern. +#[operation] +pub async fn webhook_config_upsert(ctx: &OperationCtx, input: &Input) -> Result<()> { + let topic = workflows::webhook::topic(input.namespace_id, &input.name); + + let mut complete_sub = ctx + .subscribe::(topic.clone()) + .await?; + let mut failed_sub = ctx + .subscribe::(topic.clone()) + .await?; + + let signal_res = ctx + .signal(workflows::webhook::Update { + config: input.config.clone(), + }) + .to_workflow::() + .tag("namespace_id", input.namespace_id) + .tag("name", input.name.clone()) + .graceful_not_found() + .send() + .await?; + + if signal_res.is_none() { + ctx.workflow(workflows::webhook::Input { + namespace_id: input.namespace_id, + name: input.name.clone(), + config: input.config.clone(), + }) + .tag("namespace_id", input.namespace_id) + .tag("name", input.name.clone()) + .unique() + .dispatch() + .await?; + } + + tokio::select! { + res = complete_sub.next() => { res?; } + res = failed_sub.next() => { + let msg = res?; + return Err(msg.into_body().error.build()); + } + } + + Ok(()) +} diff --git a/engine/packages/webhook/src/types.rs b/engine/packages/webhook/src/types.rs new file mode 100644 index 0000000000..c5f482ea3c --- /dev/null +++ b/engine/packages/webhook/src/types.rs @@ -0,0 +1,238 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +// Every event type the engine knows how to emit. Not all of them can be subscribed to by a +// webhook: high-throughput types are recorded for analytics but would turn a webhook into a +// per-request firehose, so `is_webhook_safe` gates which ones `validate` accepts. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum WebhookEventType { + #[serde(rename = "runner_pool.error")] + RunnerPoolError, + #[serde(rename = "runner_pool.healthy")] + RunnerPoolHealthy, + #[serde(rename = "actor.http_request")] + ActorHttpRequest, +} + +impl WebhookEventType { + // Whether a webhook is allowed to subscribe to this event type. High-throughput event types + // stay available to analytics ingestion but are rejected at webhook config upsert. + pub fn is_webhook_safe(&self) -> bool { + match self { + WebhookEventType::RunnerPoolError => true, + WebhookEventType::RunnerPoolHealthy => true, + // One event per actor HTTP request would hammer both the delivery pipeline and the + // receiving endpoint. + WebhookEventType::ActorHttpRequest => false, + } + } + + // The CloudEvents `type` attribute sent to the destination. + pub fn as_cloudevents_type(&self) -> &'static str { + match self { + WebhookEventType::RunnerPoolError => "dev.rivet.runner_pool.error", + WebhookEventType::RunnerPoolHealthy => "dev.rivet.runner_pool.healthy", + WebhookEventType::ActorHttpRequest => "dev.rivet.actor.http_request", + } + } + + // Stable name used in API payloads and error messages. Matches the serde renames above. + pub fn as_str(&self) -> &'static str { + match self { + WebhookEventType::RunnerPoolError => "runner_pool.error", + WebhookEventType::RunnerPoolHealthy => "runner_pool.healthy", + WebhookEventType::ActorHttpRequest => "actor.http_request", + } + } +} + +impl From for WebhookEventType { + fn from(value: rivet_data::generated::webhook_config_v1::WebhookEventType) -> Self { + match value { + rivet_data::generated::webhook_config_v1::WebhookEventType::RunnerPoolError => { + WebhookEventType::RunnerPoolError + } + rivet_data::generated::webhook_config_v1::WebhookEventType::RunnerPoolHealthy => { + WebhookEventType::RunnerPoolHealthy + } + rivet_data::generated::webhook_config_v1::WebhookEventType::ActorHttpRequest => { + WebhookEventType::ActorHttpRequest + } + } + } +} + +impl From for rivet_data::generated::webhook_config_v1::WebhookEventType { + fn from(value: WebhookEventType) -> Self { + match value { + WebhookEventType::RunnerPoolError => { + rivet_data::generated::webhook_config_v1::WebhookEventType::RunnerPoolError + } + WebhookEventType::RunnerPoolHealthy => { + rivet_data::generated::webhook_config_v1::WebhookEventType::RunnerPoolHealthy + } + WebhookEventType::ActorHttpRequest => { + rivet_data::generated::webhook_config_v1::WebhookEventType::ActorHttpRequest + } + } + } +} + +impl From for WebhookEventType { + fn from(value: rivet_data::generated::webhook_delivery_v1::WebhookEventType) -> Self { + match value { + rivet_data::generated::webhook_delivery_v1::WebhookEventType::RunnerPoolError => { + WebhookEventType::RunnerPoolError + } + rivet_data::generated::webhook_delivery_v1::WebhookEventType::RunnerPoolHealthy => { + WebhookEventType::RunnerPoolHealthy + } + rivet_data::generated::webhook_delivery_v1::WebhookEventType::ActorHttpRequest => { + WebhookEventType::ActorHttpRequest + } + } + } +} + +impl From for rivet_data::generated::webhook_delivery_v1::WebhookEventType { + fn from(value: WebhookEventType) -> Self { + match value { + WebhookEventType::RunnerPoolError => { + rivet_data::generated::webhook_delivery_v1::WebhookEventType::RunnerPoolError + } + WebhookEventType::RunnerPoolHealthy => { + rivet_data::generated::webhook_delivery_v1::WebhookEventType::RunnerPoolHealthy + } + WebhookEventType::ActorHttpRequest => { + rivet_data::generated::webhook_delivery_v1::WebhookEventType::ActorHttpRequest + } + } + } +} + +// Config for a single webhook, keyed by an arbitrary name within a namespace. `subscriptions` is +// which event types this webhook wants delivered; `validate` rejects any that are not +// webhook-safe (see `WebhookEventType::is_webhook_safe`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WebhookConfig { + pub url: String, + pub headers: HashMap, + pub subscriptions: Vec, +} + +impl From for WebhookConfig { + fn from(value: rivet_data::generated::webhook_config_v1::Data) -> Self { + WebhookConfig { + url: value.url, + headers: value.headers, + subscriptions: value.subscriptions.into_iter().map(Into::into).collect(), + } + } +} + +impl From for rivet_data::generated::webhook_config_v1::Data { + fn from(value: WebhookConfig) -> Self { + rivet_data::generated::webhook_config_v1::Data { + url: value.url, + headers: value.headers, + subscriptions: value.subscriptions.into_iter().map(Into::into).collect(), + } + } +} + +// Status of a single stored delivery, keyed by delivery id (see `keys::DeliveryKey`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DeliveryStatus { + Pending, + Succeeded, + Failed, +} + +impl From for DeliveryStatus { + fn from(value: rivet_data::generated::webhook_delivery_v1::DeliveryStatus) -> Self { + match value { + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Pending => { + DeliveryStatus::Pending + } + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Succeeded => { + DeliveryStatus::Succeeded + } + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Failed => { + DeliveryStatus::Failed + } + } + } +} + +impl From for rivet_data::generated::webhook_delivery_v1::DeliveryStatus { + fn from(value: DeliveryStatus) -> Self { + match value { + DeliveryStatus::Pending => { + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Pending + } + DeliveryStatus::Succeeded => { + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Succeeded + } + DeliveryStatus::Failed => { + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Failed + } + } + } +} + +// The standardized shape a producer sends to a webhook workflow, mapping onto the CloudEvents +// attributes of the same name. `data` stays polymorphic because each event type carries its own +// body and this package must not depend on the producers' types. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WebhookEvent { + pub event_type: WebhookEventType, + /// CloudEvents `subject`: which resource within the source the event is about, such as the + /// runner name for a runner pool event. + pub subject: Option, + /// CloudEvents `data`. + pub data: serde_json::Value, +} + +// Stored record for a single delivery (a triggered event, identified by delivery id, and every +// attempt made to deliver it). Not the CloudEvents payload itself, just enough to retry it and +// report its outcome. `created_at` is when the delivery was first triggered; a `Retry` reuses it +// rather than resetting it, so event history sorts by when the event actually happened. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeliveryRecord { + pub payload: String, + pub status: DeliveryStatus, + pub attempt_count: u32, + pub last_error: Option, + pub created_at: i64, + pub event_type: WebhookEventType, + pub subject: Option, +} + +impl From for DeliveryRecord { + fn from(value: rivet_data::generated::webhook_delivery_v1::Data) -> Self { + DeliveryRecord { + payload: value.payload, + status: value.status.into(), + attempt_count: value.attempt_count, + last_error: value.last_error, + created_at: value.created_at, + event_type: value.event_type.into(), + subject: value.subject, + } + } +} + +impl From for rivet_data::generated::webhook_delivery_v1::Data { + fn from(value: DeliveryRecord) -> Self { + rivet_data::generated::webhook_delivery_v1::Data { + payload: value.payload, + status: value.status.into(), + attempt_count: value.attempt_count, + last_error: value.last_error, + created_at: value.created_at, + event_type: value.event_type.into(), + subject: value.subject, + } + } +} diff --git a/engine/packages/webhook/src/workflows/mod.rs b/engine/packages/webhook/src/workflows/mod.rs new file mode 100644 index 0000000000..25f2d33efd --- /dev/null +++ b/engine/packages/webhook/src/workflows/mod.rs @@ -0,0 +1 @@ +pub mod webhook; diff --git a/engine/packages/webhook/src/workflows/webhook.rs b/engine/packages/webhook/src/workflows/webhook.rs new file mode 100644 index 0000000000..83f00f1275 --- /dev/null +++ b/engine/packages/webhook/src/workflows/webhook.rs @@ -0,0 +1,777 @@ +use std::time::Duration; + +use epoxy::ops::propose::ProposalResult; +use futures_util::FutureExt; +use gas::prelude::*; +use serde::{Deserialize, Serialize}; +use universaldb::prelude::FormalKey; +use universaldb::utils::IsolationLevel::*; +use uuid::Uuid; + +use crate::{ + errors, keys, + types::{DeliveryRecord, DeliveryStatus, WebhookConfig, WebhookEvent, WebhookEventType}, +}; + +/// Topic used to correlate `webhook::ops::upsert` (which waits for the outcome) with the +/// `UpsertComplete`/`Failed` messages this workflow sends. +pub fn topic(namespace_id: Id, name: &str) -> (&'static str, String) { + ("webhook", format!("{namespace_id}:{name}")) +} + +#[derive(Debug, Serialize)] +struct CloudEvent<'a> { + id: String, + source: &'a str, + specversion: &'static str, + #[serde(rename = "type")] + kind: &'static str, + time: String, + #[serde(skip_serializing_if = "Option::is_none")] + subject: Option<&'a str>, + datacontenttype: &'static str, + data: &'a serde_json::Value, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeliverInput { + pub delivery_id: String, + pub namespace_id: Id, + pub name: String, + pub config: WebhookConfig, + pub data: serde_json::Value, + pub event_type: WebhookEventType, + pub subject: Option, + /// When the delivery was first triggered, in epoch milliseconds. Used for the CloudEvents + /// `time` attribute, which is the time of the occurrence and so must stay fixed across + /// retries of the same delivery. + pub created_at: i64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeliverOutput { + /// `None` when the destination accepted the delivery. + pub error: Option, + /// Delay the destination asked for via `Retry-After`, in milliseconds. Only ever set + /// alongside a retryable failure. + pub retry_after_ms: Option, +} + +// The maximum number of delivery attempts for a single triggered event before giving up. +const MAX_DELIVERY_ATTEMPTS: u32 = 5; + +// Bounds on a destination-supplied `Retry-After`. The lower bound stops a `Retry-After: 0` from +// turning the retry loop into a hot loop; the upper bound matches the backoff cap so a +// misbehaving destination cannot park a delivery for an unbounded stretch. +const MIN_RETRY_AFTER: Duration = Duration::from_secs(1); +const MAX_RETRY_AFTER: Duration = Duration::from_secs(300); + +// Retries are for transient failures: 429 (rate limited) and 5xx (receiver-side error). Any +// other 4xx means the request itself is wrong and retrying with the same payload won't help. +fn is_retryable_status(status: u16) -> bool { + match status { + 429 => true, + 500..=599 => true, + _ => false, + } +} + +// Exponential backoff starting at 5s, doubling each attempt, capped at 5m. +fn delivery_backoff(attempt: u32) -> Duration { + Duration::from_secs(5u64.saturating_mul(1u64 << attempt.min(6)).min(300)) +} + +// How long to wait before the next attempt. A destination that told us how long to wait wins over +// our own backoff, since receivers such as Discord and Slack rate limit on `Retry-After` and +// ignoring it just earns more 429s. +fn next_delivery_delay(retry_after_ms: Option, attempt: u32) -> Duration { + match retry_after_ms { + Some(ms) => Duration::from_millis(ms).clamp(MIN_RETRY_AFTER, MAX_RETRY_AFTER), + None => delivery_backoff(attempt), + } +} + +// `Retry-After` is either delta-seconds or an HTTP-date (RFC 9110). An HTTP-date in the past +// yields a zero delay, which the caller clamps. +fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option { + let raw = headers.get(reqwest::header::RETRY_AFTER)?.to_str().ok()?; + + if let Ok(seconds) = raw.trim().parse::() { + return Some(seconds.saturating_mul(1_000)); + } + + let deadline = chrono::DateTime::parse_from_rfc2822(raw.trim()).ok()?; + let delta = deadline.timestamp_millis() - chrono::Utc::now().timestamp_millis(); + + Some(delta.max(0) as u64) +} + +#[activity(Deliver)] +pub async fn deliver(ctx: &ActivityCtx, input: &DeliverInput) -> Result { + let parsed_url = + url::Url::parse(&input.config.url).context("stored webhook url is not parseable")?; + + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { + return Ok(DeliverOutput { + error: Some(errors::Webhook::DestinationBlocked { + reason: reason.to_string(), + }), + retry_after_ms: None, + }); + } + + // The occurrence time, not the transmission time, so every attempt at one delivery carries + // the same value. Receivers dedupe on `id` plus `source` and would otherwise see the same + // event reported as having happened at several different times. + let occurred_at = chrono::DateTime::from_timestamp_millis(input.created_at) + .context("delivery created_at is not a valid timestamp")? + .to_rfc3339(); + + let event = CloudEvent { + id: input.delivery_id.clone(), + source: &format!("rivet:webhook:{}:{}", input.namespace_id, input.name), + specversion: "1.0", + kind: input.event_type.as_cloudevents_type(), + time: occurred_at, + subject: input.subject.as_deref(), + datacontenttype: "application/json", + data: &input.data, + }; + + let client = rivet_pools::reqwest::guarded_client(ctx.config()).await?; + let mut req = client + .post(parsed_url) + .header("Content-Type", "application/cloudevents+json") + .json(&event); + + for (k, v) in &input.config.headers { + req = req.header(k, v); + } + + match req.send().await { + Ok(res) if res.status().is_success() => Ok(DeliverOutput { + error: None, + retry_after_ms: None, + }), + Ok(res) => { + let status = res.status().as_u16(); + + Ok(DeliverOutput { + error: Some(errors::Webhook::DeliveryFailed { status }), + retry_after_ms: parse_retry_after(res.headers()), + }) + } + Err(err) => { + let err = anyhow::Error::from(err); + + // A hostname that only resolves to a disallowed address is rejected by the + // resolver at connect time, which the pre-flight check above cannot see. + if let Some(reason) = rivet_outbound_guard::block_reason(&err) { + return Ok(DeliverOutput { + error: Some(errors::Webhook::DestinationBlocked { + reason: reason.to_string(), + }), + retry_after_ms: None, + }); + } + + bail!("webhook delivery request failed: {err}"); + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecordDeliveryInput { + pub namespace_id: Id, + pub name: String, + pub delivery_id: String, + pub payload: String, + pub status: DeliveryStatus, + pub attempt_count: u32, + pub last_error: Option, + pub event_type: WebhookEventType, + pub subject: Option, +} + +// Writes the current state of a delivery to the local UDB mirror so it can be looked up later by +// `Retry` or listed for event history. Local only, not proposed through epoxy: a delivery only +// ever matters to the datacenter that ran it (see `keys::DeliveryKey`). +// +// Preserves `created_at` from any existing record for this delivery id instead of taking it from +// the caller, so a `Retry` (which re-enters this same activity) doesn't reset when the delivery +// was first triggered. Stamps a fresh `created_at` only the first time a delivery id is recorded. +// +// Returns the `created_at` the record now carries, which the caller needs for the CloudEvents +// `time` attribute. +#[activity(RecordDelivery)] +pub async fn record_delivery(ctx: &ActivityCtx, input: &RecordDeliveryInput) -> Result { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + let delivery_id = input.delivery_id.clone(); + let payload = input.payload.clone(); + let status = input.status; + let attempt_count = input.attempt_count; + let last_error = input.last_error.clone(); + let event_type = input.event_type; + let subject = input.subject.clone(); + let now = ctx.ts(); + + ctx.udb()? + .txn("webhook_record_delivery", move |tx| { + let name = name.clone(); + let delivery_id = delivery_id.clone(); + let payload = payload.clone(); + let last_error = last_error.clone(); + let subject = subject.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + let key = keys::DeliveryKey::new(namespace_id, name, delivery_id); + + let created_at = match tx.read_opt(&key, Serializable).await? { + Some(existing) => existing.created_at, + None => now, + }; + + tx.write( + &key, + DeliveryRecord { + payload, + status, + attempt_count, + last_error, + created_at, + event_type, + subject, + }, + )?; + Ok(created_at) + } + }) + .await +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GetDeliveryInput { + pub namespace_id: Id, + pub name: String, + pub delivery_id: String, +} + +#[activity(GetDelivery)] +pub async fn get_delivery( + ctx: &ActivityCtx, + input: &GetDeliveryInput, +) -> Result> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + let delivery_id = input.delivery_id.clone(); + + ctx.udb()? + .txn("webhook_get_delivery", move |tx| { + let name = name.clone(); + let delivery_id = delivery_id.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + tx.read_opt( + &keys::DeliveryKey::new(namespace_id, name, delivery_id), + Serializable, + ) + .await + } + }) + .await +} + +// One workflow instance per (namespace_id, name, dc) - the dc is implicit since a workflow +// always runs on the datacenter it was dispatched from (see webhook spec). +#[derive(Debug, Deserialize, Serialize)] +pub struct Input { + pub namespace_id: Id, + pub name: String, + pub config: WebhookConfig, +} + +#[workflow] +pub async fn webhook(ctx: &mut WorkflowCtx, input: &Input) -> Result<()> { + tracing::debug!( + namespace_id = %input.namespace_id, + name = %input.name, + "starting webhook workflow" + ); + + if !upsert( + ctx, + input.namespace_id, + input.name.clone(), + input.config.clone(), + ) + .await? + { + return Ok(()); + } + + let namespace_id = input.namespace_id; + let name = input.name.clone(); + + // Carries the current config as durable loop state so `Trigger` has something to deliver + // to; `Update` refreshes it only after `upsert` confirms the new config actually persisted. + ctx.loope(input.config.clone(), move |ctx, config| { + let name = name.clone(); + async move { + match ctx.listen::
().await? { + Main::Update(sig) => { + if upsert(ctx, namespace_id, name.clone(), sig.config.clone()).await? { + *config = sig.config; + } + } + Main::Trigger(sig) => { + // The workflow owns the config, so it is the authority on what this webhook is + // subscribed to. Producers filter by subscription before signaling, but the + // config can change between that read and this signal arriving. + if !config.subscriptions.contains(&sig.event.event_type) { + tracing::debug!( + event_type = sig.event.event_type.as_str(), + "dropping trigger for unsubscribed event type" + ); + return Ok(Loop::Continue); + } + + let delivery_id = Uuid::new_v4().to_string(); + + let outcome = deliver_with_retries( + ctx, + namespace_id, + name.clone(), + delivery_id, + config.clone(), + sig.event, + ) + .await?; + + if let DeliveryOutcome::Destroyed = outcome { + return Ok(Loop::Break(())); + } + } + Main::Retry(sig) => { + let record = ctx + .activity(GetDeliveryInput { + namespace_id, + name: name.clone(), + delivery_id: sig.delivery_id.clone(), + }) + .await?; + + let Some(record) = record else { + tracing::warn!( + delivery_id = %sig.delivery_id, + "retry requested for unknown delivery" + ); + return Ok(Loop::Continue); + }; + + if !matches!(record.status, DeliveryStatus::Failed) { + tracing::warn!( + delivery_id = %sig.delivery_id, + status = ?record.status, + "retry requested for delivery not in a failed state" + ); + return Ok(Loop::Continue); + } + + let data = match serde_json::from_str(&record.payload) { + Ok(data) => data, + Err(err) => { + tracing::warn!( + delivery_id = %sig.delivery_id, + ?err, + "stored delivery payload is not valid json" + ); + return Ok(Loop::Continue); + } + }; + + let outcome = deliver_with_retries( + ctx, + namespace_id, + name.clone(), + sig.delivery_id, + config.clone(), + WebhookEvent { + event_type: record.event_type, + subject: record.subject, + data, + }, + ) + .await?; + + if let DeliveryOutcome::Destroyed = outcome { + return Ok(Loop::Break(())); + } + } + Main::Destroy(_) => { + return Ok(Loop::Break(())); + } + } + + Ok(Loop::<()>::Continue) + } + .boxed() + }) + .await?; + + Ok(()) +} + +async fn upsert( + ctx: &mut WorkflowCtx, + namespace_id: Id, + name: String, + config: WebhookConfig, +) -> Result { + let validate_res = ctx + .activity(ValidateInput { + config: config.clone(), + }) + .await?; + + if let Err(error) = validate_res { + ctx.msg(Failed { error }) + .topic(topic(namespace_id, &name)) + .send() + .await?; + + // TODO(RVT-3928): return Ok(Err) (is what is written in the equiv namespace file) + return Ok(false); + } + + let upsert_res = ctx + .activity(UpsertConfigInput { + namespace_id, + name: name.clone(), + config, + }) + .await?; + + if let Err(error) = upsert_res { + ctx.msg(Failed { error }) + .topic(topic(namespace_id, &name)) + .send() + .await?; + + // TODO(RVT-3928): return Ok(Err) (is what is written in the equiv namespace file) + return Ok(false); + } + + ctx.msg(UpsertComplete {}) + .topic(topic(namespace_id, &name)) + .send() + .await?; + + Ok(true) +} + +enum DeliveryOutcome { + // The delivery reached a terminal state, either delivered or permanently failed after + // exhausting retries. Either way there is nothing left to wait on. + Done, + // A `Destroy` signal interrupted an in-progress backoff wait. + Destroyed, +} + +// Runs (or re-runs, for a manual `Retry`) the full attempt loop for one delivery: records it as +// `Pending`, attempts delivery with exponential backoff up to `MAX_DELIVERY_ATTEMPTS`, and records +// the terminal `Succeeded`/`Failed` outcome. Shared by `Trigger` (a brand new delivery) and +// `Retry` (re-attempting a stored, already-failed delivery), since both just need to run this +// same loop against a `delivery_id` and `payload`. +async fn deliver_with_retries( + ctx: &mut WorkflowCtx, + namespace_id: Id, + name: String, + delivery_id: String, + config: WebhookConfig, + event: WebhookEvent, +) -> Result { + let WebhookEvent { + event_type, + subject, + data, + } = event; + + // Stored as JSON text, matching the `payload: str` field in the delivery schema. + let payload = serde_json::to_string(&data).context("event data is not serializable")?; + // Also yields the delivery's `created_at`, which the CloudEvents envelope needs. On a manual + // `Retry` this is the original trigger time, preserved by the activity. + let created_at = ctx + .activity(RecordDeliveryInput { + namespace_id, + name: name.clone(), + delivery_id: delivery_id.clone(), + payload: payload.clone(), + status: DeliveryStatus::Pending, + attempt_count: 0, + last_error: None, + event_type, + subject: subject.clone(), + }) + .await?; + + let mut attempt = 0; + + loop { + let deliver_res = ctx + .activity(DeliverInput { + delivery_id: delivery_id.clone(), + namespace_id, + name: name.clone(), + config: config.clone(), + data: data.clone(), + event_type, + subject: subject.clone(), + created_at, + }) + .await?; + + match deliver_res.error { + None => { + ctx.activity(RecordDeliveryInput { + namespace_id, + name: name.clone(), + delivery_id: delivery_id.clone(), + payload, + status: DeliveryStatus::Succeeded, + attempt_count: attempt + 1, + last_error: None, + event_type, + subject, + }) + .await?; + + return Ok(DeliveryOutcome::Done); + } + Some(errors::Webhook::DeliveryFailed { status }) + if is_retryable_status(status) && attempt + 1 < MAX_DELIVERY_ATTEMPTS => + { + attempt += 1; + + // Race the wait against `Destroy` so a delete mid-retry stops delivery + // immediately instead of waiting out the full retry sequence before the workflow + // notices. + let destroy_sig = ctx + .listen_with_timeout::(next_delivery_delay( + deliver_res.retry_after_ms, + attempt, + )) + .await?; + + if destroy_sig.is_some() { + return Ok(DeliveryOutcome::Destroyed); + } + } + Some(error) => { + tracing::warn!(?error, attempt, "webhook delivery failed permanently"); + + ctx.activity(RecordDeliveryInput { + namespace_id, + name: name.clone(), + delivery_id: delivery_id.clone(), + payload, + status: DeliveryStatus::Failed, + attempt_count: attempt + 1, + last_error: Some(error.build().to_string()), + event_type, + subject, + }) + .await?; + + return Ok(DeliveryOutcome::Done); + } + } + } +} + +#[message("webhook_upsert_complete")] +pub struct UpsertComplete {} + +#[message("webhook_failed")] +pub struct Failed { + pub error: errors::Webhook, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ValidateInput { + pub config: WebhookConfig, +} + +#[activity(Validate)] +pub async fn validate( + ctx: &ActivityCtx, + input: &ValidateInput, +) -> Result> { + let parsed_url = match url::Url::parse(&input.config.url) { + Ok(parsed_url) => parsed_url, + Err(err) => { + return Ok(Err(errors::Webhook::Invalid { + reason: format!("invalid url: {err}"), + })); + } + }; + + // Reject destinations the engine is not allowed to reach before the config is stored. + // Delivery re-checks this at request time, which also catches configs written before this + // gate existed and hosts whose DNS answer changes afterwards. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { + return Ok(Err(errors::Webhook::Invalid { + reason: format!("invalid url: {reason}"), + })); + } + + // Enforce the webhook-safe event type allowlist. High-throughput types are still ingested for + // analytics; they just cannot be a webhook trigger (see the webhook spec). + for event_type in &input.config.subscriptions { + if !event_type.is_webhook_safe() { + return Ok(Err(errors::Webhook::EventTypeNotAllowed { + event_type: event_type.as_str().to_string(), + })); + } + } + + if input.config.headers.len() > 16 { + return Ok(Err(errors::Webhook::Invalid { + reason: "too many headers (max 16)".to_string(), + })); + } + + for (name, value) in &input.config.headers { + if name.len() > 128 { + return Ok(Err(errors::Webhook::Invalid { + reason: "invalid header name: too long (max 128)".to_string(), + })); + } + if let Err(err) = name.parse::() { + return Ok(Err(errors::Webhook::Invalid { + reason: format!("invalid header name: {err}"), + })); + } + if value.len() > 4096 { + return Ok(Err(errors::Webhook::Invalid { + reason: "invalid header value: too long (max 4096)".to_string(), + })); + } + if let Err(err) = value.parse::() { + return Ok(Err(errors::Webhook::Invalid { + reason: format!("invalid header value: {err}"), + })); + } + } + + Ok(Ok(())) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UpsertConfigInput { + pub namespace_id: Id, + pub name: String, + pub config: WebhookConfig, +} + +// Writes the webhook config to epoxy (the durable, replicated copy) and mirrors it into local +// UDB (what `list` reads, since epoxy is slow and not meant for frequent/scan-style reads). +// +// `expect_one_of` is always `vec![None]` because epoxy v2 does not implement value-conditional +// compare-and-swap; it accepts only that value. Concurrency is still detected, just at a +// different granularity: consensus decides one value per round, and a proposal that loses the +// round comes back as `ExpectedValueDoesNotMatch`, surfaced here as `Conflict`. +#[activity(UpsertConfig)] +pub async fn upsert_config( + ctx: &ActivityCtx, + input: &UpsertConfigInput, +) -> Result> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + + let global_key = keys::GlobalDataKey::new(namespace_id, name.clone()); + + let propose_res = ctx + .op(epoxy::ops::propose::Input { + proposal: epoxy::ops::propose::Proposal { + commands: vec![epoxy::ops::propose::Command { + kind: epoxy::ops::propose::CommandKind::CheckAndSetCommand( + epoxy::ops::propose::CheckAndSetCommand { + key: namespace::keys::subspace().pack(&global_key), + expect_one_of: vec![None], + new_value: Some(global_key.serialize(input.config.clone())?), + }, + ), + }], + }, + purge_cache: true, + mutable: true, + target_replicas: None, + }) + .await?; + + match propose_res { + ProposalResult::Committed => {} + ProposalResult::ConsensusFailed { reason } => match reason { + epoxy::ops::propose::ConsensusFailedReason::ExpectedValueDoesNotMatch { .. } => { + // Another proposer's value won this round, so this config was not the one + // committed. The caller should re-read and retry. + return Ok(Err(errors::Webhook::Conflict)); + } + epoxy::ops::propose::ConsensusFailedReason::PreparePhaseConsensusFailed => { + bail!("epoxy propose failed: prepare phase consensus failed"); + } + epoxy::ops::propose::ConsensusFailedReason::AcceptPhaseConsensusFailed => { + bail!("epoxy propose failed: accept phase consensus failed"); + } + epoxy::ops::propose::ConsensusFailedReason::StaleBallot => { + bail!("epoxy propose failed: stale ballot"); + } + }, + } + + // We still have to write locally for listing. + // TODO: non-transactional. Epoxy propose and the local UDB write can diverge if we crash or + // error between them. + let config = input.config.clone(); + ctx.udb()? + .txn("webhook_upsert_config", |tx| { + let name = name.clone(); + let config = config.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + tx.write(&keys::DataKey::new(namespace_id, name), config)?; + Ok(()) + } + }) + .await?; + + Ok(Ok(())) +} + +#[signal("webhook_trigger")] +pub struct Trigger { + pub event: WebhookEvent, +} + +#[signal("webhook_update")] +pub struct Update { + pub config: WebhookConfig, +} + +#[signal("webhook_destroy")] +pub struct Destroy {} + +#[signal("webhook_retry")] +pub struct Retry { + pub delivery_id: String, +} + +join_signal!(Main { + Trigger, + Update, + Destroy, + Retry, +}); diff --git a/engine/packages/workflow-worker/Cargo.toml b/engine/packages/workflow-worker/Cargo.toml index 4fd24b2c8b..047fc4a9c6 100644 --- a/engine/packages/workflow-worker/Cargo.toml +++ b/engine/packages/workflow-worker/Cargo.toml @@ -17,3 +17,4 @@ namespace.workspace = true pegboard.workspace = true rivet-config.workspace = true tracing.workspace = true +webhook.workspace = true diff --git a/engine/packages/workflow-worker/src/lib.rs b/engine/packages/workflow-worker/src/lib.rs index 53351a2bf5..7f120ca2d7 100644 --- a/engine/packages/workflow-worker/src/lib.rs +++ b/engine/packages/workflow-worker/src/lib.rs @@ -17,6 +17,7 @@ pub fn registry() -> Result { .merge(epoxy::registry()?)? .merge(gasoline_runtime::registry()?)? .merge(datacenter::registry()?)? - .merge(depot::registry()?) + .merge(depot::registry()?)? + .merge(webhook::registry()?) .map_err(Into::into) } diff --git a/engine/sdks/rust/data/src/lib.rs b/engine/sdks/rust/data/src/lib.rs index f17cf72bbb..8754cfe157 100644 --- a/engine/sdks/rust/data/src/lib.rs +++ b/engine/sdks/rust/data/src/lib.rs @@ -6,4 +6,5 @@ pub use generated::{ PEGBOARD_NAMESPACE_ACTOR_BY_KEY_VERSION, PEGBOARD_NAMESPACE_ACTOR_NAME_VERSION, PEGBOARD_NAMESPACE_RUNNER_ALLOC_IDX_VERSION, PEGBOARD_NAMESPACE_RUNNER_BY_KEY_VERSION, PEGBOARD_NAMESPACE_RUNNER_CONFIG_VERSION, PEGBOARD_RUNNER_METADATA_VERSION, + WEBHOOK_CONFIG_VERSION, WEBHOOK_DELIVERY_VERSION, }; diff --git a/engine/sdks/rust/data/src/versioned/mod.rs b/engine/sdks/rust/data/src/versioned/mod.rs index 245fae1313..6e46c0fb10 100644 --- a/engine/sdks/rust/data/src/versioned/mod.rs +++ b/engine/sdks/rust/data/src/versioned/mod.rs @@ -125,6 +125,74 @@ impl RunnerAllocIdxKeyData { } } +pub enum WebhookConfigData { + V1(webhook_config_v1::Data), +} + +impl OwnedVersionedData for WebhookConfigData { + type Latest = webhook_config_v1::Data; + + fn wrap_latest(latest: webhook_config_v1::Data) -> Self { + WebhookConfigData::V1(latest) + } + + fn unwrap_latest(self) -> Result { + #[allow(irrefutable_let_patterns)] + if let WebhookConfigData::V1(data) = self { + Ok(data) + } else { + bail!("version not latest"); + } + } + + fn deserialize_version(payload: &[u8], version: u16) -> Result { + match version { + 1 => Ok(WebhookConfigData::V1(serde_bare::from_slice(payload)?)), + _ => bail!("invalid version: {version}"), + } + } + + fn serialize_version(self, _version: u16) -> Result> { + match self { + WebhookConfigData::V1(data) => serde_bare::to_vec(&data).map_err(Into::into), + } + } +} + +pub enum WebhookDeliveryData { + V1(webhook_delivery_v1::Data), +} + +impl OwnedVersionedData for WebhookDeliveryData { + type Latest = webhook_delivery_v1::Data; + + fn wrap_latest(latest: webhook_delivery_v1::Data) -> Self { + WebhookDeliveryData::V1(latest) + } + + fn unwrap_latest(self) -> Result { + #[allow(irrefutable_let_patterns)] + if let WebhookDeliveryData::V1(data) = self { + Ok(data) + } else { + bail!("version not latest"); + } + } + + fn deserialize_version(payload: &[u8], version: u16) -> Result { + match version { + 1 => Ok(WebhookDeliveryData::V1(serde_bare::from_slice(payload)?)), + _ => bail!("invalid version: {version}"), + } + } + + fn serialize_version(self, _version: u16) -> Result> { + match self { + WebhookDeliveryData::V1(data) => serde_bare::to_vec(&data).map_err(Into::into), + } + } +} + pub enum MetadataKeyData { V1(pegboard_runner_metadata_v1::Data), } diff --git a/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs b/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs index 26097a4bd2..6c4a40ad23 100644 --- a/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs +++ b/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs @@ -30,48 +30,24 @@ impl OwnedVersionedData for NamespaceRunnerConfig { fn deserialize_version(payload: &[u8], version: u16) -> Result { match version { - 1 => Ok(NamespaceRunnerConfig::V1( - serde_bare::from_slice(payload)?, - )), - 2 => Ok(NamespaceRunnerConfig::V2( - serde_bare::from_slice(payload)?, - )), - 3 => Ok(NamespaceRunnerConfig::V3( - serde_bare::from_slice(payload)?, - )), - 4 => Ok(NamespaceRunnerConfig::V4( - serde_bare::from_slice(payload)?, - )), - 5 => Ok(NamespaceRunnerConfig::V5( - serde_bare::from_slice(payload)?, - )), - 6 => Ok(NamespaceRunnerConfig::V6( - serde_bare::from_slice(payload)?, - )), + 1 => Ok(NamespaceRunnerConfig::V1(serde_bare::from_slice(payload)?)), + 2 => Ok(NamespaceRunnerConfig::V2(serde_bare::from_slice(payload)?)), + 3 => Ok(NamespaceRunnerConfig::V3(serde_bare::from_slice(payload)?)), + 4 => Ok(NamespaceRunnerConfig::V4(serde_bare::from_slice(payload)?)), + 5 => Ok(NamespaceRunnerConfig::V5(serde_bare::from_slice(payload)?)), + 6 => Ok(NamespaceRunnerConfig::V6(serde_bare::from_slice(payload)?)), _ => bail!("invalid version: {version}"), } } fn serialize_version(self, _version: u16) -> Result> { match self { - NamespaceRunnerConfig::V1(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V2(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V3(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V4(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V5(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V6(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } + NamespaceRunnerConfig::V1(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V2(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V3(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V4(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V5(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V6(data) => serde_bare::to_vec(&data).map_err(Into::into), } } diff --git a/engine/sdks/schemas/data/webhook.config.v1.bare b/engine/sdks/schemas/data/webhook.config.v1.bare new file mode 100644 index 0000000000..2cb13802e0 --- /dev/null +++ b/engine/sdks/schemas/data/webhook.config.v1.bare @@ -0,0 +1,11 @@ +type WebhookEventType enum { + RUNNER_POOL_ERROR + RUNNER_POOL_HEALTHY + ACTOR_HTTP_REQUEST +} + +type Data struct { + url: str + headers: map + subscriptions: list +} diff --git a/engine/sdks/schemas/data/webhook.delivery.v1.bare b/engine/sdks/schemas/data/webhook.delivery.v1.bare new file mode 100644 index 0000000000..56cd50616d --- /dev/null +++ b/engine/sdks/schemas/data/webhook.delivery.v1.bare @@ -0,0 +1,23 @@ +type DeliveryStatus enum { + PENDING + SUCCEEDED + FAILED +} + +# Mirrors the enum in `webhook.config.v1.bare`. BARE schemas cannot reference types across +# files, so each generated module gets its own copy; both convert to `types::WebhookEventType`. +type WebhookEventType enum { + RUNNER_POOL_ERROR + RUNNER_POOL_HEALTHY + ACTOR_HTTP_REQUEST +} + +type Data struct { + payload: str + status: DeliveryStatus + attemptCount: u32 + lastError: optional + createdAt: i64 + eventType: WebhookEventType + subject: optional +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..ffc9e7505e --- /dev/null +++ b/flake.lock @@ -0,0 +1,117 @@ +{ + "nodes": { + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1787555220, + "narHash": "sha256-ZNw7FDQm2PVOIyE9WaLFgK1QYw8vrOchxMYHzDDxlUY=", + "owner": "nix-community", + "repo": "fenix", + "rev": "2013c981829bd5e93db06749b5639450c81bece3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1787498568, + "narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1787498568, + "narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "fenix": "fenix", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgs-unstable": "nixpkgs-unstable" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1787472531, + "narHash": "sha256-cq+wrv+Zadl1NiJIWxYKHpcXfUQXDKBYa/7eq0JWFZk=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "5c156cdfb047ea171ba255dcdcd86236d98a389e", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..7099e26281 --- /dev/null +++ b/flake.nix @@ -0,0 +1,48 @@ +{ + description = "Rust development environment"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nixpkgs-unstable.url = "github:nixos/nixpkgs/nixos-unstable"; + }; + outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, fenix }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + toolchain = fenix.packages.${system}.combine [ + (fenix.packages.${system}.fromToolchainFile { + file = ./rust-toolchain.toml; + sha256 = "sha256-P30Tm3O7vQAE725YtDCDHGjNrSsfZO4us11UwJGZSJo="; + }) + fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.rust-std + fenix.packages.${system}.stable.rust-src + ]; + in + { + devShells.default = pkgs.mkShell rec { + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = with pkgs; [ + toolchain + clang + llvmPackages.bintools + stdenv.cc.cc.lib + nodejs_22 + pnpm + ]; + LIBCLANG_PATH = pkgs.lib.makeLibraryPath [ pkgs.llvmPackages_latest.libclang.lib ]; + LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath (buildInputs ++ nativeBuildInputs); + BINDGEN_EXTRA_CLANG_ARGS = + (builtins.map (a: ''-I"${a}/include"'') [ pkgs.glibc.dev ]) + ++ [ + ''-I"${pkgs.llvmPackages_latest.libclang.lib}/lib/clang/${pkgs.llvmPackages_latest.libclang.version}/include"'' + ''-I"${pkgs.glib.dev}/include/glib-2.0"'' + ''-I${pkgs.glib.out}/lib/glib-2.0/include/'' + ]; + }; + } + ); +} \ No newline at end of file diff --git a/frontend/src/app/runner-pool-error-popover.tsx b/frontend/src/app/runner-pool-error-popover.tsx index 268a9f118f..18a1f1b371 100644 --- a/frontend/src/app/runner-pool-error-popover.tsx +++ b/frontend/src/app/runner-pool-error-popover.tsx @@ -43,6 +43,7 @@ interface ClassifiedError { kind: | "serverless_http" | "serverless_connection" + | "serverless_destination_blocked" | "serverless_invalid_sse" | "serverless_stream_ended_early" | "downgrade" @@ -116,6 +117,18 @@ function classifyRunnerError(error: RivetActorError): ClassifiedError { fingerprint: `conn:${e.serverless_connection_error.message.slice(0, 64)}`, }), ) + .with( + P.shape({ + serverless_destination_blocked: P.shape({ reason: P.string }), + }), + (e) => ({ + severity: "error", + kind: "serverless_destination_blocked", + title: "Serverless URL is not an allowed destination", + body: e.serverless_destination_blocked.reason, + fingerprint: `blocked:${e.serverless_destination_blocked.reason.slice(0, 64)}`, + }), + ) .with( P.shape({ serverless_invalid_sse_payload: P.shape({ message: P.string }), @@ -502,6 +515,8 @@ function describeKind(kind: ClassifiedError["kind"]): string { return "Runner pool was downgraded to an unsupported version. Revert to a higher version."; case "serverless_stream_ended_early": return "Connection terminated before the runner stopped. Check the request lifespan limits on your serverless provider."; + case "serverless_destination_blocked": + return "The configured serverless URL points at a destination Rivet is not allowed to reach. Use a publicly routable URL, or allow the address range in the engine's outbound configuration."; case "internal": return "An internal error occurred in the runner pool."; default: diff --git a/frontend/src/components/actors/actor-status-label.tsx b/frontend/src/components/actors/actor-status-label.tsx index 7ce47442c2..5d08591928 100644 --- a/frontend/src/components/actors/actor-status-label.tsx +++ b/frontend/src/components/actors/actor-status-label.tsx @@ -103,6 +103,7 @@ export function ActorError({ error }: { error: object | string }) { .or(P.shape({ serverless_http_error: P.any })) .or(P.string) .or(P.shape({ serverless_connection_error: P.any })) + .or(P.shape({ serverless_destination_blocked: P.any })) .or(P.shape({ serverless_invalid_sse_payload: P.any })), }), (err) => , @@ -221,6 +222,23 @@ export function RunnerPoolError({ error }: { error: RivetActorError }) { ); }, ) + .with( + P.shape({ + serverless_destination_blocked: P.shape({ reason: P.string }), + }), + (errObj) => { + const reason = errObj.serverless_destination_blocked?.reason; + return ( + <> +

+ Serverless endpoint URL is not an allowed + destination +

+ {reason ? : null} + + ); + }, + ) .with( P.shape({ serverless_invalid_sse_payload: P.shape({ message: P.string }), diff --git a/frontend/src/queries/types.ts b/frontend/src/queries/types.ts index fdffce099f..f69860e2b3 100644 --- a/frontend/src/queries/types.ts +++ b/frontend/src/queries/types.ts @@ -35,4 +35,5 @@ export type RivetActorError = | { runner_id: string } | { serverless_http_error: unknown } | { serverless_connection_error: unknown } + | { serverless_destination_blocked: unknown } | { serverless_invalid_sse_payload: unknown }; diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000000..292fe499e3 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable" diff --git a/scripts/tests/webhooks_api.sh b/scripts/tests/webhooks_api.sh new file mode 100755 index 0000000000..998da57653 --- /dev/null +++ b/scripts/tests/webhooks_api.sh @@ -0,0 +1,226 @@ +#!/usr/bin/env bash +# API test suite for the webhook endpoints. +# +# Covers everything in the webhook spec that is reachable over HTTP: config CRUD, the event-type +# allowlist, destination/header validation, event history, and the retry endpoint's error paths. +# +# Does NOT cover actual webhook delivery. Nothing in the API can fire a `Trigger` - the only +# producer is the runner pool error tracker - so delivery, retry-of-a-real-delivery, and event +# recording need a runner pool error to exercise. See the webhook spec. +# +# Usage: +# scripts/run/engine-rocksdb.sh # in another shell +# scripts/tests/webhooks_api.sh +# +# Env: +# RIVET_ENDPOINT default http://localhost:6420 +# RIVET_TOKEN bearer token; omit when the engine runs without auth configured +# NAMESPACE default webhook-test + +set -uo pipefail + +R="${RIVET_ENDPOINT:-http://localhost:6420}" +NS="${NAMESPACE:-webhook-test}" +AUTH=() +if [ -n "${RIVET_TOKEN:-}" ]; then + AUTH=(-H "Authorization: Bearer ${RIVET_TOKEN}") +fi + +pass=0 +fail=0 +RESP="" +STATUS="" + +# Extracts "group.code" from an error body, or "ok" for a success body. +errcode() { + python3 - "$1" <<'PY' +import sys, json +try: + d = json.loads(sys.argv[1]) +except Exception: + print(""); sys.exit() +if isinstance(d, dict) and "group" in d and "code" in d: + print(str(d["group"]) + "." + str(d["code"])) +else: + print("ok") +PY +} + +# Reads a dotted path out of a JSON body, printing when absent. +jpath() { + python3 - "$1" "$2" <<'PY' +import sys, json +try: + cur = json.loads(sys.argv[1]) +except Exception: + print(""); sys.exit() +for part in sys.argv[2].split("."): + if isinstance(cur, dict) and part in cur: + cur = cur[part] + elif isinstance(cur, list) and part.isdigit() and int(part) < len(cur): + cur = cur[int(part)] + else: + print(""); sys.exit() +print("null" if cur is None else (json.dumps(cur, sort_keys=True, separators=(",", ":")) if isinstance(cur, (dict, list)) else cur)) +PY +} + +req() { # req METHOD PATH [BODY] + local method="$1" path="$2" body="${3:-}" + local out + if [ -n "$body" ]; then + out=$(curl -s -w '\n%{http_code}' -X "$method" "$R$path" ${AUTH[@]+"${AUTH[@]}"} \ + -H 'Content-Type: application/json' -d "$body") + else + out=$(curl -s -w '\n%{http_code}' -X "$method" "$R$path" ${AUTH[@]+"${AUTH[@]}"}) + fi + STATUS="${out##*$'\n'}" + RESP="${out%$'\n'*}" +} + +ok() { pass=$((pass + 1)); printf ' \033[32mPASS\033[0m %s\n' "$1"; } +no() { + fail=$((fail + 1)) + printf ' \033[31mFAIL\033[0m %s\n expected: %s\n got: %s\n' "$1" "$2" "$3" +} + +expect_err() { # expect_err NAME EXPECTED + local got; got=$(errcode "$RESP") + [ "$got" = "$2" ] && ok "$1" || no "$1" "$2" "$got (body: $RESP)" +} + +expect_field() { # expect_field NAME PATH EXPECTED + local got; got=$(jpath "$RESP" "$2") + [ "$got" = "$3" ] && ok "$1" || no "$1" "$3" "$got" +} + +section() { printf '\n\033[1m%s\033[0m\n' "$1"; } + +# ---------------------------------------------------------------- setup + +if ! curl -sf -o /dev/null --max-time 3 "$R/health"; then + echo "engine not reachable at $R - start it with scripts/run/engine-rocksdb.sh" >&2 + exit 1 +fi + +req POST /namespaces "{\"name\":\"$NS\",\"display_name\":\"Webhook Test\"}" +case "$(errcode "$RESP")" in + ok|namespace.name_not_unique) ;; + *) echo "could not create or reuse namespace $NS: $RESP" >&2; exit 1 ;; +esac + +# Start from a known state. +req DELETE "/webhooks/wh-main?namespace=$NS" >/dev/null 2>&1 + +# ---------------------------------------------------------------- tests + +section "config CRUD" +req PUT "/webhooks/wh-main?namespace=$NS" \ + '{"url":"http://127.0.0.1:8099/v1","headers":{"X-Token":"abc"},"subscriptions":["runner_pool.error"]}' +expect_err "create webhook" ok + +req GET "/webhooks?namespace=$NS" +expect_field "create is listed" "webhooks.wh-main.url" "http://127.0.0.1:8099/v1" +expect_field "create stored headers" "webhooks.wh-main.headers.X-Token" "abc" +expect_field "create stored subs" "webhooks.wh-main.subscriptions" '["runner_pool.error"]' + +# Regression: update used to fail because epoxy v2 rejects a value-based CheckAndSet expectation. +req PUT "/webhooks/wh-main?namespace=$NS" \ + '{"url":"http://127.0.0.1:8099/v2","subscriptions":["runner_pool.error","runner_pool.healthy"]}' +expect_err "update existing webhook" ok + +req GET "/webhooks?namespace=$NS" +expect_field "update changed url" "webhooks.wh-main.url" "http://127.0.0.1:8099/v2" +expect_field "update changed subs" "webhooks.wh-main.subscriptions" '["runner_pool.error","runner_pool.healthy"]' +expect_field "update cleared headers" "webhooks.wh-main.headers" '{}' + +section "event history" +req GET "/webhooks/wh-main/events?namespace=$NS" +expect_err "events on real webhook" ok +expect_field "events empty history" "events" "[]" +expect_field "events null cursor" "pagination.cursor" "null" + +req GET "/webhooks/wh-main/events?namespace=$NS&limit=2" +expect_err "events honours limit param" ok + +req GET "/webhooks/nope/events?namespace=$NS" +expect_err "events on missing webhook" webhook.not_found + +req GET "/webhooks/wh-main/events?namespace=$NS&cursor=garbage" +expect_err "events rejects bad cursor" api.bad_request + +req GET "/webhooks/wh-main/events?namespace=$NS&cursor=notanum:abc" +expect_err "events rejects non-int cursor" api.bad_request + +req GET "/webhooks/wh-main/events?namespace=$NS&cursor=123:abc" +expect_err "events accepts valid cursor" ok + +section "retry endpoint" +req POST "/webhooks/wh-main/deliveries/00000000-0000-0000-0000-000000000000/retry?namespace=$NS" +expect_err "retry unknown delivery" webhook.delivery_not_found + +req POST "/webhooks/nope/deliveries/abc/retry?namespace=$NS" +expect_err "retry on missing webhook" webhook.delivery_not_found + +section "event-type allowlist" +req PUT "/webhooks/wh-bad?namespace=$NS" \ + '{"url":"http://127.0.0.1:8099/x","subscriptions":["actor.http_request"]}' +expect_err "rejects high-throughput type" api.bad_request + +req PUT "/webhooks/wh-bad?namespace=$NS" \ + '{"url":"http://127.0.0.1:8099/x","subscriptions":["nonsense"]}' +expect_err "rejects unknown event type" api.bad_request + +section "destination validation (SSRF)" +req PUT "/webhooks/wh-bad?namespace=$NS" '{"url":"not-a-url","subscriptions":[]}' +expect_err "rejects malformed url" webhook.invalid + +req PUT "/webhooks/wh-bad?namespace=$NS" '{"url":"http://169.254.169.254/latest/meta-data/","subscriptions":[]}' +expect_err "rejects link-local metadata" webhook.invalid + +req PUT "/webhooks/wh-bad?namespace=$NS" '{"url":"http://10.0.0.1/hook","subscriptions":[]}' +expect_err "rejects private network" webhook.invalid + +section "header validation" +req PUT "/webhooks/wh-bad?namespace=$NS" \ + "$(python3 -c 'import json;print(json.dumps({"url":"http://127.0.0.1:8099/x","headers":{f"X-{i}":"v" for i in range(17)},"subscriptions":[]}))')" +expect_err "rejects >16 headers" webhook.invalid + +req PUT "/webhooks/wh-bad?namespace=$NS" \ + "$(python3 -c 'import json;print(json.dumps({"url":"http://127.0.0.1:8099/x","headers":{"X-Big":"a"*5000},"subscriptions":[]}))')" +expect_err "rejects oversize header value" webhook.invalid + +req PUT "/webhooks/wh-bad?namespace=$NS" '{"url":"http://127.0.0.1:8099/x","headers":{"Bad Header":"v"},"subscriptions":[]}' +expect_err "rejects invalid header name" webhook.invalid + +req PUT "/webhooks/wh-bad?namespace=$NS" '{"url":"http://127.0.0.1:8099/x","bogus":1}' +expect_err "rejects unknown body field" api.bad_request + +section "failed update leaves config intact" +req PUT "/webhooks/wh-main?namespace=$NS" '{"url":"nope","subscriptions":[]}' +expect_err "bad update is rejected" webhook.invalid +req GET "/webhooks?namespace=$NS" +expect_field "config unchanged after fail" "webhooks.wh-main.url" "http://127.0.0.1:8099/v2" + +section "namespace validation" +req GET "/webhooks?namespace=no-such-ns"; expect_err "list bad namespace" namespace.not_found +req GET "/webhooks/wh-main/events?namespace=no-such-ns"; expect_err "events bad namespace" namespace.not_found +req PUT "/webhooks/wh-main?namespace=no-such-ns" '{"url":"http://127.0.0.1:8099/x","subscriptions":[]}' +expect_err "upsert bad namespace" namespace.not_found +req DELETE "/webhooks/wh-main?namespace=no-such-ns"; expect_err "delete bad namespace" namespace.not_found +req POST "/webhooks/wh-main/deliveries/abc/retry?namespace=no-such-ns"; expect_err "retry bad namespace" namespace.not_found + +section "delete" +req DELETE "/webhooks/wh-main?namespace=$NS" +expect_err "delete existing webhook" ok +req DELETE "/webhooks/wh-main?namespace=$NS" +expect_err "delete is idempotent" ok +req GET "/webhooks?namespace=$NS" +expect_field "webhook is gone" "webhooks" '{}' + +# ---------------------------------------------------------------- summary + +printf '\n\033[1m%d passed, %d failed\033[0m\n' "$pass" "$fail" +printf 'not covered: webhook delivery, retry of a real delivery, and event recording.\n' +printf 'those need a runner pool error to fire a Trigger; no API endpoint can produce one.\n' +[ "$fail" -eq 0 ] diff --git a/self-host/compose/dev-host/rivet-engine/config.jsonc b/self-host/compose/dev-host/rivet-engine/config.jsonc index ee79af406a..1cd7e292d1 100644 --- a/self-host/compose/dev-host/rivet-engine/config.jsonc +++ b/self-host/compose/dev-host/rivet-engine/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@127.0.0.1:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/docker-compose.yml b/self-host/compose/dev-multidc-multinode/docker-compose.yml index 81b39d39eb..a971a9716d 100644 --- a/self-host/compose/dev-multidc-multinode/docker-compose.yml +++ b/self-host/compose/dev-multidc-multinode/docker-compose.yml @@ -208,7 +208,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -254,7 +253,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -298,7 +296,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -519,7 +516,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -563,7 +559,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -607,7 +602,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -826,7 +820,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: @@ -870,7 +863,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: @@ -914,7 +906,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: diff --git a/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc index ac3c4bc3c6..eaef4154fa 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc index 8e5c6aaa45..c42e0a0ffb 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc index 6ba3be6d65..6a6238fec7 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev/rivet-engine/config.jsonc b/self-host/compose/dev/rivet-engine/config.jsonc index 4c6312b6d3..cad7a1e600 100644 --- a/self-host/compose/dev/rivet-engine/config.jsonc +++ b/self-host/compose/dev/rivet-engine/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/template/src/main.ts b/self-host/compose/template/src/main.ts index 82a08e1f73..4a984d1f70 100644 --- a/self-host/compose/template/src/main.ts +++ b/self-host/compose/template/src/main.ts @@ -18,7 +18,7 @@ import { generateDatacenterVectorClient } from "./services/edge/vector-client"; import { generateDatacenterVectorServer } from "./services/edge/vector-server"; function generateTemplate(templateName: string, config: TemplateConfig) { - const outputDir = path.join(__dirname, "../../../", templateName); + const outputDir = path.join(__dirname, "../../", templateName); // Remove existing directory if it exists if (fs.existsSync(outputDir)) { diff --git a/self-host/compose/template/src/services/edge/rivet-engine.ts b/self-host/compose/template/src/services/edge/rivet-engine.ts index 1c5b8247da..554d7b8454 100644 --- a/self-host/compose/template/src/services/edge/rivet-engine.ts +++ b/self-host/compose/template/src/services/edge/rivet-engine.ts @@ -45,6 +45,11 @@ export function generateDatacenterRivetEngine( host: "0.0.0.0", }, topology, + // Serverless runner URLs in a compose deployment point at other containers on this + // network, which the engine refuses to dial by default. + outbound: { + allow_private_networks: true, + }, postgres: { url: `postgresql://postgres:postgres@${context.getServiceHost("postgres", datacenter.name)}:5432/rivet_engine`, },