diff --git a/.direnv/flake-profile b/.direnv/flake-profile new file mode 120000 index 0000000000..c7ae5b709f --- /dev/null +++ b/.direnv/flake-profile @@ -0,0 +1 @@ +flake-profile-2-link \ No newline at end of file diff --git a/.direnv/flake-profile-2-link b/.direnv/flake-profile-2-link new file mode 120000 index 0000000000..140a682073 --- /dev/null +++ b/.direnv/flake-profile-2-link @@ -0,0 +1 @@ +/nix/store/pknafnv7zx67x4misiglnnvbj3sgm30r-nix-shell-env \ No newline at end of file diff --git a/.gitignore b/.gitignore index 2d5e84f164..e3b90ecf92 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ frontend/build/ .idea/ *.swp *.swo +.envrc +flake-profile* # OS .DS_Store diff --git a/Cargo.lock b/Cargo.lock index a2ade3bdd1..56e2169143 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3951,6 +3951,7 @@ dependencies = [ "rivet-envoy-protocol", "rivet-error", "rivet-metrics", + "rivet-outbound-guard", "rivet-pools", "rivet-runner-protocol", "rivet-runtime", @@ -3976,6 +3977,7 @@ dependencies = [ "utoipa", "uuid", "vbare", + "webhook", "xxhash-rust", ] @@ -4112,12 +4114,15 @@ dependencies = [ "rivet-config", "rivet-envoy-protocol", "rivet-metrics", + "rivet-outbound-guard", + "rivet-pools", "rivet-runtime", "rivet-types", "tokio", "tracing", "universaldb", "universalpubsub", + "url", "vbare", ] @@ -5106,6 +5111,7 @@ dependencies = [ "tracing", "urlencoding", "utoipa", + "webhook", ] [[package]] @@ -5468,6 +5474,7 @@ dependencies = [ "urlencoding", "uuid", "vbare", + "webhook", ] [[package]] @@ -5693,6 +5700,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "rivet-outbound-guard" +version = "2.3.7" +dependencies = [ + "anyhow", + "ipnet", + "reqwest 0.12.22", + "rivet-config", + "thiserror 1.0.69", + "tokio", + "tracing", + "url", +] + [[package]] name = "rivet-perf" version = "2.3.7" @@ -5719,6 +5740,7 @@ dependencies = [ "rivet-async-nats", "rivet-config", "rivet-metrics", + "rivet-outbound-guard", "rivet-util", "rustls", "serde", @@ -6077,6 +6099,7 @@ dependencies = [ "pegboard", "rivet-config", "tracing", + "webhook", ] [[package]] @@ -8722,6 +8745,33 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webhook" +version = "2.3.7" +dependencies = [ + "anyhow", + "chrono", + "epoxy", + "epoxy-protocol", + "futures-util", + "gasoline", + "namespace", + "reqwest 0.12.22", + "rivet-data", + "rivet-error", + "rivet-outbound-guard", + "rivet-pools", + "rivet-util", + "serde", + "serde_bare", + "serde_json", + "tracing", + "universaldb", + "url", + "uuid", + "vbare", +] + [[package]] name = "webpki-root-certs" version = "1.0.7" diff --git a/Cargo.toml b/Cargo.toml index a9048c159f..26f0b4f26b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,6 +32,7 @@ members = [ "engine/packages/metrics", "engine/packages/metrics-server", "engine/packages/namespace", + "engine/packages/outbound-guard", "engine/packages/pegboard", "engine/packages/pegboard-envoy", "engine/packages/pegboard-gateway", @@ -56,6 +57,7 @@ members = [ "engine/packages/universalpubsub", "engine/packages/util", "engine/packages/util-id", + "engine/packages/webhook", "engine/packages/util-serde", "engine/packages/workflow-worker", "engine/sdks/rust/api-full", @@ -500,6 +502,9 @@ members = [ [workspace.dependencies.namespace] path = "engine/packages/namespace" + [workspace.dependencies.rivet-outbound-guard] + path = "engine/packages/outbound-guard" + [workspace.dependencies.pegboard] path = "engine/packages/pegboard" @@ -573,6 +578,9 @@ members = [ [workspace.dependencies.rivet-util-id] path = "engine/packages/util-id" + [workspace.dependencies.webhook] + path = "engine/packages/webhook" + [workspace.dependencies.rivet-util-serde] path = "engine/packages/util-serde" version = "=2.3.7" diff --git a/docs/content/docs/debugging.mdx b/docs/content/docs/debugging.mdx index 93cfe96e1b..432ec7deaf 100644 --- a/docs/content/docs/debugging.mdx +++ b/docs/content/docs/debugging.mdx @@ -199,7 +199,7 @@ Returns the configured provider settings per datacenter and the latest pool erro } ``` -`runner_pool_error` mirrors actor scheduling errors such as `serverless_http_error`, `serverless_connection_error`, and `serverless_stream_ended_early`. +`runner_pool_error` mirrors actor scheduling errors such as `serverless_http_error`, `serverless_connection_error`, `serverless_destination_blocked`, and `serverless_stream_ended_early`. ### Check Serverless Provider Health diff --git a/docs/content/docs/troubleshooting.mdx b/docs/content/docs/troubleshooting.mdx index f06c30fe14..75ba34704d 100644 --- a/docs/content/docs/troubleshooting.mdx +++ b/docs/content/docs/troubleshooting.mdx @@ -97,6 +97,14 @@ Rivet was unable to connect to your serverless endpoint. Check that: - Your server is publicly reachable from the internet. - There are no DNS or firewall issues blocking the connection. +### `serverless_destination_blocked` + +The configured serverless URL points at a destination Rivet is not allowed to reach. Rivet dials serverless endpoints from inside its own network, so by default it refuses any destination that is not publicly routable: private ranges, link-local and cloud metadata addresses, carrier-grade NAT, and other reserved ranges. Check that: + +- Your endpoint URL is publicly reachable, and is not an internal hostname or address. +- The URL scheme is `http` or `https`, and the URL does not embed credentials. +- If you are self-hosting and intentionally point runners at an address on your own network, set `outbound.allow_private_networks` to `true` in your engine config, or list the specific range in `outbound.allow_cidrs`. + ### `serverless_stream_ended_early` The connection to your serverless endpoint was terminated before the actor finished. This usually means your serverless function hit its execution time limit. Ensure that your Rivet provider's request lifespan is configured to match the max duration of your serverless platform. diff --git a/engine/artifacts/config-schema.json b/engine/artifacts/config-schema.json index 9147b7a7d1..23533f876c 100644 --- a/engine/artifacts/config-schema.json +++ b/engine/artifacts/config-schema.json @@ -106,6 +106,17 @@ } ] }, + "outbound": { + "default": null, + "anyOf": [ + { + "$ref": "#/definitions/Outbound" + }, + { + "type": "null" + } + ] + }, "pegboard": { "default": null, "anyOf": [ @@ -803,6 +814,63 @@ }, "additionalProperties": false }, + "Outbound": { + "description": "Policy for outbound HTTP requests to user-configured destinations, such as serverless runner URLs.\n\nThese requests originate from inside the trusted engine network, so without restrictions a caller who can configure a runner can reach internal-only services. The defaults deny every non-globally-routable destination except loopback.", + "type": "object", + "properties": { + "allow_cidrs": { + "description": "Additional CIDRs that are always permitted, evaluated after `deny_cidrs`.\n\nUse this to reach a specific internal service without opening up the whole private range.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "allow_insecure_scheme": { + "description": "Allow plaintext `http://` destinations. When disabled only `https://` is permitted.", + "type": [ + "boolean", + "null" + ] + }, + "allow_loopback": { + "description": "Allow destinations that resolve to loopback addresses (127.0.0.0/8, ::1).\n\nEnabled by default so local development against `http://localhost:...` works without configuration.", + "type": [ + "boolean", + "null" + ] + }, + "allow_private_networks": { + "description": "Allow destinations that resolve to private, link-local, shared (CGNAT), or otherwise non-globally-routable addresses.\n\nSelf-hosted deployments that point runners at addresses on their own network, such as a Docker Compose service name, need this enabled.", + "type": [ + "boolean", + "null" + ] + }, + "deny_cidrs": { + "description": "Additional CIDRs that are always denied. Takes precedence over every allow rule.", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + } + }, + "max_redirects": { + "description": "Maximum number of redirects to follow. Every hop is re-checked against this policy.", + "type": [ + "integer", + "null" + ], + "format": "uint", + "minimum": 0.0 + } + }, + "additionalProperties": false + }, "Pegboard": { "type": "object", "properties": { diff --git a/engine/artifacts/errors/webhook.conflict.json b/engine/artifacts/errors/webhook.conflict.json new file mode 100644 index 0000000000..f0ad50310a --- /dev/null +++ b/engine/artifacts/errors/webhook.conflict.json @@ -0,0 +1,5 @@ +{ + "code": "conflict", + "group": "webhook", + "message": "Webhook config changed concurrently." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.delivery_failed.json b/engine/artifacts/errors/webhook.delivery_failed.json new file mode 100644 index 0000000000..7454fcc450 --- /dev/null +++ b/engine/artifacts/errors/webhook.delivery_failed.json @@ -0,0 +1,5 @@ +{ + "code": "delivery_failed", + "group": "webhook", + "message": "Webhook delivery failed." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.delivery_not_found.json b/engine/artifacts/errors/webhook.delivery_not_found.json new file mode 100644 index 0000000000..0217d3cb8f --- /dev/null +++ b/engine/artifacts/errors/webhook.delivery_not_found.json @@ -0,0 +1,5 @@ +{ + "code": "delivery_not_found", + "group": "webhook", + "message": "Webhook delivery not found." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.delivery_not_retryable.json b/engine/artifacts/errors/webhook.delivery_not_retryable.json new file mode 100644 index 0000000000..b5f02f7b7f --- /dev/null +++ b/engine/artifacts/errors/webhook.delivery_not_retryable.json @@ -0,0 +1,5 @@ +{ + "code": "delivery_not_retryable", + "group": "webhook", + "message": "Webhook delivery is not in a retryable state." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.destination_blocked.json b/engine/artifacts/errors/webhook.destination_blocked.json new file mode 100644 index 0000000000..e9fa286998 --- /dev/null +++ b/engine/artifacts/errors/webhook.destination_blocked.json @@ -0,0 +1,5 @@ +{ + "code": "destination_blocked", + "group": "webhook", + "message": "Webhook destination is not allowed." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.event_type_not_allowed.json b/engine/artifacts/errors/webhook.event_type_not_allowed.json new file mode 100644 index 0000000000..c7d19f6278 --- /dev/null +++ b/engine/artifacts/errors/webhook.event_type_not_allowed.json @@ -0,0 +1,5 @@ +{ + "code": "event_type_not_allowed", + "group": "webhook", + "message": "Event type cannot be subscribed to by a webhook." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.invalid.json b/engine/artifacts/errors/webhook.invalid.json new file mode 100644 index 0000000000..3c00e6e1d5 --- /dev/null +++ b/engine/artifacts/errors/webhook.invalid.json @@ -0,0 +1,5 @@ +{ + "code": "invalid", + "group": "webhook", + "message": "Invalid webhook config." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.not_found.json b/engine/artifacts/errors/webhook.not_found.json new file mode 100644 index 0000000000..6b843aecf7 --- /dev/null +++ b/engine/artifacts/errors/webhook.not_found.json @@ -0,0 +1,5 @@ +{ + "code": "not_found", + "group": "webhook", + "message": "Webhook not found." +} \ No newline at end of file diff --git a/engine/artifacts/errors/webhook.request_failed.json b/engine/artifacts/errors/webhook.request_failed.json new file mode 100644 index 0000000000..27da83ee75 --- /dev/null +++ b/engine/artifacts/errors/webhook.request_failed.json @@ -0,0 +1,5 @@ +{ + "code": "request_failed", + "group": "webhook", + "message": "Webhook delivery request failed." +} \ No newline at end of file diff --git a/engine/artifacts/openapi.json b/engine/artifacts/openapi.json index e54b6132c2..15fe9ec92f 100644 --- a/engine/artifacts/openapi.json +++ b/engine/artifacts/openapi.json @@ -1103,6 +1103,246 @@ } ] } + }, + "/webhooks": { + "get": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_list", + "parameters": [ + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksListResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + } + }, + "/webhooks/{webhook_name}": { + "put": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_upsert", + "parameters": [ + { + "name": "webhook_name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksUpsertRequestBody" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksUpsertResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + }, + "delete": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_delete", + "parameters": [ + { + "name": "webhook_name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksDeleteResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + } + }, + "/webhooks/{webhook_name}/deliveries/{delivery_id}/retry": { + "post": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_retry_delivery", + "parameters": [ + { + "name": "webhook_name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "delivery_id", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksRetryDeliveryResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + } + }, + "/webhooks/{webhook_name}/events": { + "get": { + "tags": [ + "webhooks" + ], + "operationId": "webhooks_events", + "parameters": [ + { + "name": "webhook_name", + "in": "path", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "namespace", + "in": "query", + "required": true, + "schema": { + "type": "string" + } + }, + { + "name": "limit", + "in": "query", + "required": false, + "schema": { + "type": "integer", + "minimum": 0 + } + }, + { + "name": "cursor", + "in": "query", + "required": false, + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WebhooksEventsResponse" + } + } + } + } + }, + "security": [ + { + "bearer_auth": [] + } + ] + } } }, "components": { @@ -2290,6 +2530,133 @@ } }, "additionalProperties": false + }, + "WebhookConfig": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "headers": { + "type": "object", + "additionalProperties": { + "type": "string" + }, + "propertyNames": { + "type": "string" + } + }, + "subscriptions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEventType" + } + }, + "url": { + "type": "string" + } + }, + "additionalProperties": false + }, + "WebhookEvent": { + "type": "object", + "required": [ + "id", + "create_ts", + "status", + "event_type", + "attempt_count" + ], + "properties": { + "attempt_count": { + "type": "integer", + "format": "int32", + "minimum": 0 + }, + "create_ts": { + "type": "integer", + "format": "int64" + }, + "event_type": { + "type": "string" + }, + "id": { + "type": "string" + }, + "last_error": { + "type": [ + "string", + "null" + ] + }, + "status": { + "type": "string" + } + }, + "additionalProperties": false + }, + "WebhookEventType": { + "type": "string", + "enum": [ + "runner_pool.error", + "runner_pool.healthy" + ] + }, + "WebhooksDeleteResponse": { + "type": "object", + "additionalProperties": false + }, + "WebhooksEventsResponse": { + "type": "object", + "required": [ + "events", + "pagination" + ], + "properties": { + "events": { + "type": "array", + "items": { + "$ref": "#/components/schemas/WebhookEvent" + } + }, + "pagination": { + "$ref": "#/components/schemas/Pagination" + } + }, + "additionalProperties": false + }, + "WebhooksListResponse": { + "type": "object", + "required": [ + "webhooks", + "pagination" + ], + "properties": { + "pagination": { + "$ref": "#/components/schemas/Pagination" + }, + "webhooks": { + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/WebhookConfig" + }, + "propertyNames": { + "type": "string" + } + } + }, + "additionalProperties": false + }, + "WebhooksRetryDeliveryResponse": { + "type": "object", + "additionalProperties": false + }, + "WebhooksUpsertRequestBody": { + "$ref": "#/components/schemas/WebhookConfig" + }, + "WebhooksUpsertResponse": { + "type": "object", + "additionalProperties": false } }, "securitySchemes": { diff --git a/engine/packages/api-public/Cargo.toml b/engine/packages/api-public/Cargo.toml index b5289c97bc..d82f5596c6 100644 --- a/engine/packages/api-public/Cargo.toml +++ b/engine/packages/api-public/Cargo.toml @@ -37,6 +37,7 @@ tower-http.workspace = true tracing.workspace = true urlencoding.workspace = true utoipa.workspace = true +webhook.workspace = true [build-dependencies] anyhow.workspace = true diff --git a/engine/packages/api-public/src/lib.rs b/engine/packages/api-public/src/lib.rs index c7a2c33706..0638a41d9e 100644 --- a/engine/packages/api-public/src/lib.rs +++ b/engine/packages/api-public/src/lib.rs @@ -10,5 +10,6 @@ pub mod router; pub mod runner_configs; pub mod runners; pub mod ui; +pub mod webhooks; pub use router::router; diff --git a/engine/packages/api-public/src/router.rs b/engine/packages/api-public/src/router.rs index d2a79fb99b..ce80749b72 100644 --- a/engine/packages/api-public/src/router.rs +++ b/engine/packages/api-public/src/router.rs @@ -10,6 +10,7 @@ use utoipa::OpenApi; use crate::{ actors, ctx, datacenters, envoys, health, metadata, namespaces, runner_configs, runners, ui, + webhooks, }; #[derive(OpenApi)] @@ -33,6 +34,11 @@ use crate::{ runner_configs::delete::delete, runner_configs::serverless_health_check::serverless_health_check, runner_configs::refresh_metadata::refresh_metadata, + webhooks::list, + webhooks::upsert, + webhooks::delete, + webhooks::retry_delivery, + webhooks::events, datacenters::list, health::fanout, metadata::get, @@ -81,6 +87,24 @@ pub async fn router( "/runner-configs/{runner_name}/refresh-metadata", axum::routing::post(runner_configs::refresh_metadata), ) + // MARK: Webhooks + .route("/webhooks", axum::routing::get(webhooks::list)) + .route( + "/webhooks/{webhook_name}", + axum::routing::put(webhooks::upsert), + ) + .route( + "/webhooks/{webhook_name}", + axum::routing::delete(webhooks::delete), + ) + .route( + "/webhooks/{webhook_name}/deliveries/{delivery_id}/retry", + axum::routing::post(webhooks::retry_delivery), + ) + .route( + "/webhooks/{webhook_name}/events", + axum::routing::get(webhooks::events), + ) // MARK: Actors .route("/actors", axum::routing::get(actors::list::list)) .route("/actors", axum::routing::post(actors::create::create)) diff --git a/engine/packages/api-public/src/webhooks.rs b/engine/packages/api-public/src/webhooks.rs new file mode 100644 index 0000000000..228a3ab477 --- /dev/null +++ b/engine/packages/api-public/src/webhooks.rs @@ -0,0 +1,500 @@ +use std::collections::HashMap; + +use anyhow::Result; +use axum::response::{IntoResponse, Response}; +use rivet_api_builder::{ + ApiError, + errors::ApiBadRequest, + extract::{Extension, Json, Path, Query}, +}; +use rivet_api_types::pagination::Pagination; +use serde::{Deserialize, Serialize}; +use utoipa::{IntoParams, ToSchema}; + +use namespace::errors::Namespace as NamespaceError; +use namespace::ops::resolve_for_name_global::Input as ResolveNamespace; + +use crate::ctx::ApiCtx; + +#[derive(Debug, Deserialize, Serialize, Clone, ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WebhookConfig { + pub url: String, + #[serde(default)] + pub headers: HashMap, + #[serde(default)] + pub subscriptions: Vec, +} + +#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, ToSchema)] +pub enum WebhookEventType { + #[serde(rename = "runner_pool.error")] + RunnerPoolError, + #[serde(rename = "runner_pool.healthy")] + RunnerPoolHealthy, +} + +impl From for webhook::types::WebhookEventType { + fn from(value: WebhookEventType) -> Self { + match value { + WebhookEventType::RunnerPoolError => webhook::types::WebhookEventType::RunnerPoolError, + WebhookEventType::RunnerPoolHealthy => { + webhook::types::WebhookEventType::RunnerPoolHealthy + } + } + } +} + +impl WebhookEventType { + fn from_internal(value: webhook::types::WebhookEventType) -> Option { + match value { + webhook::types::WebhookEventType::RunnerPoolError => { + Some(WebhookEventType::RunnerPoolError) + } + webhook::types::WebhookEventType::RunnerPoolHealthy => { + Some(WebhookEventType::RunnerPoolHealthy) + } + webhook::types::WebhookEventType::ActorHttpRequest => None, + } + } +} + +// MARK: List + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct ListQuery { + pub namespace: String, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksListResponse)] +pub struct ListResponse { + pub webhooks: HashMap, + pub pagination: Pagination, +} + +#[utoipa::path( + get, + operation_id = "webhooks_list", + path = "/webhooks", + params(ListQuery), + responses( + (status = 200, body = ListResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn list(Extension(ctx): Extension, Query(query): Query) -> Response { + match list_inner(ctx, query).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +#[tracing::instrument(skip_all)] +async fn list_inner(ctx: ApiCtx, query: ListQuery) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(ResolveNamespace { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| NamespaceError::NotFound.build())?; + + let webhooks = ctx + .op(webhook::ops::list::Input { + namespace_id: namespace.namespace_id, + }) + .await?; + + Ok(ListResponse { + webhooks: webhooks + .into_iter() + .map(|w| { + ( + w.name, + WebhookConfig { + url: w.config.url, + headers: w.config.headers, + subscriptions: w + .config + .subscriptions + .into_iter() + .filter_map(WebhookEventType::from_internal) + .collect(), + }, + ) + }) + .collect(), + pagination: Pagination { cursor: None }, + }) +} + +// MARK: Upsert + +#[derive(Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct UpsertPath { + pub webhook_name: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct UpsertQuery { + pub namespace: String, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksUpsertRequestBody)] +pub struct UpsertRequest(pub WebhookConfig); + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksUpsertResponse)] +pub struct UpsertResponse {} + +#[utoipa::path( + put, + operation_id = "webhooks_upsert", + path = "/webhooks/{webhook_name}", + params( + ("webhook_name" = String, Path), + UpsertQuery, + ), + request_body(content = UpsertRequest, content_type = "application/json"), + responses( + (status = 200, body = UpsertResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn upsert( + Extension(ctx): Extension, + Path(path): Path, + Query(query): Query, + Json(body): Json, +) -> Response { + match upsert_inner(ctx, path, query, body).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +#[tracing::instrument(skip_all)] +async fn upsert_inner( + ctx: ApiCtx, + path: UpsertPath, + query: UpsertQuery, + body: UpsertRequest, +) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(ResolveNamespace { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| NamespaceError::NotFound.build())?; + + ctx.op(webhook::ops::upsert::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + config: webhook::types::WebhookConfig { + url: body.0.url, + headers: body.0.headers, + subscriptions: body.0.subscriptions.into_iter().map(Into::into).collect(), + }, + }) + .await?; + + Ok(UpsertResponse {}) +} + +// MARK: Delete + +#[derive(Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct DeletePath { + pub webhook_name: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct DeleteQuery { + pub namespace: String, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksDeleteResponse)] +pub struct DeleteResponse {} + +#[utoipa::path( + delete, + operation_id = "webhooks_delete", + path = "/webhooks/{webhook_name}", + params( + ("webhook_name" = String, Path), + DeleteQuery, + ), + responses( + (status = 200, body = DeleteResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn delete( + Extension(ctx): Extension, + Path(path): Path, + Query(query): Query, +) -> Response { + match delete_inner(ctx, path, query).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +#[tracing::instrument(skip_all)] +async fn delete_inner(ctx: ApiCtx, path: DeletePath, query: DeleteQuery) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(ResolveNamespace { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| NamespaceError::NotFound.build())?; + + ctx.op(webhook::ops::delete::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + }) + .await?; + + Ok(DeleteResponse {}) +} + +// MARK: Retry delivery + +#[derive(Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct RetryDeliveryPath { + pub webhook_name: String, + pub delivery_id: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct RetryDeliveryQuery { + pub namespace: String, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksRetryDeliveryResponse)] +pub struct RetryDeliveryResponse {} + +#[utoipa::path( + post, + operation_id = "webhooks_retry_delivery", + path = "/webhooks/{webhook_name}/deliveries/{delivery_id}/retry", + params( + ("webhook_name" = String, Path), + ("delivery_id" = String, Path), + RetryDeliveryQuery, + ), + responses( + (status = 200, body = RetryDeliveryResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn retry_delivery( + Extension(ctx): Extension, + Path(path): Path, + Query(query): Query, +) -> Response { + match retry_delivery_inner(ctx, path, query).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +#[tracing::instrument(skip_all)] +async fn retry_delivery_inner( + ctx: ApiCtx, + path: RetryDeliveryPath, + query: RetryDeliveryQuery, +) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(ResolveNamespace { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| NamespaceError::NotFound.build())?; + + ctx.op(webhook::ops::retry::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + delivery_id: path.delivery_id.clone(), + }) + .await?; + + Ok(RetryDeliveryResponse {}) +} + +// MARK: Events + +#[derive(Deserialize, Clone)] +#[serde(deny_unknown_fields)] +pub struct EventsPath { + pub webhook_name: String, +} + +#[derive(Debug, Deserialize, Serialize, Clone, IntoParams)] +#[serde(deny_unknown_fields)] +#[into_params(parameter_in = Query)] +pub struct EventsQuery { + pub namespace: String, + pub limit: Option, + pub cursor: Option, +} + +// One delivery in a webhook's event history. `id` is the delivery id, which is also the +// CloudEvents `id` sent to the destination and what the retry endpoint takes. +#[derive(Debug, Deserialize, Serialize, Clone, ToSchema)] +#[serde(deny_unknown_fields)] +pub struct WebhookEvent { + pub id: String, + pub create_ts: i64, + pub status: String, + pub event_type: String, + pub attempt_count: u32, + pub last_error: Option, +} + +#[derive(Deserialize, Serialize, ToSchema)] +#[serde(deny_unknown_fields)] +#[schema(as = WebhooksEventsResponse)] +pub struct EventsResponse { + pub events: Vec, + pub pagination: Pagination, +} + +#[utoipa::path( + get, + operation_id = "webhooks_events", + path = "/webhooks/{webhook_name}/events", + params( + ("webhook_name" = String, Path), + EventsQuery, + ), + responses( + (status = 200, body = EventsResponse), + ), + security(("bearer_auth" = [])), +)] +#[tracing::instrument(skip_all)] +pub async fn events( + Extension(ctx): Extension, + Path(path): Path, + Query(query): Query, +) -> Response { + match events_inner(ctx, path, query).await { + Ok(response) => Json(response).into_response(), + Err(err) => ApiError::from(err).into_response(), + } +} + +const DEFAULT_EVENTS_LIMIT: usize = 20; + +#[tracing::instrument(skip_all)] +async fn events_inner(ctx: ApiCtx, path: EventsPath, query: EventsQuery) -> Result { + ctx.auth().await?; + + let namespace = ctx + .op(ResolveNamespace { + name: query.namespace.clone(), + }) + .await? + .ok_or_else(|| NamespaceError::NotFound.build())?; + + if ctx + .op(webhook::ops::get::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + }) + .await? + .is_none() + { + return Err(webhook::errors::Webhook::NotFound.build()); + } + + let mut deliveries = ctx + .op(webhook::ops::list_deliveries::Input { + namespace_id: namespace.namespace_id, + name: path.webhook_name.clone(), + }) + .await?; + + deliveries.sort_by(|a, b| { + b.record + .created_at + .cmp(&a.record.created_at) + .then_with(|| b.delivery_id.cmp(&a.delivery_id)) + }); + + if let Some(cursor) = query.cursor { + let (created_at, delivery_id) = cursor + .split_once(':') + .and_then(|(ts, id)| ts.parse::().ok().map(|ts| (ts, id.to_string()))) + .ok_or_else(|| { + ApiBadRequest { + reason: "cursor must be formatted as `{create_ts}:{delivery_id}`".to_string(), + } + .build() + })?; + + deliveries.retain(|d| { + (d.record.created_at, d.delivery_id.as_str()) < (created_at, delivery_id.as_str()) + }); + } + + let limit = query.limit.unwrap_or(DEFAULT_EVENTS_LIMIT); + let has_more = deliveries.len() > limit; + deliveries.truncate(limit); + + let cursor = has_more + .then(|| deliveries.last()) + .flatten() + .map(|last| format!("{}:{}", last.record.created_at, last.delivery_id)); + + Ok(EventsResponse { + events: deliveries + .into_iter() + .map(|d| WebhookEvent { + id: d.delivery_id, + create_ts: d.record.created_at, + status: match d.record.status { + webhook::types::DeliveryStatus::Pending => "pending".to_string(), + webhook::types::DeliveryStatus::Succeeded => "succeeded".to_string(), + webhook::types::DeliveryStatus::Failed => "failed".to_string(), + }, + event_type: d.record.event_type.as_str().to_string(), + attempt_count: d.record.attempt_count, + last_error: d.record.last_error, + }) + .collect(), + pagination: Pagination { cursor }, + }) +} diff --git a/engine/packages/config/src/config/mod.rs b/engine/packages/config/src/config/mod.rs index 5ee03c6d2a..babed64296 100644 --- a/engine/packages/config/src/config/mod.rs +++ b/engine/packages/config/src/config/mod.rs @@ -11,6 +11,7 @@ pub mod db; pub mod guard; pub mod logs; pub mod metrics; +pub mod outbound; pub mod pegboard; pub mod pubsub; pub mod pyroscope; @@ -27,6 +28,7 @@ pub use db::Database; pub use guard::*; pub use logs::*; pub use metrics::*; +pub use outbound::*; pub use pegboard::*; pub use pubsub::PubSub; pub use pyroscope::*; @@ -110,6 +112,9 @@ pub struct Root { #[serde(default)] pub pyroscope: Option, + + #[serde(default)] + pub outbound: Option, } impl Default for Root { @@ -130,6 +135,7 @@ impl Default for Root { sqlite: None, metrics: Default::default(), pyroscope: None, + outbound: None, } } } @@ -145,6 +151,11 @@ impl Root { self.api_peer.as_ref().unwrap_or(&DEFAULT) } + pub fn outbound(&self) -> &Outbound { + static DEFAULT: LazyLock = LazyLock::new(Outbound::default); + self.outbound.as_ref().unwrap_or(&DEFAULT) + } + pub fn pegboard(&self) -> &Pegboard { static DEFAULT: LazyLock = LazyLock::new(Pegboard::default); self.pegboard.as_ref().unwrap_or(&DEFAULT) diff --git a/engine/packages/config/src/config/outbound.rs b/engine/packages/config/src/config/outbound.rs new file mode 100644 index 0000000000..d0aa65b35f --- /dev/null +++ b/engine/packages/config/src/config/outbound.rs @@ -0,0 +1,60 @@ +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Policy for outbound HTTP requests to user-configured destinations, such as serverless runner +/// URLs. +/// +/// These requests originate from inside the trusted engine network, so without restrictions a +/// caller who can configure a runner can reach internal-only services. The defaults deny every +/// non-globally-routable destination except loopback. +#[derive(Debug, Serialize, Deserialize, Clone, Default, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct Outbound { + /// Allow destinations that resolve to loopback addresses (127.0.0.0/8, ::1). + /// + /// Enabled by default so local development against `http://localhost:...` works without + /// configuration. + pub allow_loopback: Option, + /// Allow destinations that resolve to private, link-local, shared (CGNAT), or otherwise + /// non-globally-routable addresses. + /// + /// Self-hosted deployments that point runners at addresses on their own network, such as a + /// Docker Compose service name, need this enabled. + pub allow_private_networks: Option, + /// Allow plaintext `http://` destinations. When disabled only `https://` is permitted. + pub allow_insecure_scheme: Option, + /// Additional CIDRs that are always permitted, evaluated after `deny_cidrs`. + /// + /// Use this to reach a specific internal service without opening up the whole private range. + pub allow_cidrs: Option>, + /// Additional CIDRs that are always denied. Takes precedence over every allow rule. + pub deny_cidrs: Option>, + /// Maximum number of redirects to follow. Every hop is re-checked against this policy. + pub max_redirects: Option, +} + +impl Outbound { + pub fn allow_loopback(&self) -> bool { + self.allow_loopback.unwrap_or(true) + } + + pub fn allow_private_networks(&self) -> bool { + self.allow_private_networks.unwrap_or(false) + } + + pub fn allow_insecure_scheme(&self) -> bool { + self.allow_insecure_scheme.unwrap_or(true) + } + + pub fn allow_cidrs(&self) -> &[String] { + self.allow_cidrs.as_deref().unwrap_or(&[]) + } + + pub fn deny_cidrs(&self) -> &[String] { + self.deny_cidrs.as_deref().unwrap_or(&[]) + } + + pub fn max_redirects(&self) -> usize { + self.max_redirects.unwrap_or(4) + } +} diff --git a/engine/packages/engine/Cargo.toml b/engine/packages/engine/Cargo.toml index eb1b4ba069..af778dc573 100644 --- a/engine/packages/engine/Cargo.toml +++ b/engine/packages/engine/Cargo.toml @@ -94,3 +94,4 @@ tracing-subscriber.workspace = true urlencoding.workspace = true universalpubsub.workspace = true vbare.workspace = true +webhook.workspace = true diff --git a/engine/packages/engine/tests/api_webhooks.rs b/engine/packages/engine/tests/api_webhooks.rs new file mode 100644 index 0000000000..e90871a175 --- /dev/null +++ b/engine/packages/engine/tests/api_webhooks.rs @@ -0,0 +1,427 @@ +#[path = "common/mod.rs"] +mod common; + +use std::collections::HashMap; +use std::sync::{ + Arc, + atomic::{AtomicU16, Ordering}, +}; +use std::time::Duration; + +use axum::{ + Router, + body::Bytes, + extract::State, + http::{HeaderMap, StatusCode}, + routing::post, +}; +use rivet_api_public::webhooks::{ + DeleteQuery, EventsQuery, RetryDeliveryQuery, UpsertQuery, UpsertRequest, WebhookConfig, + WebhookEvent, WebhookEventType, +}; +use serde_json::json; +use tokio::sync::mpsc; + +const WEBHOOK_NAME: &str = "test-webhook"; + +fn config(url: &str) -> WebhookConfig { + WebhookConfig { + url: url.to_string(), + headers: HashMap::new(), + subscriptions: vec![WebhookEventType::RunnerPoolError], + } +} + +async fn upsert_response(port: u16, namespace: &str, config: WebhookConfig) -> reqwest::Response { + common::api::public::build_webhooks_upsert_request( + port, + WEBHOOK_NAME, + UpsertQuery { + namespace: namespace.to_string(), + }, + UpsertRequest(config), + ) + .await + .expect("failed to build upsert request") + .send() + .await + .expect("failed to send upsert request") +} + +async fn events_response(port: u16, namespace: &str) -> reqwest::Response { + common::api::public::build_webhooks_events_request( + port, + WEBHOOK_NAME, + EventsQuery { + namespace: namespace.to_string(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to build events request") + .send() + .await + .expect("failed to send events request") +} + +// MARK: Upsert + +#[test] +fn upsert_creates_webhook() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + common::api::public::webhooks_upsert( + port, + WEBHOOK_NAME, + UpsertQuery { + namespace: namespace.clone(), + }, + UpsertRequest(config("http://127.0.0.1:1")), + ) + .await + .expect("failed to upsert webhook"); + + let events = common::api::public::webhooks_events( + port, + WEBHOOK_NAME, + EventsQuery { + namespace, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to get webhook events"); + + assert!(events.events.is_empty()); + }); +} + +#[test] +fn upsert_updates_existing_webhook() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + for url in ["http://127.0.0.1:1", "http://127.0.0.1:2"] { + common::api::public::webhooks_upsert( + port, + WEBHOOK_NAME, + UpsertQuery { + namespace: namespace.clone(), + }, + UpsertRequest(config(url)), + ) + .await + .expect("failed to upsert webhook"); + } + }); +} + +// MARK: Validation + +#[test] +fn upsert_rejects_invalid_url() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + let response = upsert_response(port, &namespace, config("not a url")).await; + common::assert_error_response(response, "invalid").await; + + // A rejected config must not be stored. + let response = events_response(port, &namespace).await; + common::assert_error_response(response, "not_found").await; + }); +} + +#[test] +fn upsert_rejects_too_many_headers() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + let mut config = config("http://127.0.0.1:1"); + config.headers = (0..17) + .map(|i| (format!("x-header-{i}"), "value".to_string())) + .collect(); + + let response = upsert_response(port, &namespace, config).await; + common::assert_error_response(response, "invalid").await; + }); +} + +#[test] +fn upsert_rejects_invalid_header_name() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + let mut config = config("http://127.0.0.1:1"); + config + .headers + .insert("bad header".to_string(), "value".to_string()); + + let response = upsert_response(port, &namespace, config).await; + common::assert_error_response(response, "invalid").await; + }); +} + +// MARK: Delete + +#[test] +fn delete_removes_webhook() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + common::api::public::webhooks_upsert( + port, + WEBHOOK_NAME, + UpsertQuery { + namespace: namespace.clone(), + }, + UpsertRequest(config("http://127.0.0.1:1")), + ) + .await + .expect("failed to upsert webhook"); + + common::api::public::webhooks_delete( + port, + WEBHOOK_NAME, + DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete webhook"); + + let response = events_response(port, &namespace).await; + common::assert_error_response(response, "not_found").await; + }); +} + +#[test] +fn events_for_unknown_webhook_is_not_found() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + let response = events_response(port, &namespace).await; + common::assert_error_response(response, "not_found").await; + }); +} + +// MARK: Delivery + +type ReceivedRequest = (HeaderMap, serde_json::Value); + +struct Receiver { + status: AtomicU16, + tx: mpsc::UnboundedSender, +} + +async fn receive( + State(state): State>, + headers: HeaderMap, + body: Bytes, +) -> StatusCode { + let body = serde_json::from_slice(&body).expect("webhook body is not json"); + let _ = state.tx.send((headers, body)); + StatusCode::from_u16(state.status.load(Ordering::SeqCst)).expect("invalid status") +} + +async fn spawn_receiver( + status: StatusCode, +) -> ( + String, + Arc, + mpsc::UnboundedReceiver, +) { + let (tx, rx) = mpsc::unbounded_channel(); + let state = Arc::new(Receiver { + status: AtomicU16::new(status.as_u16()), + tx, + }); + let app = Router::new() + .route("/", post(receive)) + .with_state(state.clone()); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("failed to bind webhook receiver"); + let url = format!( + "http://{}/", + listener + .local_addr() + .expect("failed to read receiver address") + ); + tokio::spawn(async move { + axum::serve(listener, app).await.expect("receiver error"); + }); + + (url, state, rx) +} + +async fn trigger(ctx: &common::TestCtx, namespace_id: rivet_util::Id) { + ctx.leader_dc() + .workflow_ctx + .signal(webhook::workflows::webhook::Trigger { + event: webhook::types::WebhookEvent { + event_type: webhook::types::WebhookEventType::RunnerPoolError, + subject: Some("test-runner".to_string()), + data: json!({ "message": "boom" }), + }, + }) + .to_workflow::() + .tag("namespace_id", namespace_id) + .tag("name", WEBHOOK_NAME) + .send() + .await + .expect("failed to send trigger"); +} + +// The delivery record is written by the workflow after the receiver responds, and the test has +// nothing to await for that write, so poll the events endpoint until the status shows up. +async fn wait_for_delivery(port: u16, namespace: &str, status: &str) -> WebhookEvent { + loop { + let events = common::api::public::webhooks_events( + port, + WEBHOOK_NAME, + EventsQuery { + namespace: namespace.to_string(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to get webhook events"); + + if let Some(event) = events.events.into_iter().find(|e| e.status == status) { + return event; + } + + tokio::time::sleep(Duration::from_millis(50)).await; + } +} + +async fn upsert_receiver(port: u16, namespace: &str, url: &str) { + let mut config = config(url); + config + .headers + .insert("x-test".to_string(), "value".to_string()); + + common::api::public::webhooks_upsert( + port, + WEBHOOK_NAME, + UpsertQuery { + namespace: namespace.to_string(), + }, + UpsertRequest(config), + ) + .await + .expect("failed to upsert webhook"); +} + +#[test] +fn delivery_sends_cloudevent() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, namespace_id) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + let (url, _state, mut rx) = spawn_receiver(StatusCode::OK).await; + upsert_receiver(port, &namespace, &url).await; + trigger(&ctx, namespace_id).await; + + let (headers, body) = rx.recv().await.expect("receiver closed"); + assert_eq!( + headers.get("content-type").expect("missing content-type"), + "application/cloudevents+json" + ); + assert_eq!( + headers.get("x-test").expect("missing custom header"), + "value" + ); + assert_eq!(body["specversion"], "1.0"); + assert_eq!(body["type"], "dev.rivet.runner_pool.error"); + assert_eq!( + body["source"], + format!("rivet:webhook:{namespace_id}:{WEBHOOK_NAME}") + ); + assert_eq!(body["subject"], "test-runner"); + assert_eq!(body["data"], json!({ "message": "boom" })); + + let event = wait_for_delivery(port, &namespace, "succeeded").await; + assert_eq!(body["id"], event.id); + assert_eq!(event.event_type, "runner_pool.error"); + assert_eq!(event.attempt_count, 1); + assert!(event.last_error.is_none()); + }); +} + +#[test] +fn failed_delivery_can_be_retried() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, namespace_id) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + // A 400 is not retryable, so the delivery fails on its first attempt. + let (url, state, mut rx) = spawn_receiver(StatusCode::BAD_REQUEST).await; + upsert_receiver(port, &namespace, &url).await; + trigger(&ctx, namespace_id).await; + + let (_, first_body) = rx.recv().await.expect("receiver closed"); + let failed = wait_for_delivery(port, &namespace, "failed").await; + assert_eq!(failed.attempt_count, 1); + assert!(failed.last_error.is_some()); + + state + .status + .store(StatusCode::OK.as_u16(), Ordering::SeqCst); + + common::api::public::webhooks_retry_delivery( + port, + WEBHOOK_NAME, + &failed.id, + RetryDeliveryQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to retry delivery"); + + let (_, retry_body) = rx.recv().await.expect("receiver closed"); + assert_eq!(retry_body["id"], first_body["id"]); + + let succeeded = wait_for_delivery(port, &namespace, "succeeded").await; + assert_eq!(succeeded.id, failed.id); + }); +} + +#[test] +fn retry_unknown_delivery_is_not_found() { + common::run(common::TestOpts::new(1), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + let port = ctx.leader_dc().guard_port(); + + upsert_receiver(port, &namespace, "http://127.0.0.1:1").await; + + let response = common::api::public::build_webhooks_retry_delivery_request( + port, + WEBHOOK_NAME, + "missing", + RetryDeliveryQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to build retry request") + .send() + .await + .expect("failed to send retry request"); + common::assert_error_response(response, "delivery_not_found").await; + }); +} diff --git a/engine/packages/engine/tests/common/api/public.rs b/engine/packages/engine/tests/common/api/public.rs index 5b724ba8a9..0f6488a737 100644 --- a/engine/packages/engine/tests/common/api/public.rs +++ b/engine/packages/engine/tests/common/api/public.rs @@ -473,3 +473,109 @@ pub async fn health_fanout(port: u16) -> Result { let response = request.send().await?; parse_response(response).await } + +// MARK: Webhooks + +pub async fn build_webhooks_upsert_request( + port: u16, + webhook_name: &str, + query: rivet_api_public::webhooks::UpsertQuery, + request: rivet_api_public::webhooks::UpsertRequest, +) -> Result { + let client = rivet_pools::reqwest::client().await?; + Ok(client + .put(format!( + "{}/webhooks/{}?{}", + get_endpoint(port), + webhook_name, + serde_html_form::to_string(&query)?, + )) + .json(&request)) +} + +pub async fn webhooks_upsert( + port: u16, + webhook_name: &str, + query: rivet_api_public::webhooks::UpsertQuery, + request: rivet_api_public::webhooks::UpsertRequest, +) -> Result { + let request = build_webhooks_upsert_request(port, webhook_name, query, request).await?; + let response = request.send().await?; + parse_response(response).await +} + +pub async fn build_webhooks_delete_request( + port: u16, + webhook_name: &str, + query: rivet_api_public::webhooks::DeleteQuery, +) -> Result { + let client = rivet_pools::reqwest::client().await?; + Ok(client.delete(format!( + "{}/webhooks/{}?{}", + get_endpoint(port), + webhook_name, + serde_html_form::to_string(&query)?, + ))) +} + +pub async fn webhooks_delete( + port: u16, + webhook_name: &str, + query: rivet_api_public::webhooks::DeleteQuery, +) -> Result { + let request = build_webhooks_delete_request(port, webhook_name, query).await?; + let response = request.send().await?; + parse_response(response).await +} + +pub async fn build_webhooks_retry_delivery_request( + port: u16, + webhook_name: &str, + delivery_id: &str, + query: rivet_api_public::webhooks::RetryDeliveryQuery, +) -> Result { + let client = rivet_pools::reqwest::client().await?; + Ok(client.post(format!( + "{}/webhooks/{}/deliveries/{}/retry?{}", + get_endpoint(port), + webhook_name, + delivery_id, + serde_html_form::to_string(&query)?, + ))) +} + +pub async fn webhooks_retry_delivery( + port: u16, + webhook_name: &str, + delivery_id: &str, + query: rivet_api_public::webhooks::RetryDeliveryQuery, +) -> Result { + let request = + build_webhooks_retry_delivery_request(port, webhook_name, delivery_id, query).await?; + let response = request.send().await?; + parse_response(response).await +} + +pub async fn build_webhooks_events_request( + port: u16, + webhook_name: &str, + query: rivet_api_public::webhooks::EventsQuery, +) -> Result { + let client = rivet_pools::reqwest::client().await?; + Ok(client.get(format!( + "{}/webhooks/{}/events?{}", + get_endpoint(port), + webhook_name, + serde_html_form::to_string(&query)?, + ))) +} + +pub async fn webhooks_events( + port: u16, + webhook_name: &str, + query: rivet_api_public::webhooks::EventsQuery, +) -> Result { + let request = build_webhooks_events_request(port, webhook_name, query).await?; + let response = request.send().await?; + parse_response(response).await +} diff --git a/engine/packages/outbound-guard/Cargo.toml b/engine/packages/outbound-guard/Cargo.toml new file mode 100644 index 0000000000..a47d61ea49 --- /dev/null +++ b/engine/packages/outbound-guard/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "rivet-outbound-guard" +publish = false +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +ipnet.workspace = true +reqwest.workspace = true +rivet-config.workspace = true +thiserror.workspace = true +tokio.workspace = true +tracing.workspace = true +url.workspace = true + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } diff --git a/engine/packages/outbound-guard/src/client.rs b/engine/packages/outbound-guard/src/client.rs new file mode 100644 index 0000000000..8dc44cedf8 --- /dev/null +++ b/engine/packages/outbound-guard/src/client.rs @@ -0,0 +1,82 @@ +use std::net::SocketAddr; +use std::sync::Arc; + +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; + +use crate::policy::{BlockReason, Policy}; + +/// A DNS resolver that drops every address the policy disallows. +/// +/// Placing the check here rather than before the request closes the DNS rebinding window: reqwest +/// connects to exactly the addresses this returns, and it runs for every redirect hop as well as +/// the initial request. +#[derive(Debug)] +pub struct GuardedResolver { + policy: Arc, +} + +impl GuardedResolver { + pub fn new(policy: Arc) -> Self { + GuardedResolver { policy } + } +} + +impl Resolve for GuardedResolver { + fn resolve(&self, name: Name) -> Resolving { + let policy = self.policy.clone(); + let host = name.as_str().to_string(); + + Box::pin(async move { + // The port is discarded by the connector, which substitutes the real one. + let resolved = tokio::net::lookup_host((host.as_str(), 0)) + .await + .map_err(|err| { + tracing::debug!(%host, ?err, "failed to resolve outbound host"); + Box::new(BlockReason::ResolutionFailed { host: host.clone() }) + as Box + })? + .map(|addr| addr.ip()) + .collect::>(); + + let addrs = policy + .filter_addrs(&host, resolved)? + .into_iter() + .map(|ip| SocketAddr::new(ip, 0)) + .collect::>(); + + Ok(Box::new(addrs.into_iter()) as Addrs) + }) + } +} + +/// Build the redirect policy a guarded client must be configured with. +/// +/// Every hop is re-checked, so a destination cannot bounce the engine somewhere it was not +/// allowed to reach directly. +pub fn redirect_policy(policy: Arc) -> reqwest::redirect::Policy { + let max_redirects = policy.max_redirects(); + + reqwest::redirect::Policy::custom(move |attempt| { + if attempt.previous().len() >= max_redirects { + return attempt.error(BlockReason::TooManyRedirects { max: max_redirects }); + } + + match policy.check_url(attempt.url()) { + Ok(()) => attempt.follow(), + Err(reason) => { + tracing::debug!(url = %attempt.url(), %reason, "blocked outbound redirect"); + attempt.error(reason) + } + } + }) +} + +/// Recover the [`BlockReason`] that caused a request to fail, if the policy is what stopped it. +/// +/// The reason is buried in the source chain of a `reqwest::Error`, so callers that want to tell +/// "the destination is not allowed" apart from "the destination is down" have to walk it. +pub fn block_reason(err: &anyhow::Error) -> Option { + err.chain() + .find_map(|err| err.downcast_ref::()) + .cloned() +} diff --git a/engine/packages/outbound-guard/src/lib.rs b/engine/packages/outbound-guard/src/lib.rs new file mode 100644 index 0000000000..4076d4299f --- /dev/null +++ b/engine/packages/outbound-guard/src/lib.rs @@ -0,0 +1,14 @@ +//! Destination policy for outbound HTTP requests to user-configured URLs. +//! +//! Serverless runner URLs are supplied by whoever can write a runner config, and the engine dials +//! them from inside its own trusted network. This crate is the trust boundary for those requests: +//! it decides which destinations are reachable, and supplies the resolver and redirect policy that +//! enforce that decision at connect time. +//! +//! Clients are built in `rivet_pools::reqwest`, which owns every `reqwest::Client` in the process. + +mod client; +mod policy; + +pub use client::{GuardedResolver, block_reason, redirect_policy}; +pub use policy::{AddressClass, BlockReason, Policy}; diff --git a/engine/packages/outbound-guard/src/policy.rs b/engine/packages/outbound-guard/src/policy.rs new file mode 100644 index 0000000000..a0d255c9b7 --- /dev/null +++ b/engine/packages/outbound-guard/src/policy.rs @@ -0,0 +1,375 @@ +use std::fmt; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + +use anyhow::{Context, Result}; +use ipnet::IpNet; +use url::Url; + +/// Why a destination was rejected. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum BlockReason { + #[error("url is not a valid absolute url")] + InvalidUrl, + #[error("scheme {scheme:?} is not allowed, expected http or https")] + UnsupportedScheme { scheme: String }, + #[error("plaintext http is not allowed, use https")] + InsecureScheme, + #[error("url must not contain embedded credentials")] + EmbeddedCredentials, + #[error("url has no host")] + MissingHost, + #[error("address {addr} is not an allowed destination ({class})")] + BlockedAddress { addr: IpAddr, class: AddressClass }, + #[error("address {addr} is explicitly denied")] + DeniedAddress { addr: IpAddr }, + #[error("host {host:?} resolved to no allowed addresses")] + NoAllowedAddresses { host: String }, + #[error("failed to resolve host {host:?}")] + ResolutionFailed { host: String }, + #[error("exceeded the maximum of {max} redirects")] + TooManyRedirects { max: usize }, +} + +/// The reason an address is not globally routable. +/// +/// Every class except [`AddressClass::Loopback`] is governed by +/// `outbound.allow_private_networks`. An address that matches no class is globally +/// routable and always allowed. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AddressClass { + /// `0.0.0.0` or `::`, which the kernel routes to a local interface. + Unspecified, + /// `127.0.0.0/8` or `::1`. + Loopback, + /// `169.254.0.0/16` or `fe80::/10`. Covers the cloud metadata endpoint. + LinkLocal, + /// `10/8`, `172.16/12`, `192.168/16`. + Private, + /// `255.255.255.255`. + Broadcast, + /// `224.0.0.0/4` or `ff00::/8`. + Multicast, + /// `192.0.2.0/24`, `198.51.100.0/24`, `203.0.113.0/24`, or `2001:db8::/32`. + Documentation, + /// `100.64.0.0/10`, the carrier-grade NAT range. + Shared, + /// `192.0.0.0/24`, reserved for IETF protocol assignments. + ProtocolAssignments, + /// `198.18.0.0/15`, reserved for network benchmarking. + Benchmarking, + /// `240.0.0.0/4`. + Reserved, + /// `fc00::/7`, the IPv6 equivalent of the private ranges. + UniqueLocal, + /// `100::/64`, which is discarded rather than routed. + Discard, + /// An IPv6 address carrying an IPv4 destination that is itself globally routable. + /// + /// These are held to the same rule as the private classes because they are an easy way to + /// smuggle a destination past a filter that only understands one address family. + Ipv4Embedded, +} + +impl fmt::Display for AddressClass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let name = match self { + AddressClass::Unspecified => "unspecified", + AddressClass::Loopback => "loopback", + AddressClass::LinkLocal => "link-local", + AddressClass::Private => "private", + AddressClass::Broadcast => "broadcast", + AddressClass::Multicast => "multicast", + AddressClass::Documentation => "documentation", + AddressClass::Shared => "shared", + AddressClass::ProtocolAssignments => "protocol-assignments", + AddressClass::Benchmarking => "benchmarking", + AddressClass::Reserved => "reserved", + AddressClass::UniqueLocal => "unique-local", + AddressClass::Discard => "discard", + AddressClass::Ipv4Embedded => "ipv4-embedded", + }; + + f.write_str(name) + } +} + +impl AddressClass { + /// Classify an address, or `None` if it is globally routable. + pub fn of(addr: IpAddr) -> Option { + match addr { + IpAddr::V4(v4) => AddressClass::of_ipv4(v4), + IpAddr::V6(v6) => AddressClass::of_ipv6(v6), + } + } + + fn of_ipv4(addr: Ipv4Addr) -> Option { + let [a, b, c, _] = addr.octets(); + + if addr.is_unspecified() { + return Some(AddressClass::Unspecified); + } + if addr.is_loopback() { + return Some(AddressClass::Loopback); + } + if addr.is_link_local() { + return Some(AddressClass::LinkLocal); + } + if addr.is_private() { + return Some(AddressClass::Private); + } + if addr.is_broadcast() { + return Some(AddressClass::Broadcast); + } + if addr.is_multicast() { + return Some(AddressClass::Multicast); + } + if addr.is_documentation() { + return Some(AddressClass::Documentation); + } + if a == 100 && (64..128).contains(&b) { + return Some(AddressClass::Shared); + } + if a == 192 && b == 0 && c == 0 { + return Some(AddressClass::ProtocolAssignments); + } + if a == 198 && (b == 18 || b == 19) { + return Some(AddressClass::Benchmarking); + } + if a >= 240 { + return Some(AddressClass::Reserved); + } + + None + } + + fn of_ipv6(addr: Ipv6Addr) -> Option { + // An IPv4 address wearing an IPv6 costume routes to the embedded IPv4 destination, so + // classify it as that address rather than trusting the outer form. + if let Some(v4) = unwrap_embedded_ipv4(addr) { + return AddressClass::of_ipv4(v4).or(Some(AddressClass::Ipv4Embedded)); + } + + let segments = addr.segments(); + + if addr.is_unspecified() { + return Some(AddressClass::Unspecified); + } + if addr.is_loopback() { + return Some(AddressClass::Loopback); + } + if addr.is_multicast() { + return Some(AddressClass::Multicast); + } + if segments[0] & 0xfe00 == 0xfc00 { + return Some(AddressClass::UniqueLocal); + } + if segments[0] & 0xffc0 == 0xfe80 { + return Some(AddressClass::LinkLocal); + } + if segments[0] == 0x2001 && segments[1] == 0x0db8 { + return Some(AddressClass::Documentation); + } + if segments[0] == 0x0100 && segments[1..4] == [0, 0, 0] { + return Some(AddressClass::Discard); + } + + None + } +} + +/// Extract the IPv4 destination an IPv6 address actually routes to, if any. +/// +/// Covers IPv4-mapped (`::ffff:0:0/96`), IPv4-compatible (`::/96`), and the well-known NAT64 +/// prefix (`64:ff9b::/96`). +fn unwrap_embedded_ipv4(addr: Ipv6Addr) -> Option { + if let Some(v4) = addr.to_ipv4_mapped() { + return Some(v4); + } + + let segments = addr.segments(); + let tail = Ipv4Addr::new( + (segments[6] >> 8) as u8, + (segments[6] & 0xff) as u8, + (segments[7] >> 8) as u8, + (segments[7] & 0xff) as u8, + ); + + if segments[0] == 0x0064 && segments[1] == 0xff9b && segments[2..6] == [0, 0, 0, 0] { + return Some(tail); + } + + // IPv4-compatible addresses are deprecated but still routed. `::` and `::1` have their own + // classes, so skip anything in the lowest /104. + if segments[0..6] == [0, 0, 0, 0, 0, 0] && segments[6] != 0 { + return Some(tail); + } + + None +} + +/// Which destinations the engine may reach on behalf of a user-supplied URL. +#[derive(Debug, Clone)] +pub struct Policy { + allow_loopback: bool, + allow_private_networks: bool, + allow_insecure_scheme: bool, + allow_cidrs: Vec, + deny_cidrs: Vec, + max_redirects: usize, +} + +impl Policy { + pub fn from_config(config: &rivet_config::Config) -> Result { + let outbound = config.outbound(); + + Ok(Policy { + allow_loopback: outbound.allow_loopback(), + allow_private_networks: outbound.allow_private_networks(), + allow_insecure_scheme: outbound.allow_insecure_scheme(), + allow_cidrs: parse_cidrs(outbound.allow_cidrs(), "outbound.allow_cidrs")?, + deny_cidrs: parse_cidrs(outbound.deny_cidrs(), "outbound.deny_cidrs")?, + max_redirects: outbound.max_redirects(), + }) + } + + pub fn max_redirects(&self) -> usize { + self.max_redirects + } + + /// Check everything about a destination that can be known without resolving DNS. + /// + /// This is the gate that runs when a runner config is written, so a bad URL is rejected before + /// it is ever stored. It is also re-run on every redirect hop. + pub fn check_url(&self, url: &Url) -> Result<(), BlockReason> { + match url.scheme() { + "https" => {} + "http" => { + if !self.allow_insecure_scheme { + return Err(BlockReason::InsecureScheme); + } + } + scheme => { + return Err(BlockReason::UnsupportedScheme { + scheme: scheme.to_string(), + }); + } + } + + // Credentials in the URL would be replayed to whatever the destination redirects to. + if !url.username().is_empty() || url.password().is_some() { + return Err(BlockReason::EmbeddedCredentials); + } + + let Some(host) = url.host() else { + return Err(BlockReason::MissingHost); + }; + + // A hostname is checked once it resolves, at connect time. An address literal skips the + // resolver entirely, so it has to be checked here. + match host { + url::Host::Ipv4(addr) => self.check_addr(IpAddr::V4(addr)), + url::Host::Ipv6(addr) => self.check_addr(IpAddr::V6(addr)), + url::Host::Domain(_) => Ok(()), + } + } + + /// Check a single resolved address. + pub fn check_addr(&self, addr: IpAddr) -> Result<(), BlockReason> { + // An IPv6 address that carries an IPv4 destination has to match CIDR rules written for + // either form, so both are tested. + let mut forms = vec![addr]; + if let IpAddr::V6(v6) = addr { + if let Some(v4) = unwrap_embedded_ipv4(v6) { + forms.push(IpAddr::V4(v4)); + } + } + + // An explicit deny always wins, including over the allow list. + if forms + .iter() + .any(|form| self.deny_cidrs.iter().any(|net| net.contains(form))) + { + return Err(BlockReason::DeniedAddress { addr }); + } + + if forms + .iter() + .any(|form| self.allow_cidrs.iter().any(|net| net.contains(form))) + { + return Ok(()); + } + + let Some(class) = AddressClass::of(addr) else { + return Ok(()); + }; + + let allowed = match class { + AddressClass::Loopback => self.allow_loopback, + AddressClass::Unspecified + | AddressClass::LinkLocal + | AddressClass::Private + | AddressClass::Broadcast + | AddressClass::Multicast + | AddressClass::Documentation + | AddressClass::Shared + | AddressClass::ProtocolAssignments + | AddressClass::Benchmarking + | AddressClass::Reserved + | AddressClass::UniqueLocal + | AddressClass::Discard + | AddressClass::Ipv4Embedded => self.allow_private_networks, + }; + + if allowed { + Ok(()) + } else { + Err(BlockReason::BlockedAddress { addr, class }) + } + } + + /// Filter a resolver's answer down to the addresses this policy permits. + /// + /// Dropping individual addresses rather than rejecting the whole answer keeps a dual-stack + /// host reachable when only one of its families is allowed, and still guarantees the + /// connection can only land on an address that passed. + pub fn filter_addrs( + &self, + host: &str, + addrs: impl IntoIterator, + ) -> Result, BlockReason> { + let mut allowed = Vec::new(); + + for addr in addrs { + match self.check_addr(addr) { + Ok(()) => allowed.push(addr), + Err(reason) => { + tracing::debug!(%host, %addr, %reason, "dropped disallowed resolved address"); + } + } + } + + if allowed.is_empty() { + Err(BlockReason::NoAllowedAddresses { + host: host.to_string(), + }) + } else { + Ok(allowed) + } + } +} + +fn parse_cidrs(raw: &[String], label: &str) -> Result> { + raw.iter() + .map(|entry| { + let entry = entry.trim(); + // Accept a bare address as a single-host CIDR so operators do not have to write /32. + if let Ok(addr) = entry.parse::() { + return Ok(IpNet::from(addr)); + } + + entry + .parse::() + .with_context(|| format!("invalid cidr in {label}: {entry:?}")) + }) + .collect() +} diff --git a/engine/packages/outbound-guard/tests/policy.rs b/engine/packages/outbound-guard/tests/policy.rs new file mode 100644 index 0000000000..dd4b19b368 --- /dev/null +++ b/engine/packages/outbound-guard/tests/policy.rs @@ -0,0 +1,240 @@ +use std::net::IpAddr; + +use rivet_config::config::{Outbound, Root}; +use rivet_outbound_guard::{AddressClass, BlockReason, Policy}; +use url::Url; + +fn policy(outbound: Outbound) -> Policy { + let config = rivet_config::Config::from_root(Root { + outbound: Some(outbound), + ..Default::default() + }); + + Policy::from_config(&config).expect("policy should build") +} + +fn default_policy() -> Policy { + policy(Outbound::default()) +} + +fn check(policy: &Policy, url: &str) -> Result<(), BlockReason> { + policy.check_url(&Url::parse(url).expect("test url should parse")) +} + +fn addr(raw: &str) -> IpAddr { + raw.parse().expect("test address should parse") +} + +#[test] +fn allows_public_destinations() { + let policy = default_policy(); + + check(&policy, "https://runner.example.com/start").expect("public host should be allowed"); + check(&policy, "https://8.8.8.8/start").expect("public literal should be allowed"); + policy + .check_addr(addr("2606:4700::1")) + .expect("public v6 should be allowed"); +} + +#[test] +fn allows_loopback_by_default() { + let policy = default_policy(); + + check(&policy, "http://127.0.0.1:6420/start").expect("loopback v4 should be allowed"); + check(&policy, "http://[::1]:6420/start").expect("loopback v6 should be allowed"); +} + +#[test] +fn denies_loopback_when_disabled() { + let policy = policy(Outbound { + allow_loopback: Some(false), + ..Default::default() + }); + + assert_eq!( + check(&policy, "http://127.0.0.1:6420/start"), + Err(BlockReason::BlockedAddress { + addr: addr("127.0.0.1"), + class: AddressClass::Loopback, + }), + ); +} + +#[test] +fn denies_private_ranges_by_default() { + let policy = default_policy(); + + for (raw, class) in [ + ("10.0.0.5", AddressClass::Private), + ("172.16.4.1", AddressClass::Private), + ("192.168.1.1", AddressClass::Private), + ("169.254.169.254", AddressClass::LinkLocal), + ("100.64.0.1", AddressClass::Shared), + ("198.18.0.1", AddressClass::Benchmarking), + ("0.0.0.0", AddressClass::Unspecified), + ("240.0.0.1", AddressClass::Reserved), + ("fd00::1", AddressClass::UniqueLocal), + ("fe80::1", AddressClass::LinkLocal), + ] { + assert_eq!( + policy.check_addr(addr(raw)), + Err(BlockReason::BlockedAddress { + addr: addr(raw), + class, + }), + "{raw} should be blocked", + ); + } +} + +#[test] +fn denies_ipv6_wrapped_ipv4_metadata_endpoint() { + let policy = default_policy(); + + // The same destination reached through three different IPv6 encodings. + for raw in ["::ffff:169.254.169.254", "64:ff9b::169.254.169.254"] { + assert_eq!( + policy.check_addr(addr(raw)), + Err(BlockReason::BlockedAddress { + addr: addr(raw), + class: AddressClass::LinkLocal, + }), + "{raw} should be blocked", + ); + } +} + +#[test] +fn allows_private_ranges_when_enabled() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + ..Default::default() + }); + + policy + .check_addr(addr("10.0.0.5")) + .expect("private should be allowed when enabled"); + policy + .check_addr(addr("169.254.169.254")) + .expect("link-local should be allowed when enabled"); +} + +#[test] +fn allow_cidrs_open_a_single_destination() { + let policy = policy(Outbound { + allow_cidrs: Some(vec!["10.1.2.0/24".to_string(), "192.168.5.9".to_string()]), + ..Default::default() + }); + + policy + .check_addr(addr("10.1.2.7")) + .expect("allow-listed cidr should be allowed"); + policy + .check_addr(addr("192.168.5.9")) + .expect("bare address should be read as a single-host cidr"); + assert!( + policy.check_addr(addr("10.1.3.7")).is_err(), + "address outside the allow-listed cidr should stay blocked", + ); +} + +#[test] +fn deny_cidrs_win_over_allow_rules() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + allow_cidrs: Some(vec!["169.254.0.0/16".to_string()]), + deny_cidrs: Some(vec!["169.254.169.254".to_string()]), + ..Default::default() + }); + + policy + .check_addr(addr("169.254.1.1")) + .expect("rest of the range should still be reachable"); + assert_eq!( + policy.check_addr(addr("169.254.169.254")), + Err(BlockReason::DeniedAddress { + addr: addr("169.254.169.254"), + }), + ); +} + +#[test] +fn deny_cidrs_catch_the_ipv6_wrapped_form() { + let policy = policy(Outbound { + allow_private_networks: Some(true), + deny_cidrs: Some(vec!["169.254.169.254".to_string()]), + ..Default::default() + }); + + assert_eq!( + policy.check_addr(addr("::ffff:169.254.169.254")), + Err(BlockReason::DeniedAddress { + addr: addr("::ffff:169.254.169.254"), + }), + ); +} + +#[test] +fn rejects_non_http_schemes() { + let policy = default_policy(); + + assert_eq!( + check(&policy, "file:///etc/passwd"), + Err(BlockReason::UnsupportedScheme { + scheme: "file".to_string(), + }), + ); + assert_eq!( + check(&policy, "gopher://example.com/"), + Err(BlockReason::UnsupportedScheme { + scheme: "gopher".to_string(), + }), + ); +} + +#[test] +fn rejects_plaintext_http_when_disabled() { + let policy = policy(Outbound { + allow_insecure_scheme: Some(false), + ..Default::default() + }); + + assert_eq!( + check(&policy, "http://runner.example.com/"), + Err(BlockReason::InsecureScheme), + ); + check(&policy, "https://runner.example.com/").expect("https should still be allowed"); +} + +#[test] +fn rejects_embedded_credentials() { + let policy = default_policy(); + + assert_eq!( + check(&policy, "https://user:pass@runner.example.com/"), + Err(BlockReason::EmbeddedCredentials), + ); +} + +#[test] +fn filter_addrs_keeps_only_allowed_answers() { + let policy = default_policy(); + + let allowed = policy + .filter_addrs( + "rebind.example.com", + [addr("10.0.0.1"), addr("93.184.216.34")], + ) + .expect("a dual answer with one public address should still connect"); + assert_eq!(allowed, vec![addr("93.184.216.34")]); + + assert_eq!( + policy.filter_addrs( + "rebind.example.com", + [addr("10.0.0.1"), addr("192.168.0.1")] + ), + Err(BlockReason::NoAllowedAddresses { + host: "rebind.example.com".to_string(), + }), + ); +} diff --git a/engine/packages/pegboard-outbound/Cargo.toml b/engine/packages/pegboard-outbound/Cargo.toml index e48c0ba430..6095f44b01 100644 --- a/engine/packages/pegboard-outbound/Cargo.toml +++ b/engine/packages/pegboard-outbound/Cargo.toml @@ -18,10 +18,13 @@ reqwest.workspace = true rivet-config.workspace = true rivet-envoy-protocol.workspace = true rivet-metrics.workspace = true +rivet-outbound-guard.workspace = true +rivet-pools.workspace = true rivet-runtime.workspace = true rivet-types.workspace = true tokio.workspace = true tracing.workspace = true universaldb.workspace = true universalpubsub.workspace = true +url.workspace = true vbare.workspace = true diff --git a/engine/packages/pegboard-outbound/src/lib.rs b/engine/packages/pegboard-outbound/src/lib.rs index 00697ea7b0..4a1341fd7c 100644 --- a/engine/packages/pegboard-outbound/src/lib.rs +++ b/engine/packages/pegboard-outbound/src/lib.rs @@ -327,6 +327,7 @@ fn error_label(error: &RunnerPoolError) -> &'static str { match error { RunnerPoolError::ServerlessHttpError { .. } => "http_error", RunnerPoolError::ServerlessConnectionError { .. } => "connection_error", + RunnerPoolError::ServerlessDestinationBlocked { .. } => "destination_blocked", RunnerPoolError::ServerlessStreamEndedEarly => "stream_ended_early", RunnerPoolError::ServerlessInvalidSsePayload { .. } => "invalid_payload", RunnerPoolError::Downgrade => "downgrade", @@ -345,6 +346,7 @@ fn status_label(error: &RunnerPoolError) -> &'static str { _ => "other", }, RunnerPoolError::ServerlessConnectionError { .. } + | RunnerPoolError::ServerlessDestinationBlocked { .. } | RunnerPoolError::ServerlessStreamEndedEarly | RunnerPoolError::ServerlessInvalidSsePayload { .. } | RunnerPoolError::Downgrade @@ -362,6 +364,7 @@ fn error_result_label(error: &RunnerPoolError) -> &'static str { _ => "error_http_other", }, RunnerPoolError::ServerlessConnectionError { .. } => "error_connection", + RunnerPoolError::ServerlessDestinationBlocked { .. } => "error_destination_blocked", RunnerPoolError::ServerlessStreamEndedEarly => "error_stream_ended", RunnerPoolError::ServerlessInvalidSsePayload { .. } => "error_invalid_payload", RunnerPoolError::Downgrade => "error_downgrade", @@ -441,7 +444,37 @@ async fn serverless_outbound_req( let endpoint_url = format!("{}/start", url.trim_end_matches('/')); - let client = rivet_pools::reqwest::client_no_timeout().await?; + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here, which also re-gates configs stored before this + // policy existed. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + let block_reason = match url::Url::parse(&endpoint_url) { + Ok(parsed_url) => policy.check_url(&parsed_url).err(), + Err(_) => Some(rivet_outbound_guard::BlockReason::InvalidUrl), + }; + if let Some(reason) = block_reason { + tracing::warn!( + ?namespace_id, + %pool_name, + %reason, + "serverless url is not an allowed destination, dropping outbound req" + ); + + report_error( + ctx, + namespace_id, + pool_name, + RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + ) + .await; + + return Ok(()); + } + + let client = rivet_pools::reqwest::guarded_client_no_timeout(ctx.config()).await?; let req = client .post(endpoint_url.clone()) .body(payload) @@ -550,13 +583,20 @@ async fn serverless_outbound_req( Err(err) => { let wrapped_err = anyhow::Error::from(err); - let error = RunnerPoolError::ServerlessConnectionError { - // Print entire error chain - message: wrapped_err - .chain() - .map(|err| err.to_string()) - .collect::>() - .join("\n"), + // A hostname that only resolves to disallowed addresses is rejected by the + // resolver, which the pre-flight check above cannot see. + let error = match rivet_outbound_guard::block_reason(&wrapped_err) { + Some(reason) => RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + None => RunnerPoolError::ServerlessConnectionError { + // Print entire error chain + message: wrapped_err + .chain() + .map(|err| err.to_string()) + .collect::>() + .join("\n"), + }, }; report_error(ctx, namespace_id, &pool_name, error.clone()).await; observe_req_duration( diff --git a/engine/packages/pegboard/Cargo.toml b/engine/packages/pegboard/Cargo.toml index 6e9701246f..7f47ada9b1 100644 --- a/engine/packages/pegboard/Cargo.toml +++ b/engine/packages/pegboard/Cargo.toml @@ -30,6 +30,7 @@ rivet-data.workspace = true rivet-envoy-protocol.workspace = true rivet-error.workspace = true rivet-metrics.workspace = true +rivet-outbound-guard.workspace = true rivet-pools.workspace = true rivet-runner-protocol.workspace = true rivet-runtime.workspace = true @@ -52,6 +53,7 @@ url.workspace = true utoipa.workspace = true uuid.workspace = true vbare.workspace = true +webhook.workspace = true [dev-dependencies] portpicker.workspace = true diff --git a/engine/packages/pegboard/src/ops/runner_config/upsert.rs b/engine/packages/pegboard/src/ops/runner_config/upsert.rs index e5d2c3c4b3..ad3865612e 100644 --- a/engine/packages/pegboard/src/ops/runner_config/upsert.rs +++ b/engine/packages/pegboard/src/ops/runner_config/upsert.rs @@ -34,9 +34,23 @@ pub async fn pegboard_runner_config_upsert(ctx: &OperationCtx, input: &Input) -> slots_per_runner, .. } => { - if let Err(err) = url::Url::parse(url) { + let parsed_url = match url::Url::parse(url) { + Ok(parsed_url) => parsed_url, + Err(err) => { + return Err(errors::RunnerConfig::Invalid { + reason: format!("invalid serverless url: {err}"), + } + .build()); + } + }; + + // Reject destinations the engine is not allowed to reach before the config is stored. + // Requests are checked again when they connect, which catches configs written before + // this gate existed and hosts whose DNS answer changes afterwards. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { return Err(errors::RunnerConfig::Invalid { - reason: format!("invalid serverless url: {err}"), + reason: format!("invalid serverless url: {reason}"), } .build()); } diff --git a/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs b/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs index 2d25ba3c70..da4dab6cd8 100644 --- a/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs +++ b/engine/packages/pegboard/src/ops/serverless_metadata/fetch.rs @@ -21,6 +21,7 @@ pub struct Input { #[derive(Clone, Debug, PartialEq, Eq)] pub enum ServerlessMetadataError { InvalidRequest {}, + DestinationBlocked { reason: String }, RequestFailed {}, RequestTimedOut {}, NonSuccessStatus { status_code: u16, body: String }, @@ -59,6 +60,14 @@ impl From for ServerlessMetadataErrorEnvelope { details: None, metadata: serde_json::json!({ "kind": "invalid_request" }), }, + ServerlessMetadataError::DestinationBlocked { reason } => Self { + message: "serverless endpoint is not an allowed destination".to_string(), + details: Some(reason.clone()), + metadata: serde_json::json!({ + "kind": "destination_blocked", + "reason": reason, + }), + }, ServerlessMetadataError::RequestFailed {} => Self { message: "failed to reach serverless endpoint".to_string(), details: None, @@ -158,8 +167,19 @@ pub async fn pegboard_serverless_metadata_fetch( let metadata_url = format!("{}/metadata", trimmed_url.trim_end_matches('/')); - if reqwest::Url::parse(&metadata_url).is_err() { + let Ok(parsed_url) = reqwest::Url::parse(&metadata_url) else { return Ok(Err(ServerlessMetadataError::InvalidRequest {})); + }; + + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here. This op also backs the health check endpoint, + // where an unchecked URL is a probe for whatever the engine can reach. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { + return Ok(Err(ServerlessMetadataError::DestinationBlocked { + reason: reason.to_string(), + })); } let mut header_map = ReqwestHeaderMap::new(); @@ -177,7 +197,7 @@ pub async fn pegboard_serverless_metadata_fetch( header_map.insert(header_name, header_value); } - let client = match rivet_pools::reqwest::client().await { + let client = match rivet_pools::reqwest::guarded_client(ctx.config()).await { Ok(c) => c, Err(_) => return Ok(Err(ServerlessMetadataError::RequestFailed {})), }; @@ -193,7 +213,18 @@ pub async fn pegboard_serverless_metadata_fetch( { Ok(r) => r, Err(err) => { - return Ok(Err(if err.is_timeout() { + let is_timeout = err.is_timeout(); + let err = anyhow::Error::from(err); + + // A hostname that only resolves to disallowed addresses is rejected by the resolver, + // which the pre-flight check above cannot see. + if let Some(reason) = rivet_outbound_guard::block_reason(&err) { + return Ok(Err(ServerlessMetadataError::DestinationBlocked { + reason: reason.to_string(), + })); + } + + return Ok(Err(if is_timeout { ServerlessMetadataError::RequestTimedOut {} } else { ServerlessMetadataError::RequestFailed {} diff --git a/engine/packages/pegboard/src/workflows/serverless/conn.rs b/engine/packages/pegboard/src/workflows/serverless/conn.rs index 0064fab984..d621ba4976 100644 --- a/engine/packages/pegboard/src/workflows/serverless/conn.rs +++ b/engine/packages/pegboard/src/workflows/serverless/conn.rs @@ -299,9 +299,22 @@ async fn outbound_req_inner( let endpoint_url = format!("{}/start", url.trim_end_matches('/')); + // The client only sees this URL as a host to connect to: an address literal never reaches its + // resolver, and its redirect policy only runs on later hops. So the scheme, credentials, and a + // literal destination have to be checked here, which also re-gates configs stored before this + // policy existed. + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + let block_reason = match url::Url::parse(&endpoint_url) { + Ok(parsed_url) => policy.check_url(&parsed_url).err(), + Err(_) => Some(rivet_outbound_guard::BlockReason::InvalidUrl), + }; + if let Some(reason) = block_reason { + return Ok(blocked_destination(ctx, input, reason).await); + } + tracing::debug!(%endpoint_url, "sending outbound req"); - let client = rivet_pools::reqwest::client_no_timeout().await?; + let client = rivet_pools::reqwest::guarded_client_no_timeout(ctx.config()).await?; let req = client.get(endpoint_url).headers(headers); let conn_started = Instant::now(); @@ -411,11 +424,11 @@ async fn outbound_req_inner( _ => { let wrapped_err = anyhow::Error::from(err); - report_error( - ctx, - input.namespace_id, - &input.runner_name, - RunnerPoolError::ServerlessConnectionError { + let error = match rivet_outbound_guard::block_reason(&wrapped_err) { + Some(reason) => RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + None => RunnerPoolError::ServerlessConnectionError { // Print entire error chain message: wrapped_err .chain() @@ -423,8 +436,9 @@ async fn outbound_req_inner( .collect::>() .join("\n"), }, - ) - .await; + }; + + report_error(ctx, input.namespace_id, &input.runner_name, error).await; return Err(wrapped_err); } @@ -484,6 +498,34 @@ async fn outbound_req_inner( Ok(OutboundReqOutput::Draining { drain_sent: true }) } +/// Report a destination the policy refuses to dial and stop the connection loop. +/// +/// Retrying cannot help: the config has to change before this URL becomes reachable. +async fn blocked_destination( + ctx: &ActivityCtx, + input: &OutboundReqInput, + reason: rivet_outbound_guard::BlockReason, +) -> OutboundReqOutput { + tracing::warn!( + namespace_id = %input.namespace_id, + runner_name = %input.runner_name, + %reason, + "serverless url is not an allowed destination, ending outbound req" + ); + + report_error( + ctx, + input.namespace_id, + &input.runner_name, + RunnerPoolError::ServerlessDestinationBlocked { + reason: reason.to_string(), + }, + ) + .await; + + OutboundReqOutput::Draining { drain_sent: false } +} + /// Reads from the adjacent serverless runner wf which is keeping track of signals while this workflow runs /// outbound requests. #[tracing::instrument(skip_all)] diff --git a/engine/packages/pools/Cargo.toml b/engine/packages/pools/Cargo.toml index 2dcf4390b7..00fe13a97d 100644 --- a/engine/packages/pools/Cargo.toml +++ b/engine/packages/pools/Cargo.toml @@ -17,6 +17,7 @@ hyper-util.workspace = true lazy_static.workspace = true reqwest.workspace = true rivet-config.workspace = true +rivet-outbound-guard.workspace = true rivet-metrics.workspace = true rivet-util.workspace = true rustls.workspace = true diff --git a/engine/packages/pools/src/pools.rs b/engine/packages/pools/src/pools.rs index e2a10d1ac9..0c3acf8c91 100644 --- a/engine/packages/pools/src/pools.rs +++ b/engine/packages/pools/src/pools.rs @@ -46,7 +46,8 @@ impl Pools { // Initialize here to avoid cold starts elsewhere crate::reqwest::client().await?; - crate::reqwest::client_no_timeout().await?; + crate::reqwest::guarded_client(pool.config()).await?; + crate::reqwest::guarded_client_no_timeout(pool.config()).await?; Ok(pool) } diff --git a/engine/packages/pools/src/reqwest.rs b/engine/packages/pools/src/reqwest.rs index 779c08e219..24179f2bca 100644 --- a/engine/packages/pools/src/reqwest.rs +++ b/engine/packages/pools/src/reqwest.rs @@ -1,10 +1,21 @@ +use std::sync::Arc; + +use anyhow::{Context, Result}; use reqwest::Client; +use rivet_outbound_guard::{GuardedResolver, Policy}; use tokio::sync::OnceCell; static CLIENT: OnceCell = OnceCell::const_new(); -static CLIENT_NO_TIMEOUT: OnceCell = OnceCell::const_new(); +static GUARDED_CLIENT: OnceCell = OnceCell::const_new(); +static GUARDED_CLIENT_NO_TIMEOUT: OnceCell = OnceCell::const_new(); +static OUTBOUND_POLICY: OnceCell> = OnceCell::const_new(); static CLIENT_USER_AGENT: &str = concat!("RivetEngine/", env!("CARGO_PKG_VERSION")); +/// Client for trusted destinations inside the engine network, such as peer datacenters and epoxy +/// replicas. +/// +/// Never use this for a URL that came from user configuration. Those go through +/// [`guarded_client`], which restricts what the request can reach. pub async fn client() -> Result { CLIENT .get_or_try_init(|| async { @@ -17,9 +28,53 @@ pub async fn client() -> Result { .cloned() } -pub async fn client_no_timeout() -> Result { - CLIENT_NO_TIMEOUT - .get_or_try_init(|| async { Client::builder().user_agent(CLIENT_USER_AGENT).build() }) +/// Client for destinations that come from user configuration, such as serverless runner URLs. +/// +/// The `outbound` security policy is enforced at DNS resolution time and on every redirect, so +/// these requests cannot be steered at services only reachable from inside the engine network. +pub async fn guarded_client(config: &rivet_config::Config) -> Result { + GUARDED_CLIENT + .get_or_try_init(|| async { + build_guarded_client(config, Some(std::time::Duration::from_secs(30))).await + }) + .await + .cloned() +} + +/// Same as [`guarded_client`] but without a request timeout, for long-lived streaming requests +/// such as the serverless SSE connection. +pub async fn guarded_client_no_timeout(config: &rivet_config::Config) -> Result { + GUARDED_CLIENT_NO_TIMEOUT + .get_or_try_init(|| async { build_guarded_client(config, None).await }) + .await + .cloned() +} + +async fn build_guarded_client( + config: &rivet_config::Config, + timeout: Option, +) -> Result { + let policy = outbound_policy(config).await?; + + let mut builder = Client::builder() + .user_agent(CLIENT_USER_AGENT) + .dns_resolver(Arc::new(GuardedResolver::new(policy.clone()))) + .redirect(rivet_outbound_guard::redirect_policy(policy)); + + if let Some(timeout) = timeout { + builder = builder.timeout(timeout); + } + + builder.build().context("failed building guarded client") +} + +/// The destination policy applied to every request to a user-configured URL. +/// +/// Callers use this to reject a URL at the point it is submitted, before it is ever stored. The +/// guarded clients apply the same policy again when they connect. +pub async fn outbound_policy(config: &rivet_config::Config) -> Result> { + OUTBOUND_POLICY + .get_or_try_init(|| async { Policy::from_config(config).map(Arc::new) }) .await .cloned() } diff --git a/engine/packages/types/src/actor/error.rs b/engine/packages/types/src/actor/error.rs index c8a5e94095..158fddc036 100644 --- a/engine/packages/types/src/actor/error.rs +++ b/engine/packages/types/src/actor/error.rs @@ -15,6 +15,9 @@ pub enum RunnerPoolError { /// Serverless: SSE connection or network error ServerlessConnectionError { message: String }, + /// Serverless: the configured URL is not a destination the engine is allowed to reach + ServerlessDestinationBlocked { reason: String }, + /// Serverless: Runner sent invalid payload ServerlessInvalidSsePayload { message: String, diff --git a/engine/packages/universaldb/src/utils/keys.rs b/engine/packages/universaldb/src/utils/keys.rs index e8c61ebe13..a6c4fb8300 100644 --- a/engine/packages/universaldb/src/utils/keys.rs +++ b/engine/packages/universaldb/src/utils/keys.rs @@ -158,4 +158,6 @@ define_keys! { (130, GENERATION, "generation"), (131, ENVOY_HASH_IDX, "envoy_hash_idx"), (132, VIRTUAL_NODES, "virtual_nodes"), + (133, WEBHOOK, "webhook"), + (134, DELIVERY, "delivery"), } diff --git a/engine/packages/webhook/Cargo.toml b/engine/packages/webhook/Cargo.toml new file mode 100644 index 0000000000..7efd875d91 --- /dev/null +++ b/engine/packages/webhook/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "webhook" +publish = false +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +epoxy-protocol.workspace = true +epoxy.workspace = true +futures-util.workspace = true +gas.workspace = true +namespace.workspace = true +reqwest.workspace = true +rivet-data.workspace = true +rivet-error.workspace = true +rivet-outbound-guard.workspace = true +rivet-pools.workspace = true +rivet-util.workspace = true +serde.workspace = true +serde_bare.workspace = true +serde_json.workspace = true +tracing.workspace = true +universaldb.workspace = true +url.workspace = true +uuid = { workspace = true, features = ["v7"] } +vbare.workspace = true diff --git a/engine/packages/webhook/src/errors.rs b/engine/packages/webhook/src/errors.rs new file mode 100644 index 0000000000..b4c5a41f34 --- /dev/null +++ b/engine/packages/webhook/src/errors.rs @@ -0,0 +1,57 @@ +use rivet_error::*; +use serde::{Deserialize, Serialize}; + +#[derive(RivetError, Debug, Deserialize, Serialize)] +#[error("webhook")] +pub enum Webhook { + #[error( + "invalid", + "Invalid webhook config.", + "Invalid webhook config: {reason}" + )] + Invalid { reason: String }, + #[error( + "conflict", + "Webhook config changed concurrently.", + "Webhook config was modified concurrently, please retry." + )] + Conflict, + #[error( + "delivery_failed", + "Webhook delivery failed.", + "Webhook delivery failed with status {status}." + )] + DeliveryFailed { status: u16 }, + #[error( + "request_failed", + "Webhook delivery request failed.", + "Webhook delivery request failed: {reason}" + )] + RequestFailed { reason: String }, + #[error( + "destination_blocked", + "Webhook destination is not allowed.", + "Webhook destination is not allowed: {reason}" + )] + DestinationBlocked { reason: String }, + #[error("not_found", "Webhook not found.", "Webhook not found.")] + NotFound, + #[error( + "delivery_not_found", + "Webhook delivery not found.", + "Webhook delivery not found." + )] + DeliveryNotFound, + #[error( + "delivery_not_retryable", + "Webhook delivery is not in a retryable state.", + "Webhook delivery is not in a retryable state; only failed deliveries can be retried." + )] + DeliveryNotRetryable, + #[error( + "event_type_not_allowed", + "Event type cannot be subscribed to by a webhook.", + "Event type {event_type} cannot be subscribed to by a webhook because it is too high-throughput." + )] + EventTypeNotAllowed { event_type: String }, +} diff --git a/engine/packages/webhook/src/keys.rs b/engine/packages/webhook/src/keys.rs new file mode 100644 index 0000000000..82fb13aa17 --- /dev/null +++ b/engine/packages/webhook/src/keys.rs @@ -0,0 +1,231 @@ +use anyhow::Result; +use gas::prelude::*; +use universaldb::prelude::*; +use vbare::OwnedVersionedData; + +use crate::types::{DeliveryRecord, WebhookConfig}; + +fn serialize_config(value: WebhookConfig) -> Result> { + rivet_data::versioned::WebhookConfigData::wrap_latest(value.into()) + .serialize_with_embedded_version(rivet_data::WEBHOOK_CONFIG_VERSION) +} + +fn deserialize_config(raw: &[u8]) -> Result { + Ok(rivet_data::versioned::WebhookConfigData::deserialize_with_embedded_version(raw)?.into()) +} + +fn serialize_delivery(value: DeliveryRecord) -> Result> { + rivet_data::versioned::WebhookDeliveryData::wrap_latest(value.into()) + .serialize_with_embedded_version(rivet_data::WEBHOOK_DELIVERY_VERSION) +} + +fn deserialize_delivery(raw: &[u8]) -> Result { + Ok(rivet_data::versioned::WebhookDeliveryData::deserialize_with_embedded_version(raw)?.into()) +} + +#[derive(Debug)] +pub struct GlobalDataKey { + pub namespace_id: Id, + pub name: String, +} + +impl GlobalDataKey { + pub fn new(namespace_id: Id, name: String) -> Self { + GlobalDataKey { namespace_id, name } + } +} + +impl FormalKey for GlobalDataKey { + type Value = WebhookConfig; + + fn deserialize(&self, raw: &[u8]) -> Result { + deserialize_config(raw) + } + + fn serialize(&self, value: Self::Value) -> Result> { + serialize_config(value) + } +} + +impl TuplePack for GlobalDataKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = (WEBHOOK, CONFIG, GLOBAL, DATA, self.namespace_id, &self.name); + t.pack(w, tuple_depth) + } +} + +impl<'de> TupleUnpack<'de> for GlobalDataKey { + fn unpack(input: &[u8], tuple_depth: TupleDepth) -> PackResult<(&[u8], Self)> { + let (input, (_, _, _, _, namespace_id, name)) = + <(usize, usize, usize, usize, Id, String)>::unpack(input, tuple_depth)?; + + let v = GlobalDataKey { namespace_id, name }; + + Ok((input, v)) + } +} + +#[derive(Debug)] +pub struct DataKey { + pub namespace_id: Id, + pub name: String, +} + +impl DataKey { + pub fn new(namespace_id: Id, name: String) -> Self { + DataKey { namespace_id, name } + } + + pub fn subspace(namespace_id: Id) -> DataSubspaceKey { + DataSubspaceKey::new(namespace_id) + } +} + +impl FormalKey for DataKey { + type Value = WebhookConfig; + + fn deserialize(&self, raw: &[u8]) -> Result { + deserialize_config(raw) + } + + fn serialize(&self, value: Self::Value) -> Result> { + serialize_config(value) + } +} + +impl TuplePack for DataKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = (WEBHOOK, CONFIG, DATA, self.namespace_id, &self.name); + t.pack(w, tuple_depth) + } +} + +impl<'de> TupleUnpack<'de> for DataKey { + fn unpack(input: &[u8], tuple_depth: TupleDepth) -> PackResult<(&[u8], Self)> { + let (input, (_, _, _, namespace_id, name)) = + <(usize, usize, usize, Id, String)>::unpack(input, tuple_depth)?; + + let v = DataKey { namespace_id, name }; + + Ok((input, v)) + } +} + +#[derive(Debug)] +pub struct DataSubspaceKey { + pub namespace_id: Id, +} + +impl DataSubspaceKey { + pub fn new(namespace_id: Id) -> Self { + DataSubspaceKey { namespace_id } + } +} + +impl TuplePack for DataSubspaceKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = (WEBHOOK, CONFIG, DATA, self.namespace_id); + t.pack(w, tuple_depth) + } +} + +#[derive(Debug)] +pub struct DeliveryKey { + pub namespace_id: Id, + pub name: String, + pub delivery_id: String, +} + +impl DeliveryKey { + pub fn new(namespace_id: Id, name: String, delivery_id: String) -> Self { + DeliveryKey { + namespace_id, + name, + delivery_id, + } + } + + pub fn subspace(namespace_id: Id, name: String) -> DeliverySubspaceKey { + DeliverySubspaceKey::new(namespace_id, name) + } +} + +impl FormalKey for DeliveryKey { + type Value = DeliveryRecord; + + fn deserialize(&self, raw: &[u8]) -> Result { + deserialize_delivery(raw) + } + + fn serialize(&self, value: Self::Value) -> Result> { + serialize_delivery(value) + } +} + +impl TuplePack for DeliveryKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = ( + WEBHOOK, + DELIVERY, + DATA, + self.namespace_id, + &self.name, + &self.delivery_id, + ); + t.pack(w, tuple_depth) + } +} + +impl<'de> TupleUnpack<'de> for DeliveryKey { + fn unpack(input: &[u8], tuple_depth: TupleDepth) -> PackResult<(&[u8], Self)> { + let (input, (_, _, _, namespace_id, name, delivery_id)) = + <(usize, usize, usize, Id, String, String)>::unpack(input, tuple_depth)?; + + let v = DeliveryKey { + namespace_id, + name, + delivery_id, + }; + + Ok((input, v)) + } +} + +#[derive(Debug)] +pub struct DeliverySubspaceKey { + pub namespace_id: Id, + pub name: String, +} + +impl DeliverySubspaceKey { + pub fn new(namespace_id: Id, name: String) -> Self { + DeliverySubspaceKey { namespace_id, name } + } +} + +impl TuplePack for DeliverySubspaceKey { + fn pack( + &self, + w: &mut W, + tuple_depth: TupleDepth, + ) -> std::io::Result { + let t = (WEBHOOK, DELIVERY, DATA, self.namespace_id, &self.name); + t.pack(w, tuple_depth) + } +} diff --git a/engine/packages/webhook/src/lib.rs b/engine/packages/webhook/src/lib.rs new file mode 100644 index 0000000000..c93634f6f5 --- /dev/null +++ b/engine/packages/webhook/src/lib.rs @@ -0,0 +1,16 @@ +use gas::prelude::*; + +pub mod errors; +pub mod keys; +pub mod ops; +pub mod types; +pub mod workflows; + +pub fn registry() -> WorkflowResult { + use workflows::*; + + let mut registry = Registry::new(); + registry.register_workflow::()?; + + Ok(registry) +} diff --git a/engine/packages/webhook/src/ops/delete.rs b/engine/packages/webhook/src/ops/delete.rs new file mode 100644 index 0000000000..d73bf2887c --- /dev/null +++ b/engine/packages/webhook/src/ops/delete.rs @@ -0,0 +1,77 @@ +use epoxy::ops::propose::{ + CheckAndSetCommand, Command, CommandKind, ConsensusFailedReason, Proposal, ProposalResult, +}; +use epoxy_protocol::protocol::CachingBehavior; +use gas::prelude::*; + +use crate::{errors, keys, workflows}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, +} + +#[operation] +pub async fn webhook_config_delete(ctx: &OperationCtx, input: &Input) -> Result<()> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + + let global_key = keys::GlobalDataKey::new(namespace_id, name.clone()); + + let current = ctx + .op(epoxy::ops::kv::get_optimistic::Input { + replica_id: ctx.config().epoxy_replica_id(), + key: namespace::keys::subspace().pack(&global_key), + caching_behavior: CachingBehavior::Optimistic, + target_replicas: None, + save_empty: false, + }) + .await? + .value; + + let propose_res = ctx + .op(epoxy::ops::propose::Input { + proposal: Proposal { + commands: vec![Command { + kind: CommandKind::CheckAndSetCommand(CheckAndSetCommand { + key: namespace::keys::subspace().pack(&global_key), + expect_one_of: vec![current], // TODO: verify CAS is implemented. Doesn't work with epoxyV2. Implement(?). Fails tests currently + new_value: None, + }), + }], + }, + purge_cache: true, + mutable: true, + target_replicas: None, + }) + .await?; + + match propose_res { + ProposalResult::Committed => {} + ProposalResult::ConsensusFailed { reason } => match reason { + ConsensusFailedReason::ExpectedValueDoesNotMatch { .. } => { + return Err(errors::Webhook::Conflict.build()); + } + ConsensusFailedReason::PreparePhaseConsensusFailed => { + bail!("epoxy propose failed: prepare phase consensus failed"); + } + ConsensusFailedReason::AcceptPhaseConsensusFailed => { + bail!("epoxy propose failed: accept phase consensus failed"); + } + ConsensusFailedReason::StaleBallot => { + bail!("epoxy propose failed: stale ballot"); + } + }, + } + + ctx.signal(workflows::webhook::Destroy {}) + .to_workflow::() + .tag("namespace_id", input.namespace_id) + .tag("name", input.name.clone()) + .graceful_not_found() + .send() + .await?; + + Ok(()) +} diff --git a/engine/packages/webhook/src/ops/get.rs b/engine/packages/webhook/src/ops/get.rs new file mode 100644 index 0000000000..c9a199f4a6 --- /dev/null +++ b/engine/packages/webhook/src/ops/get.rs @@ -0,0 +1,33 @@ +use epoxy_protocol::protocol::CachingBehavior; +use gas::prelude::*; +use universaldb::prelude::FormalKey; + +use crate::{keys, types::WebhookConfig}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, +} + +#[operation] +pub async fn webhook_config_get( + ctx: &OperationCtx, + input: &Input, +) -> Result> { + let global_key = keys::GlobalDataKey::new(input.namespace_id, input.name.clone()); + + let res = ctx + .op(epoxy::ops::kv::get_optimistic::Input { + replica_id: ctx.config().epoxy_replica_id(), + key: namespace::keys::subspace().pack(&global_key), + caching_behavior: CachingBehavior::Optimistic, + target_replicas: None, + save_empty: false, + }) + .await?; + + res.value + .map(|raw| global_key.deserialize(&raw)) + .transpose() +} diff --git a/engine/packages/webhook/src/ops/list.rs b/engine/packages/webhook/src/ops/list.rs new file mode 100644 index 0000000000..6a8d8e494f --- /dev/null +++ b/engine/packages/webhook/src/ops/list.rs @@ -0,0 +1,56 @@ +use futures_util::{StreamExt, TryStreamExt}; +use gas::prelude::*; +use universaldb::options::StreamingMode; +use universaldb::utils::IsolationLevel::*; + +use crate::{keys, types::WebhookConfig}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, +} + +#[derive(Debug)] +pub struct Webhook { + pub name: String, + pub config: WebhookConfig, +} + +#[operation] +pub async fn webhook_config_list(ctx: &OperationCtx, input: &Input) -> Result> { + // Would be better to read from epoxy for consistency, find a way to list epoxy + let webhooks = ctx + .udb()? + .txn("webhook_config_list", |tx| async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + + let (start, end) = namespace::keys::subspace() + .subspace(&keys::DataKey::subspace(input.namespace_id)) + .range(); + + tx.get_ranges_keyvalues( + universaldb::RangeOption { + mode: StreamingMode::WantAll, + ..(start, end).into() + }, + Serializable, + ) + .map(|res| { + let tx = tx.clone(); + async move { + let entry = res?; + let (key, config) = tx.read_entry::(&entry)?; + Ok(Webhook { + name: key.name, + config, + }) + } + }) + .buffer_unordered(16) + .try_collect::>() + .await + }) + .await?; + + Ok(webhooks) +} diff --git a/engine/packages/webhook/src/ops/list_deliveries.rs b/engine/packages/webhook/src/ops/list_deliveries.rs new file mode 100644 index 0000000000..b2aa8d4152 --- /dev/null +++ b/engine/packages/webhook/src/ops/list_deliveries.rs @@ -0,0 +1,62 @@ +use futures_util::{StreamExt, TryStreamExt}; +use gas::prelude::*; +use universaldb::options::StreamingMode; +use universaldb::utils::IsolationLevel::*; + +use crate::{keys, types::DeliveryRecord}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, +} + +#[derive(Debug)] +pub struct Delivery { + pub delivery_id: String, + pub record: DeliveryRecord, +} + +#[operation] +pub async fn webhook_delivery_list(ctx: &OperationCtx, input: &Input) -> Result> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + + let deliveries = ctx + .udb()? + .txn("webhook_delivery_list", move |tx| { + let name = name.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + + let (start, end) = namespace::keys::subspace() + .subspace(&keys::DeliveryKey::subspace(namespace_id, name)) + .range(); + + tx.get_ranges_keyvalues( + universaldb::RangeOption { + mode: StreamingMode::WantAll, + ..(start, end).into() + }, + Serializable, + ) + .map(|res| { + let tx = tx.clone(); + async move { + let entry = res?; + let (key, record) = tx.read_entry::(&entry)?; + Ok(Delivery { + delivery_id: key.delivery_id, + record, + }) + } + }) + .buffer_unordered(16) + .try_collect::>() + .await + } + }) + .await?; + + Ok(deliveries) +} diff --git a/engine/packages/webhook/src/ops/mod.rs b/engine/packages/webhook/src/ops/mod.rs new file mode 100644 index 0000000000..42a0d49e49 --- /dev/null +++ b/engine/packages/webhook/src/ops/mod.rs @@ -0,0 +1,6 @@ +pub mod delete; +pub mod get; +pub mod list; +pub mod list_deliveries; +pub mod retry; +pub mod upsert; diff --git a/engine/packages/webhook/src/ops/retry.rs b/engine/packages/webhook/src/ops/retry.rs new file mode 100644 index 0000000000..d0fedfe0cf --- /dev/null +++ b/engine/packages/webhook/src/ops/retry.rs @@ -0,0 +1,59 @@ +use gas::prelude::*; +use universaldb::utils::IsolationLevel::*; + +use crate::{errors, keys, types::DeliveryStatus, workflows}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, + pub delivery_id: String, +} + +#[operation] +pub async fn webhook_delivery_retry(ctx: &OperationCtx, input: &Input) -> Result<()> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + let delivery_id = input.delivery_id.clone(); + + let record = ctx + .udb()? + .txn("webhook_delivery_retry_read", { + let name = name.clone(); + let delivery_id = delivery_id.clone(); + move |tx| { + let name = name.clone(); + let delivery_id = delivery_id.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + tx.read_opt( + &keys::DeliveryKey::new(namespace_id, name, delivery_id), + Serializable, + ) + .await + } + } + }) + .await?; + + let record = record.ok_or_else(|| errors::Webhook::DeliveryNotFound.build())?; + + if !matches!(record.status, DeliveryStatus::Failed) { + return Err(errors::Webhook::DeliveryNotRetryable.build()); + } + + let signal_res = ctx + .signal(workflows::webhook::Retry { delivery_id }) + .to_workflow::() + .tag("namespace_id", namespace_id) + .tag("name", name) + .graceful_not_found() + .send() + .await?; + + if signal_res.is_none() { + return Err(errors::Webhook::NotFound.build()); + } + + Ok(()) +} diff --git a/engine/packages/webhook/src/ops/upsert.rs b/engine/packages/webhook/src/ops/upsert.rs new file mode 100644 index 0000000000..d2ee7a5049 --- /dev/null +++ b/engine/packages/webhook/src/ops/upsert.rs @@ -0,0 +1,144 @@ +use epoxy::ops::propose::{ + CheckAndSetCommand, Command, CommandKind, ConsensusFailedReason, Proposal, ProposalResult, +}; +use gas::prelude::*; +use universaldb::prelude::FormalKey; + +use crate::{errors, keys, types::WebhookConfig, workflows}; + +#[derive(Debug)] +pub struct Input { + pub namespace_id: Id, + pub name: String, + pub config: WebhookConfig, +} + +#[operation] +pub async fn webhook_config_upsert(ctx: &OperationCtx, input: &Input) -> Result<()> { + validate(ctx, &input.config).await?; + + let global_key = keys::GlobalDataKey::new(input.namespace_id, input.name.clone()); + + let propose_res = ctx + .op(epoxy::ops::propose::Input { + proposal: Proposal { + commands: vec![Command { + kind: CommandKind::CheckAndSetCommand(CheckAndSetCommand { + key: namespace::keys::subspace().pack(&global_key), + expect_one_of: vec![None], + new_value: Some(global_key.serialize(input.config.clone())?), + }), + }], + }, + purge_cache: true, + mutable: true, + target_replicas: None, + }) + .await?; + + match propose_res { + ProposalResult::Committed => {} + ProposalResult::ConsensusFailed { reason } => match reason { + ConsensusFailedReason::ExpectedValueDoesNotMatch { .. } => { + return Err(errors::Webhook::Conflict.build()); + } + ConsensusFailedReason::PreparePhaseConsensusFailed => { + bail!("epoxy propose failed: prepare phase consensus failed"); + } + ConsensusFailedReason::AcceptPhaseConsensusFailed => { + bail!("epoxy propose failed: accept phase consensus failed"); + } + ConsensusFailedReason::StaleBallot => { + bail!("epoxy propose failed: stale ballot"); + } + }, + } + + let signal_res = ctx + .signal(workflows::webhook::Update { + config: input.config.clone(), + }) + .to_workflow::() + .tag("namespace_id", input.namespace_id) + .tag("name", input.name.clone()) + .graceful_not_found() + .send() + .await?; + + if signal_res.is_none() { + ctx.workflow(workflows::webhook::Input { + namespace_id: input.namespace_id, + name: input.name.clone(), + config: input.config.clone(), + }) + .tag("namespace_id", input.namespace_id) + .tag("name", input.name.clone()) + .unique() + .dispatch() + .await?; + } + + Ok(()) +} + +async fn validate(ctx: &OperationCtx, config: &WebhookConfig) -> Result<()> { + let parsed_url = url::Url::parse(&config.url).map_err(|err| { + errors::Webhook::Invalid { + reason: format!("invalid url: {err}"), + } + .build() + })?; + + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { + return Err(errors::Webhook::Invalid { + reason: format!("invalid url: {reason}"), + } + .build()); + } + + for event_type in &config.subscriptions { + if !event_type.is_webhook_safe() { + return Err(errors::Webhook::EventTypeNotAllowed { + event_type: event_type.as_str().to_string(), + } + .build()); + } + } + + if config.headers.len() > 16 { + return Err(errors::Webhook::Invalid { + reason: "too many headers (max 16)".to_string(), + } + .build()); + } + + for (name, value) in &config.headers { + if name.len() > 128 { + return Err(errors::Webhook::Invalid { + reason: "invalid header name: too long (max 128)".to_string(), + } + .build()); + } + if let Err(err) = name.parse::() { + return Err(errors::Webhook::Invalid { + reason: format!("invalid header name: {err}"), + } + .build()); + } + if value.len() > 4096 { + return Err(errors::Webhook::Invalid { + reason: "invalid header value: too long (max 4096)".to_string(), + } + .build()); + } + if let Err(err) = value.parse::() { + return Err(errors::Webhook::Invalid { + reason: format!("invalid header value: {err}"), + } + .build()); + } + } + + Ok(()) +} diff --git a/engine/packages/webhook/src/types.rs b/engine/packages/webhook/src/types.rs new file mode 100644 index 0000000000..3890d40b85 --- /dev/null +++ b/engine/packages/webhook/src/types.rs @@ -0,0 +1,218 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum WebhookEventType { + #[serde(rename = "runner_pool.error")] + RunnerPoolError, + #[serde(rename = "runner_pool.healthy")] + RunnerPoolHealthy, + #[serde(rename = "actor.http_request")] + ActorHttpRequest, +} + +impl WebhookEventType { + pub fn is_webhook_safe(&self) -> bool { + match self { + WebhookEventType::RunnerPoolError => true, + WebhookEventType::RunnerPoolHealthy => true, + WebhookEventType::ActorHttpRequest => false, + } + } + + pub fn as_cloudevents_type(&self) -> &'static str { + match self { + WebhookEventType::RunnerPoolError => "dev.rivet.runner_pool.error", + WebhookEventType::RunnerPoolHealthy => "dev.rivet.runner_pool.healthy", + WebhookEventType::ActorHttpRequest => "dev.rivet.actor.http_request", + } + } + + pub fn as_str(&self) -> &'static str { + match self { + WebhookEventType::RunnerPoolError => "runner_pool.error", + WebhookEventType::RunnerPoolHealthy => "runner_pool.healthy", + WebhookEventType::ActorHttpRequest => "actor.http_request", + } + } +} + +impl From for WebhookEventType { + fn from(value: rivet_data::generated::webhook_config_v1::WebhookEventType) -> Self { + match value { + rivet_data::generated::webhook_config_v1::WebhookEventType::RunnerPoolError => { + WebhookEventType::RunnerPoolError + } + rivet_data::generated::webhook_config_v1::WebhookEventType::RunnerPoolHealthy => { + WebhookEventType::RunnerPoolHealthy + } + rivet_data::generated::webhook_config_v1::WebhookEventType::ActorHttpRequest => { + WebhookEventType::ActorHttpRequest + } + } + } +} + +impl From for rivet_data::generated::webhook_config_v1::WebhookEventType { + fn from(value: WebhookEventType) -> Self { + match value { + WebhookEventType::RunnerPoolError => { + rivet_data::generated::webhook_config_v1::WebhookEventType::RunnerPoolError + } + WebhookEventType::RunnerPoolHealthy => { + rivet_data::generated::webhook_config_v1::WebhookEventType::RunnerPoolHealthy + } + WebhookEventType::ActorHttpRequest => { + rivet_data::generated::webhook_config_v1::WebhookEventType::ActorHttpRequest + } + } + } +} + +impl From for WebhookEventType { + fn from(value: rivet_data::generated::webhook_delivery_v1::WebhookEventType) -> Self { + match value { + rivet_data::generated::webhook_delivery_v1::WebhookEventType::RunnerPoolError => { + WebhookEventType::RunnerPoolError + } + rivet_data::generated::webhook_delivery_v1::WebhookEventType::RunnerPoolHealthy => { + WebhookEventType::RunnerPoolHealthy + } + rivet_data::generated::webhook_delivery_v1::WebhookEventType::ActorHttpRequest => { + WebhookEventType::ActorHttpRequest + } + } + } +} + +impl From for rivet_data::generated::webhook_delivery_v1::WebhookEventType { + fn from(value: WebhookEventType) -> Self { + match value { + WebhookEventType::RunnerPoolError => { + rivet_data::generated::webhook_delivery_v1::WebhookEventType::RunnerPoolError + } + WebhookEventType::RunnerPoolHealthy => { + rivet_data::generated::webhook_delivery_v1::WebhookEventType::RunnerPoolHealthy + } + WebhookEventType::ActorHttpRequest => { + rivet_data::generated::webhook_delivery_v1::WebhookEventType::ActorHttpRequest + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct WebhookConfig { + pub url: String, + pub headers: HashMap, + pub subscriptions: Vec, +} + +impl From for WebhookConfig { + fn from(value: rivet_data::generated::webhook_config_v1::Data) -> Self { + WebhookConfig { + url: value.url, + headers: value.headers, + subscriptions: value.subscriptions.into_iter().map(Into::into).collect(), + } + } +} + +impl From for rivet_data::generated::webhook_config_v1::Data { + fn from(value: WebhookConfig) -> Self { + rivet_data::generated::webhook_config_v1::Data { + url: value.url, + headers: value.headers, + subscriptions: value.subscriptions.into_iter().map(Into::into).collect(), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum DeliveryStatus { + Pending, + Succeeded, + Failed, +} + +impl From for DeliveryStatus { + fn from(value: rivet_data::generated::webhook_delivery_v1::DeliveryStatus) -> Self { + match value { + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Pending => { + DeliveryStatus::Pending + } + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Succeeded => { + DeliveryStatus::Succeeded + } + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Failed => { + DeliveryStatus::Failed + } + } + } +} + +impl From for rivet_data::generated::webhook_delivery_v1::DeliveryStatus { + fn from(value: DeliveryStatus) -> Self { + match value { + DeliveryStatus::Pending => { + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Pending + } + DeliveryStatus::Succeeded => { + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Succeeded + } + DeliveryStatus::Failed => { + rivet_data::generated::webhook_delivery_v1::DeliveryStatus::Failed + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WebhookEvent { + pub event_type: WebhookEventType, + /// CloudEvents `subject`: which resource within the source the event is about, such as the + /// runner name for a runner pool event. + pub subject: Option, + /// CloudEvents `data`. + pub data: serde_json::Value, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct DeliveryRecord { + pub payload: String, + pub status: DeliveryStatus, + pub attempt_count: u32, + pub last_error: Option, + pub created_at: i64, + pub event_type: WebhookEventType, + pub subject: Option, +} + +impl From for DeliveryRecord { + fn from(value: rivet_data::generated::webhook_delivery_v1::Data) -> Self { + DeliveryRecord { + payload: value.payload, + status: value.status.into(), + attempt_count: value.attempt_count, + last_error: value.last_error, + created_at: value.created_at, + event_type: value.event_type.into(), + subject: value.subject, + } + } +} + +impl From for rivet_data::generated::webhook_delivery_v1::Data { + fn from(value: DeliveryRecord) -> Self { + rivet_data::generated::webhook_delivery_v1::Data { + payload: value.payload, + status: value.status.into(), + attempt_count: value.attempt_count, + last_error: value.last_error, + created_at: value.created_at, + event_type: value.event_type.into(), + subject: value.subject, + } + } +} diff --git a/engine/packages/webhook/src/workflows/mod.rs b/engine/packages/webhook/src/workflows/mod.rs new file mode 100644 index 0000000000..25f2d33efd --- /dev/null +++ b/engine/packages/webhook/src/workflows/mod.rs @@ -0,0 +1 @@ +pub mod webhook; diff --git a/engine/packages/webhook/src/workflows/webhook.rs b/engine/packages/webhook/src/workflows/webhook.rs new file mode 100644 index 0000000000..e61ee8ddaf --- /dev/null +++ b/engine/packages/webhook/src/workflows/webhook.rs @@ -0,0 +1,533 @@ +use std::time::Duration; + +use futures_util::FutureExt; +use gas::prelude::*; +use serde::{Deserialize, Serialize}; +use universaldb::utils::IsolationLevel::*; +use uuid::Uuid; + +use crate::{ + errors, keys, + types::{DeliveryRecord, DeliveryStatus, WebhookConfig, WebhookEvent, WebhookEventType}, +}; + +#[derive(Debug, Deserialize, Serialize)] +pub struct Input { + pub namespace_id: Id, + pub name: String, + pub config: WebhookConfig, +} + +#[workflow] +pub async fn webhook(ctx: &mut WorkflowCtx, input: &Input) -> Result<()> { + tracing::debug!( + namespace_id = %input.namespace_id, + name = %input.name, + "starting webhook workflow" + ); + + let namespace_id = input.namespace_id; + let name = input.name.clone(); + + ctx.loope(input.config.clone(), move |ctx, config| { + let name = name.clone(); + async move { + match ctx.listen::
().await? { + Main::Update(sig) => { + *config = sig.config; + } + Main::Trigger(sig) => { + if !config.subscriptions.contains(&sig.event.event_type) { + tracing::debug!( + event_type = sig.event.event_type.as_str(), + "dropping trigger for unsubscribed event type" + ); + return Ok(Loop::Continue); + } + + let delivery_id = ctx.activity(GenerateDeliveryIdInput {}).await?; + + let outcome = deliver_with_retries( + ctx, + namespace_id, + name.clone(), + delivery_id, + config.clone(), + sig.event, + ) + .await?; + + if let DeliveryOutcome::Destroyed = outcome { + return Ok(Loop::Break(())); + } + } + Main::Retry(sig) => { + let record = ctx + .activity(GetDeliveryInput { + namespace_id, + name: name.clone(), + delivery_id: sig.delivery_id.clone(), + }) + .await?; + + let Some(record) = record else { + tracing::warn!( + delivery_id = %sig.delivery_id, + "retry requested for unknown delivery" + ); + return Ok(Loop::Continue); + }; + + if !matches!(record.status, DeliveryStatus::Failed) { + tracing::warn!( + delivery_id = %sig.delivery_id, + status = ?record.status, + "retry requested for delivery not in a failed state" + ); + return Ok(Loop::Continue); + } + + let data = match serde_json::from_str(&record.payload) { + Ok(data) => data, + Err(err) => { + tracing::warn!( + delivery_id = %sig.delivery_id, + ?err, + "stored delivery payload is not valid json" + ); + return Ok(Loop::Continue); + } + }; + + let outcome = deliver_with_retries( + ctx, + namespace_id, + name.clone(), + sig.delivery_id, + config.clone(), + WebhookEvent { + event_type: record.event_type, + subject: record.subject, + data, + }, + ) + .await?; + + if let DeliveryOutcome::Destroyed = outcome { + return Ok(Loop::Break(())); + } + } + Main::Destroy(_) => { + return Ok(Loop::Break(())); + } + } + + Ok(Loop::<()>::Continue) + } + .boxed() + }) + .await?; + + Ok(()) +} + +enum DeliveryOutcome { + Done, + Destroyed, +} + +// Runs (or re-runs, for a manual `Retry`) the full attempt loop for one delivery: records it as +// `Pending`, attempts delivery with exponential backoff up to `MAX_DELIVERY_ATTEMPTS`, and records +// the terminal `Succeeded`/`Failed` outcome. Shared by `Trigger` (a brand new delivery) and +// `Retry` (re-attempting a stored, already-failed delivery), since both just need to run this +// same loop against a `delivery_id` and `payload`. +async fn deliver_with_retries( + ctx: &mut WorkflowCtx, + namespace_id: Id, + name: String, + delivery_id: String, + config: WebhookConfig, + event: WebhookEvent, +) -> Result { + let WebhookEvent { + event_type, + subject, + data, + } = event; + + let payload = serde_json::to_string(&data).context("event data is not serializable")?; + // Also yields the delivery's `created_at`, which the CloudEvents envelope needs. On a manual + // `Retry` this is the original trigger time, preserved by the activity. + let created_at = ctx + .activity(RecordDeliveryInput { + namespace_id, + name: name.clone(), + delivery_id: delivery_id.clone(), + payload: payload.clone(), + status: DeliveryStatus::Pending, + attempt_count: 0, + last_error: None, + event_type, + subject: subject.clone(), + }) + .await?; + + let mut attempt = 0; + + loop { + let deliver_res = ctx + .activity(DeliverInput { + delivery_id: delivery_id.clone(), + namespace_id, + name: name.clone(), + config: config.clone(), + data: data.clone(), + event_type, + subject: subject.clone(), + created_at, + }) + .await?; + + match deliver_res.error { + None => { + ctx.activity(RecordDeliveryInput { + namespace_id, + name: name.clone(), + delivery_id: delivery_id.clone(), + payload, + status: DeliveryStatus::Succeeded, + attempt_count: attempt + 1, + last_error: None, + event_type, + subject, + }) + .await?; + + return Ok(DeliveryOutcome::Done); + } + Some(error) if is_retryable(&error) && attempt + 1 < MAX_DELIVERY_ATTEMPTS => { + attempt += 1; + + let destroy_sig = ctx + .listen_with_timeout::(next_delivery_delay( + deliver_res.retry_after_ms, + attempt, + )) + .await?; + + if destroy_sig.is_some() { + return Ok(DeliveryOutcome::Destroyed); + } + } + Some(error) => { + tracing::warn!(?error, attempt, "webhook delivery failed permanently"); + + ctx.activity(RecordDeliveryInput { + namespace_id, + name: name.clone(), + delivery_id: delivery_id.clone(), + payload, + status: DeliveryStatus::Failed, + attempt_count: attempt + 1, + last_error: Some(error.build().to_string()), + event_type, + subject, + }) + .await?; + + return Ok(DeliveryOutcome::Done); + } + } + } +} + +// The maximum number of delivery attempts for a single triggered event before giving up to DLQ. +const MAX_DELIVERY_ATTEMPTS: u32 = 5; + +const MIN_RETRY_AFTER: Duration = Duration::from_secs(1); +const MAX_RETRY_AFTER: Duration = Duration::from_secs(300); + +fn is_retryable(error: &errors::Webhook) -> bool { + match error { + errors::Webhook::RequestFailed { .. } => true, + errors::Webhook::DeliveryFailed { status } => matches!(status, 429 | 500..=599), + errors::Webhook::Invalid { .. } + | errors::Webhook::Conflict + | errors::Webhook::DestinationBlocked { .. } + | errors::Webhook::NotFound + | errors::Webhook::DeliveryNotFound + | errors::Webhook::DeliveryNotRetryable + | errors::Webhook::EventTypeNotAllowed { .. } => false, + } +} + +// Exponential backoff starting at 5s, doubling each attempt, capped at 5m. +fn delivery_backoff(attempt: u32) -> Duration { + Duration::from_secs(5u64.saturating_mul(1u64 << attempt.min(6)).min(300)) +} + +// How long to wait before the next attempt. A destination that told us how long to wait wins over +// our own backoff, using 429's +fn next_delivery_delay(retry_after_ms: Option, attempt: u32) -> Duration { + match retry_after_ms { + Some(ms) => Duration::from_millis(ms).clamp(MIN_RETRY_AFTER, MAX_RETRY_AFTER), + None => delivery_backoff(attempt), + } +} + +// `Retry-After` is either delta-seconds or an HTTP-date (RFC 9110). An HTTP-date in the past +// yields a zero delay, which the caller clamps. +fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option { + let raw = headers.get(reqwest::header::RETRY_AFTER)?.to_str().ok()?; + + if let Ok(seconds) = raw.trim().parse::() { + return Some(seconds.saturating_mul(1_000)); + } + + let deadline = chrono::DateTime::parse_from_rfc2822(raw.trim()).ok()?; + let delta = deadline.timestamp_millis() - chrono::Utc::now().timestamp_millis(); + + Some(delta.max(0) as u64) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GenerateDeliveryIdInput {} + +#[activity(GenerateDeliveryId)] +pub async fn generate_delivery_id( + _ctx: &ActivityCtx, + _input: &GenerateDeliveryIdInput, +) -> Result { + Ok(Uuid::new_v4().to_string()) +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DeliverInput { + pub delivery_id: String, + pub namespace_id: Id, + pub name: String, + pub config: WebhookConfig, + pub data: serde_json::Value, + pub event_type: WebhookEventType, + pub subject: Option, + pub created_at: i64, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct DeliverOutput { + pub error: Option, + pub retry_after_ms: Option, +} + +#[derive(Debug, Serialize)] +struct CloudEvent<'a> { + id: String, + source: &'a str, + specversion: &'static str, + #[serde(rename = "type")] + kind: &'static str, + time: String, + #[serde(skip_serializing_if = "Option::is_none")] + subject: Option<&'a str>, + datacontenttype: &'static str, + data: &'a serde_json::Value, +} + +#[activity(Deliver)] +pub async fn deliver(ctx: &ActivityCtx, input: &DeliverInput) -> Result { + let parsed_url = + url::Url::parse(&input.config.url).context("stored webhook url is not parseable")?; + + let policy = rivet_pools::reqwest::outbound_policy(ctx.config()).await?; + if let Err(reason) = policy.check_url(&parsed_url) { + return Ok(DeliverOutput { + error: Some(errors::Webhook::DestinationBlocked { + reason: reason.to_string(), + }), + retry_after_ms: None, + }); + } + + // The occurrence time, not the transmission time, so every attempt at one delivery carries + // the same value. Receivers dedupe on `id` plus `source` and would otherwise see the same + // event reported as having happened at several different times. Follows Cloudevents standards. + let occurred_at = chrono::DateTime::from_timestamp_millis(input.created_at) + .context("delivery created_at is not a valid timestamp")? + .to_rfc3339(); + + let event = CloudEvent { + id: input.delivery_id.clone(), + source: &format!("rivet:webhook:{}:{}", input.namespace_id, input.name), + specversion: "1.0", + kind: input.event_type.as_cloudevents_type(), + time: occurred_at, + subject: input.subject.as_deref(), + datacontenttype: "application/json", + data: &input.data, + }; + + let client = rivet_pools::reqwest::guarded_client(ctx.config()).await?; + let mut req = client + .post(parsed_url) + .header("Content-Type", "application/cloudevents+json") + .json(&event); + + for (k, v) in &input.config.headers { + req = req.header(k, v); + } + + match req.send().await { + Ok(res) if res.status().is_success() => Ok(DeliverOutput { + error: None, + retry_after_ms: None, + }), + Ok(res) => { + let status = res.status().as_u16(); + + Ok(DeliverOutput { + error: Some(errors::Webhook::DeliveryFailed { status }), + retry_after_ms: parse_retry_after(res.headers()), + }) + } + Err(err) => { + let err = anyhow::Error::from(err); + + if let Some(reason) = rivet_outbound_guard::block_reason(&err) { + return Ok(DeliverOutput { + error: Some(errors::Webhook::DestinationBlocked { + reason: reason.to_string(), + }), + retry_after_ms: None, + }); + } + + Ok(DeliverOutput { + error: Some(errors::Webhook::RequestFailed { + reason: format!("{err:#}"), + }), + retry_after_ms: None, + }) + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RecordDeliveryInput { + pub namespace_id: Id, + pub name: String, + pub delivery_id: String, + pub payload: String, + pub status: DeliveryStatus, + pub attempt_count: u32, + pub last_error: Option, + pub event_type: WebhookEventType, + pub subject: Option, +} + +#[activity(RecordDelivery)] +pub async fn record_delivery(ctx: &ActivityCtx, input: &RecordDeliveryInput) -> Result { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + let delivery_id = input.delivery_id.clone(); + let payload = input.payload.clone(); + let status = input.status; + let attempt_count = input.attempt_count; + let last_error = input.last_error.clone(); + let event_type = input.event_type; + let subject = input.subject.clone(); + let now = ctx.ts(); + + ctx.udb()? + .txn("webhook_record_delivery", move |tx| { + let name = name.clone(); + let delivery_id = delivery_id.clone(); + let payload = payload.clone(); + let last_error = last_error.clone(); + let subject = subject.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + let key = keys::DeliveryKey::new(namespace_id, name, delivery_id); + + let created_at = match tx.read_opt(&key, Serializable).await? { + Some(existing) => existing.created_at, + None => now, + }; + + tx.write( + &key, + DeliveryRecord { + payload, + status, + attempt_count, + last_error, + created_at, + event_type, + subject, + }, + )?; + Ok(created_at) + } + }) + .await +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GetDeliveryInput { + pub namespace_id: Id, + pub name: String, + pub delivery_id: String, +} + +#[activity(GetDelivery)] +pub async fn get_delivery( + ctx: &ActivityCtx, + input: &GetDeliveryInput, +) -> Result> { + let namespace_id = input.namespace_id; + let name = input.name.clone(); + let delivery_id = input.delivery_id.clone(); + + ctx.udb()? + .txn("webhook_get_delivery", move |tx| { + let name = name.clone(); + let delivery_id = delivery_id.clone(); + async move { + let tx = tx.with_subspace(namespace::keys::subspace()); + tx.read_opt( + &keys::DeliveryKey::new(namespace_id, name, delivery_id), + Serializable, + ) + .await + } + }) + .await +} + +#[signal("webhook_trigger")] +pub struct Trigger { + pub event: WebhookEvent, +} + +#[signal("webhook_update")] +pub struct Update { + pub config: WebhookConfig, +} + +#[signal("webhook_destroy")] +pub struct Destroy {} + +#[signal("webhook_retry")] +pub struct Retry { + pub delivery_id: String, +} + +join_signal!(Main { + Trigger, + Update, + Destroy, + Retry, +}); + +#[cfg(test)] +#[path = "../../tests/inline/workflows_webhook.rs"] +mod tests; diff --git a/engine/packages/webhook/tests/inline/workflows_webhook.rs b/engine/packages/webhook/tests/inline/workflows_webhook.rs new file mode 100644 index 0000000000..586717dc5e --- /dev/null +++ b/engine/packages/webhook/tests/inline/workflows_webhook.rs @@ -0,0 +1,103 @@ +use std::time::Duration; + +use reqwest::header::{HeaderMap, HeaderValue, RETRY_AFTER}; + +use super::*; + +fn retry_after(value: &str) -> HeaderMap { + let mut headers = HeaderMap::new(); + headers.insert( + RETRY_AFTER, + HeaderValue::from_str(value).expect("invalid header value"), + ); + headers +} + +// MARK: is_retryable + +#[test] +fn retries_transient_failures() { + assert!(is_retryable(&errors::Webhook::RequestFailed { + reason: "connection refused".to_string(), + })); + assert!(is_retryable(&errors::Webhook::DeliveryFailed { + status: 429 + })); + assert!(is_retryable(&errors::Webhook::DeliveryFailed { + status: 500 + })); + assert!(is_retryable(&errors::Webhook::DeliveryFailed { + status: 503 + })); +} + +#[test] +fn does_not_retry_permanent_failures() { + assert!(!is_retryable(&errors::Webhook::DeliveryFailed { + status: 400 + })); + assert!(!is_retryable(&errors::Webhook::DeliveryFailed { + status: 404 + })); + assert!(!is_retryable(&errors::Webhook::DestinationBlocked { + reason: "loopback".to_string(), + })); +} + +// MARK: Backoff + +#[test] +fn backoff_doubles_and_caps() { + assert_eq!(delivery_backoff(0), Duration::from_secs(5)); + assert_eq!(delivery_backoff(1), Duration::from_secs(10)); + assert_eq!(delivery_backoff(2), Duration::from_secs(20)); + assert_eq!(delivery_backoff(6), Duration::from_secs(300)); + assert_eq!(delivery_backoff(100), Duration::from_secs(300)); +} + +#[test] +fn next_delay_uses_backoff_without_retry_after() { + assert_eq!(next_delivery_delay(None, 1), delivery_backoff(1)); +} + +#[test] +fn next_delay_clamps_retry_after() { + assert_eq!(next_delivery_delay(Some(0), 1), MIN_RETRY_AFTER); + assert_eq!( + next_delivery_delay(Some(30_000), 1), + Duration::from_secs(30) + ); + assert_eq!(next_delivery_delay(Some(10_000_000), 1), MAX_RETRY_AFTER); +} + +// MARK: parse_retry_after + +#[test] +fn parses_retry_after_seconds() { + assert_eq!(parse_retry_after(&retry_after("120")), Some(120_000)); +} + +#[test] +fn parses_retry_after_past_date_as_zero() { + assert_eq!( + parse_retry_after(&retry_after("Wed, 21 Oct 2015 07:28:00 GMT")), + Some(0) + ); +} + +#[test] +fn parses_retry_after_future_date() { + let deadline = chrono::Utc::now() + chrono::Duration::seconds(60); + let delay = + parse_retry_after(&retry_after(&deadline.to_rfc2822())).expect("future date should parse"); + assert!( + delay > 50_000 && delay <= 60_000, + "unexpected delay {delay}" + ); +} + +#[test] +fn ignores_invalid_retry_after() { + assert_eq!(parse_retry_after(&retry_after("soon")), None); + assert_eq!(parse_retry_after(&HeaderMap::new()), None); +} diff --git a/engine/packages/workflow-worker/Cargo.toml b/engine/packages/workflow-worker/Cargo.toml index 4fd24b2c8b..047fc4a9c6 100644 --- a/engine/packages/workflow-worker/Cargo.toml +++ b/engine/packages/workflow-worker/Cargo.toml @@ -17,3 +17,4 @@ namespace.workspace = true pegboard.workspace = true rivet-config.workspace = true tracing.workspace = true +webhook.workspace = true diff --git a/engine/packages/workflow-worker/src/lib.rs b/engine/packages/workflow-worker/src/lib.rs index 53351a2bf5..7f120ca2d7 100644 --- a/engine/packages/workflow-worker/src/lib.rs +++ b/engine/packages/workflow-worker/src/lib.rs @@ -17,6 +17,7 @@ pub fn registry() -> Result { .merge(epoxy::registry()?)? .merge(gasoline_runtime::registry()?)? .merge(datacenter::registry()?)? - .merge(depot::registry()?) + .merge(depot::registry()?)? + .merge(webhook::registry()?) .map_err(Into::into) } diff --git a/engine/sdks/rust/data/src/lib.rs b/engine/sdks/rust/data/src/lib.rs index f17cf72bbb..8754cfe157 100644 --- a/engine/sdks/rust/data/src/lib.rs +++ b/engine/sdks/rust/data/src/lib.rs @@ -6,4 +6,5 @@ pub use generated::{ PEGBOARD_NAMESPACE_ACTOR_BY_KEY_VERSION, PEGBOARD_NAMESPACE_ACTOR_NAME_VERSION, PEGBOARD_NAMESPACE_RUNNER_ALLOC_IDX_VERSION, PEGBOARD_NAMESPACE_RUNNER_BY_KEY_VERSION, PEGBOARD_NAMESPACE_RUNNER_CONFIG_VERSION, PEGBOARD_RUNNER_METADATA_VERSION, + WEBHOOK_CONFIG_VERSION, WEBHOOK_DELIVERY_VERSION, }; diff --git a/engine/sdks/rust/data/src/versioned/mod.rs b/engine/sdks/rust/data/src/versioned/mod.rs index 245fae1313..6e46c0fb10 100644 --- a/engine/sdks/rust/data/src/versioned/mod.rs +++ b/engine/sdks/rust/data/src/versioned/mod.rs @@ -125,6 +125,74 @@ impl RunnerAllocIdxKeyData { } } +pub enum WebhookConfigData { + V1(webhook_config_v1::Data), +} + +impl OwnedVersionedData for WebhookConfigData { + type Latest = webhook_config_v1::Data; + + fn wrap_latest(latest: webhook_config_v1::Data) -> Self { + WebhookConfigData::V1(latest) + } + + fn unwrap_latest(self) -> Result { + #[allow(irrefutable_let_patterns)] + if let WebhookConfigData::V1(data) = self { + Ok(data) + } else { + bail!("version not latest"); + } + } + + fn deserialize_version(payload: &[u8], version: u16) -> Result { + match version { + 1 => Ok(WebhookConfigData::V1(serde_bare::from_slice(payload)?)), + _ => bail!("invalid version: {version}"), + } + } + + fn serialize_version(self, _version: u16) -> Result> { + match self { + WebhookConfigData::V1(data) => serde_bare::to_vec(&data).map_err(Into::into), + } + } +} + +pub enum WebhookDeliveryData { + V1(webhook_delivery_v1::Data), +} + +impl OwnedVersionedData for WebhookDeliveryData { + type Latest = webhook_delivery_v1::Data; + + fn wrap_latest(latest: webhook_delivery_v1::Data) -> Self { + WebhookDeliveryData::V1(latest) + } + + fn unwrap_latest(self) -> Result { + #[allow(irrefutable_let_patterns)] + if let WebhookDeliveryData::V1(data) = self { + Ok(data) + } else { + bail!("version not latest"); + } + } + + fn deserialize_version(payload: &[u8], version: u16) -> Result { + match version { + 1 => Ok(WebhookDeliveryData::V1(serde_bare::from_slice(payload)?)), + _ => bail!("invalid version: {version}"), + } + } + + fn serialize_version(self, _version: u16) -> Result> { + match self { + WebhookDeliveryData::V1(data) => serde_bare::to_vec(&data).map_err(Into::into), + } + } +} + pub enum MetadataKeyData { V1(pegboard_runner_metadata_v1::Data), } diff --git a/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs b/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs index 26097a4bd2..6c4a40ad23 100644 --- a/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs +++ b/engine/sdks/rust/data/src/versioned/namespace_runner_config.rs @@ -30,48 +30,24 @@ impl OwnedVersionedData for NamespaceRunnerConfig { fn deserialize_version(payload: &[u8], version: u16) -> Result { match version { - 1 => Ok(NamespaceRunnerConfig::V1( - serde_bare::from_slice(payload)?, - )), - 2 => Ok(NamespaceRunnerConfig::V2( - serde_bare::from_slice(payload)?, - )), - 3 => Ok(NamespaceRunnerConfig::V3( - serde_bare::from_slice(payload)?, - )), - 4 => Ok(NamespaceRunnerConfig::V4( - serde_bare::from_slice(payload)?, - )), - 5 => Ok(NamespaceRunnerConfig::V5( - serde_bare::from_slice(payload)?, - )), - 6 => Ok(NamespaceRunnerConfig::V6( - serde_bare::from_slice(payload)?, - )), + 1 => Ok(NamespaceRunnerConfig::V1(serde_bare::from_slice(payload)?)), + 2 => Ok(NamespaceRunnerConfig::V2(serde_bare::from_slice(payload)?)), + 3 => Ok(NamespaceRunnerConfig::V3(serde_bare::from_slice(payload)?)), + 4 => Ok(NamespaceRunnerConfig::V4(serde_bare::from_slice(payload)?)), + 5 => Ok(NamespaceRunnerConfig::V5(serde_bare::from_slice(payload)?)), + 6 => Ok(NamespaceRunnerConfig::V6(serde_bare::from_slice(payload)?)), _ => bail!("invalid version: {version}"), } } fn serialize_version(self, _version: u16) -> Result> { match self { - NamespaceRunnerConfig::V1(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V2(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V3(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V4(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V5(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } - NamespaceRunnerConfig::V6(data) => { - serde_bare::to_vec(&data).map_err(Into::into) - } + NamespaceRunnerConfig::V1(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V2(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V3(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V4(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V5(data) => serde_bare::to_vec(&data).map_err(Into::into), + NamespaceRunnerConfig::V6(data) => serde_bare::to_vec(&data).map_err(Into::into), } } diff --git a/engine/sdks/schemas/data/webhook.config.v1.bare b/engine/sdks/schemas/data/webhook.config.v1.bare new file mode 100644 index 0000000000..2cb13802e0 --- /dev/null +++ b/engine/sdks/schemas/data/webhook.config.v1.bare @@ -0,0 +1,11 @@ +type WebhookEventType enum { + RUNNER_POOL_ERROR + RUNNER_POOL_HEALTHY + ACTOR_HTTP_REQUEST +} + +type Data struct { + url: str + headers: map + subscriptions: list +} diff --git a/engine/sdks/schemas/data/webhook.delivery.v1.bare b/engine/sdks/schemas/data/webhook.delivery.v1.bare new file mode 100644 index 0000000000..56cd50616d --- /dev/null +++ b/engine/sdks/schemas/data/webhook.delivery.v1.bare @@ -0,0 +1,23 @@ +type DeliveryStatus enum { + PENDING + SUCCEEDED + FAILED +} + +# Mirrors the enum in `webhook.config.v1.bare`. BARE schemas cannot reference types across +# files, so each generated module gets its own copy; both convert to `types::WebhookEventType`. +type WebhookEventType enum { + RUNNER_POOL_ERROR + RUNNER_POOL_HEALTHY + ACTOR_HTTP_REQUEST +} + +type Data struct { + payload: str + status: DeliveryStatus + attemptCount: u32 + lastError: optional + createdAt: i64 + eventType: WebhookEventType + subject: optional +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 0000000000..ffc9e7505e --- /dev/null +++ b/flake.lock @@ -0,0 +1,117 @@ +{ + "nodes": { + "fenix": { + "inputs": { + "nixpkgs": [ + "nixpkgs" + ], + "rust-analyzer-src": "rust-analyzer-src" + }, + "locked": { + "lastModified": 1787555220, + "narHash": "sha256-ZNw7FDQm2PVOIyE9WaLFgK1QYw8vrOchxMYHzDDxlUY=", + "owner": "nix-community", + "repo": "fenix", + "rev": "2013c981829bd5e93db06749b5639450c81bece3", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "fenix", + "type": "github" + } + }, + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1787498568, + "narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-unstable": { + "locked": { + "lastModified": 1787498568, + "narHash": "sha256-9i/VTdusq/+NM/tz+J1Re+ojkMB8MBf0QshnYfzHz30=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "56c02bc00adcf003215cc4bd996d6efaf4cff188", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixos-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "fenix": "fenix", + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs", + "nixpkgs-unstable": "nixpkgs-unstable" + } + }, + "rust-analyzer-src": { + "flake": false, + "locked": { + "lastModified": 1787472531, + "narHash": "sha256-cq+wrv+Zadl1NiJIWxYKHpcXfUQXDKBYa/7eq0JWFZk=", + "owner": "rust-lang", + "repo": "rust-analyzer", + "rev": "5c156cdfb047ea171ba255dcdcd86236d98a389e", + "type": "github" + }, + "original": { + "owner": "rust-lang", + "ref": "nightly", + "repo": "rust-analyzer", + "type": "github" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 0000000000..7099e26281 --- /dev/null +++ b/flake.nix @@ -0,0 +1,48 @@ +{ + description = "Rust development environment"; + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + fenix = { + url = "github:nix-community/fenix"; + inputs.nixpkgs.follows = "nixpkgs"; + }; + nixpkgs-unstable.url = "github:nixos/nixpkgs/nixos-unstable"; + }; + outputs = { self, nixpkgs, nixpkgs-unstable, flake-utils, fenix }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = nixpkgs.legacyPackages.${system}; + toolchain = fenix.packages.${system}.combine [ + (fenix.packages.${system}.fromToolchainFile { + file = ./rust-toolchain.toml; + sha256 = "sha256-P30Tm3O7vQAE725YtDCDHGjNrSsfZO4us11UwJGZSJo="; + }) + fenix.packages.${system}.targets.wasm32-unknown-unknown.stable.rust-std + fenix.packages.${system}.stable.rust-src + ]; + in + { + devShells.default = pkgs.mkShell rec { + nativeBuildInputs = [ pkgs.pkg-config ]; + buildInputs = with pkgs; [ + toolchain + clang + llvmPackages.bintools + stdenv.cc.cc.lib + nodejs_22 + pnpm + ]; + LIBCLANG_PATH = pkgs.lib.makeLibraryPath [ pkgs.llvmPackages_latest.libclang.lib ]; + LD_LIBRARY_PATH = pkgs.lib.makeLibraryPath (buildInputs ++ nativeBuildInputs); + BINDGEN_EXTRA_CLANG_ARGS = + (builtins.map (a: ''-I"${a}/include"'') [ pkgs.glibc.dev ]) + ++ [ + ''-I"${pkgs.llvmPackages_latest.libclang.lib}/lib/clang/${pkgs.llvmPackages_latest.libclang.version}/include"'' + ''-I"${pkgs.glib.dev}/include/glib-2.0"'' + ''-I${pkgs.glib.out}/lib/glib-2.0/include/'' + ]; + }; + } + ); +} \ No newline at end of file diff --git a/frontend/src/app/runner-pool-error-popover.tsx b/frontend/src/app/runner-pool-error-popover.tsx index 268a9f118f..18a1f1b371 100644 --- a/frontend/src/app/runner-pool-error-popover.tsx +++ b/frontend/src/app/runner-pool-error-popover.tsx @@ -43,6 +43,7 @@ interface ClassifiedError { kind: | "serverless_http" | "serverless_connection" + | "serverless_destination_blocked" | "serverless_invalid_sse" | "serverless_stream_ended_early" | "downgrade" @@ -116,6 +117,18 @@ function classifyRunnerError(error: RivetActorError): ClassifiedError { fingerprint: `conn:${e.serverless_connection_error.message.slice(0, 64)}`, }), ) + .with( + P.shape({ + serverless_destination_blocked: P.shape({ reason: P.string }), + }), + (e) => ({ + severity: "error", + kind: "serverless_destination_blocked", + title: "Serverless URL is not an allowed destination", + body: e.serverless_destination_blocked.reason, + fingerprint: `blocked:${e.serverless_destination_blocked.reason.slice(0, 64)}`, + }), + ) .with( P.shape({ serverless_invalid_sse_payload: P.shape({ message: P.string }), @@ -502,6 +515,8 @@ function describeKind(kind: ClassifiedError["kind"]): string { return "Runner pool was downgraded to an unsupported version. Revert to a higher version."; case "serverless_stream_ended_early": return "Connection terminated before the runner stopped. Check the request lifespan limits on your serverless provider."; + case "serverless_destination_blocked": + return "The configured serverless URL points at a destination Rivet is not allowed to reach. Use a publicly routable URL, or allow the address range in the engine's outbound configuration."; case "internal": return "An internal error occurred in the runner pool."; default: diff --git a/frontend/src/components/actors/actor-status-label.tsx b/frontend/src/components/actors/actor-status-label.tsx index 7ce47442c2..5d08591928 100644 --- a/frontend/src/components/actors/actor-status-label.tsx +++ b/frontend/src/components/actors/actor-status-label.tsx @@ -103,6 +103,7 @@ export function ActorError({ error }: { error: object | string }) { .or(P.shape({ serverless_http_error: P.any })) .or(P.string) .or(P.shape({ serverless_connection_error: P.any })) + .or(P.shape({ serverless_destination_blocked: P.any })) .or(P.shape({ serverless_invalid_sse_payload: P.any })), }), (err) => , @@ -221,6 +222,23 @@ export function RunnerPoolError({ error }: { error: RivetActorError }) { ); }, ) + .with( + P.shape({ + serverless_destination_blocked: P.shape({ reason: P.string }), + }), + (errObj) => { + const reason = errObj.serverless_destination_blocked?.reason; + return ( + <> +

+ Serverless endpoint URL is not an allowed + destination +

+ {reason ? : null} + + ); + }, + ) .with( P.shape({ serverless_invalid_sse_payload: P.shape({ message: P.string }), diff --git a/frontend/src/queries/types.ts b/frontend/src/queries/types.ts index fdffce099f..f69860e2b3 100644 --- a/frontend/src/queries/types.ts +++ b/frontend/src/queries/types.ts @@ -35,4 +35,5 @@ export type RivetActorError = | { runner_id: string } | { serverless_http_error: unknown } | { serverless_connection_error: unknown } + | { serverless_destination_blocked: unknown } | { serverless_invalid_sse_payload: unknown }; diff --git a/rust-toolchain.toml b/rust-toolchain.toml new file mode 100644 index 0000000000..292fe499e3 --- /dev/null +++ b/rust-toolchain.toml @@ -0,0 +1,2 @@ +[toolchain] +channel = "stable" diff --git a/self-host/compose/dev-host/rivet-engine/config.jsonc b/self-host/compose/dev-host/rivet-engine/config.jsonc index ee79af406a..1cd7e292d1 100644 --- a/self-host/compose/dev-host/rivet-engine/config.jsonc +++ b/self-host/compose/dev-host/rivet-engine/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@127.0.0.1:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc index 29ca3ee404..058a1b0a44 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-a/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc index 0fe743310a..e8136b2656 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-b/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/0/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/1/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc index 5a639b2fd4..be419e4ab6 100644 --- a/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multidc-multinode/datacenters/dc-c/rivet-engine/2/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc-multinode/docker-compose.yml b/self-host/compose/dev-multidc-multinode/docker-compose.yml index 81b39d39eb..a971a9716d 100644 --- a/self-host/compose/dev-multidc-multinode/docker-compose.yml +++ b/self-host/compose/dev-multidc-multinode/docker-compose.yml @@ -208,7 +208,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -254,7 +253,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -298,7 +296,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-a:4317 stop_grace_period: 0s depends_on: @@ -519,7 +516,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -563,7 +559,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -607,7 +602,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-b:4317 stop_grace_period: 0s depends_on: @@ -826,7 +820,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: @@ -870,7 +863,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: @@ -914,7 +906,6 @@ services: - RUST_LOG_ANSI_COLOR=1 - RUST_LOG=universaldb::driver::postgres=debug - RIVET_OTEL_ENABLED=1 - - RIVET_OTEL_SAMPLER_RATIO=1 - RIVET_OTEL_GRPC_ENDPOINT=http://otel-collector-dc-c:4317 stop_grace_period: 0s depends_on: diff --git a/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc index ac3c4bc3c6..eaef4154fa 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-a/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-a:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc index 8e5c6aaa45..c42e0a0ffb 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-b/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-b:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc b/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc index 6ba3be6d65..6a6238fec7 100644 --- a/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc +++ b/self-host/compose/dev-multidc/datacenters/dc-c/rivet-engine/config.jsonc @@ -43,6 +43,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres-dc-c:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/0/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/1/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc b/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc index 13e9996d6e..b57d5c4e01 100644 --- a/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc +++ b/self-host/compose/dev-multinode/rivet-engine/2/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/dev/rivet-engine/config.jsonc b/self-host/compose/dev/rivet-engine/config.jsonc index 4c6312b6d3..cad7a1e600 100644 --- a/self-host/compose/dev/rivet-engine/config.jsonc +++ b/self-host/compose/dev/rivet-engine/config.jsonc @@ -21,6 +21,9 @@ } } }, + "outbound": { + "allow_private_networks": true + }, "postgres": { "url": "postgresql://postgres:postgres@postgres:5432/rivet_engine" }, diff --git a/self-host/compose/template/src/main.ts b/self-host/compose/template/src/main.ts index 82a08e1f73..4a984d1f70 100644 --- a/self-host/compose/template/src/main.ts +++ b/self-host/compose/template/src/main.ts @@ -18,7 +18,7 @@ import { generateDatacenterVectorClient } from "./services/edge/vector-client"; import { generateDatacenterVectorServer } from "./services/edge/vector-server"; function generateTemplate(templateName: string, config: TemplateConfig) { - const outputDir = path.join(__dirname, "../../../", templateName); + const outputDir = path.join(__dirname, "../../", templateName); // Remove existing directory if it exists if (fs.existsSync(outputDir)) { diff --git a/self-host/compose/template/src/services/edge/rivet-engine.ts b/self-host/compose/template/src/services/edge/rivet-engine.ts index 1c5b8247da..554d7b8454 100644 --- a/self-host/compose/template/src/services/edge/rivet-engine.ts +++ b/self-host/compose/template/src/services/edge/rivet-engine.ts @@ -45,6 +45,11 @@ export function generateDatacenterRivetEngine( host: "0.0.0.0", }, topology, + // Serverless runner URLs in a compose deployment point at other containers on this + // network, which the engine refuses to dial by default. + outbound: { + allow_private_networks: true, + }, postgres: { url: `postgresql://postgres:postgres@${context.getServiceHost("postgres", datacenter.name)}:5432/rivet_engine`, },