diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2eea525 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/cloud/google/cloud-run/.env.example b/cloud/google/cloud-run/.env.example new file mode 100644 index 0000000..8da31a5 --- /dev/null +++ b/cloud/google/cloud-run/.env.example @@ -0,0 +1,64 @@ +# Copy this file to .env and fill in your values before running deploy.sh. +# deploy.sh will source .env automatically if it exists. +# Never commit .env to source control — it contains sensitive values. + +# ── Required ────────────────────────────────────────────────────────────────── + +# Google Cloud project ID +GCP_PROJECT_ID=my-project-id + +# GCS bucket containing your Deepgram model files. +# The bucket must already exist and be populated with model files before deploying. +# Example bucket layout: +# gs://my-models-bucket/nova-2-general/ +# gs://my-models-bucket/aura-asteria-en/ +DEEPGRAM_MODELS_BUCKET=my-models-bucket + +# Your Deepgram self-hosted API key. +# Obtain from https://developers.deepgram.com/docs/self-hosted-self-service-tutorial +DEEPGRAM_API_KEY=your_deepgram_api_key_here + +# ── Optional (defaults shown) ────────────────────────────────────────────────── + +# GCP region. Must support Cloud Run GPU (L4). +# Supported regions: https://cloud.google.com/run/docs/configuring/services/gpu +# GCP_REGION=us-central1 + +# Container image tag for all Deepgram services. +# IMAGE_TAG=release-260305 + +# Name of the Secret Manager secret that stores your Deepgram API key. +# The secret is created automatically by deploy.sh if it does not exist. +# DEEPGRAM_API_KEY_SECRET=deepgram-api-key + +# Email of the service account created by deploy.sh for the Cloud Run services. +# Override this if you want to use a pre-existing service account instead. +# SERVICE_ACCOUNT_NAME=deepgram-cloud-run + +# Cloud Run Engine service name. +# ENGINE_SERVICE_NAME=deepgram-engine + +# Cloud Run API service name. +# API_SERVICE_NAME=deepgram-api + +# Container image registry. Defaults to quay.io/deepgram (public). +# If your GCP project restricts images to Artifact Registry, set this to your +# AR remote repo path (created with --remote-docker-repo=https://quay.io): +# IMAGE_REGISTRY=us-central1-docker.pkg.dev/YOUR_PROJECT/deepgram-quay/deepgram + +# VPC network for Cloud Run Direct VPC Egress. +# If VPC_NETWORK does not exist, deploy.sh creates it automatically. +# Set both variables to use a pre-existing VPC and subnet instead. +# VPC_NETWORK=deepgram-vpc +# VPC_SUBNETWORK=deepgram-subnet +# VPC_SUBNET_RANGE=10.8.0.0/24 + +# Proxy-only subnet required by the internal HTTP load balancer. +# Created automatically if it does not exist. +# VPC_PROXY_SUBNET=deepgram-proxy-subnet +# VPC_PROXY_SUBNET_RANGE=10.8.1.0/24 + +# Internal HTTP load balancer placed in front of the Engine. +# The API uses the LB's internal IP (HTTP, no TLS) to reach the Engine. +# ENGINE_LB_NAME=deepgram-engine-lb +# ENGINE_LB_IP_NAME=deepgram-engine-lb-ip diff --git a/cloud/google/cloud-run/README.md b/cloud/google/cloud-run/README.md new file mode 100644 index 0000000..6af4615 --- /dev/null +++ b/cloud/google/cloud-run/README.md @@ -0,0 +1,465 @@ +# Deepgram Self-Hosted on Google Cloud Run (NVIDIA GPU) + +This directory contains everything needed to deploy Deepgram's self-hosted speech AI services on [Google Cloud Run](https://cloud.google.com/run) using NVIDIA L4 GPUs. + +## Architecture + +Each workload type requires its own dedicated API+Engine pair. The Deepgram API can only route to a single engine pool — there is no cross-workload request routing. Flux and Aura TTS **must not share** GPU instances with other models. + +``` + Internet + │ + ┌────────────────┼────────────────┐ + ▼ ▼ ▼ +┌────────────────┐ ┌────────────────┐ ┌────────────────┐ +│ deepgram-api │ │ deepgram-api │ │ deepgram-api │ +│ (STT / nova) │ │ -flux │ │ -tts │ +│ public ingress │ │ public ingress │ │ public ingress │ +│ listen_v2=false│ │ listen_v2=true │ │ speak_ │ +│ speak_ │ │ speak_ │ │ streaming=true │ +│ streaming=false│ │ streaming=false│ │ listen_v2=false│ +└───────┬────────┘ └───────┬────────┘ └───────┬────────┘ + │ HTTPS │ HTTPS │ HTTPS + │ (internal) │ (internal) │ (internal) + ▼ ▼ ▼ +┌───────────────┐ ┌────────────────┐ ┌────────────────────┐ +│ deepgram- │ │ deepgram- │ │ deepgram- │ +│ engine │ │ engine-flux │ │ engine-tts │ +│ (STT / Nova) │ │ (Flux only) │ │ (Aura TTS only) │ +│ 1x L4 GPU │ │ 1x L4 GPU │ │ 2x L4 GPUs │ +│ 8 vCPU │ │ 8 vCPU │ │ 16 vCPU │ +│ 32 GiB RAM │ │ 32 GiB RAM │ │ 64 GiB RAM │ +│ Up to 5 nova │ │ Dedicated; │ │ Dedicated; │ +│ models/GPU │ │ no other models│ │ CUDA_VISIBLE=0,1 │ +└───────────────┘ └────────────────┘ └────────────────────┘ +``` + +| API Service | Engine Service | Config Template | Workload | +|-------------|----------------|-----------------|----------| +| `deepgram-api` | `deepgram-engine` | `api.toml.tmpl` → `engine.toml` | nova-2/nova-3 STT | +| `deepgram-api-flux` | `deepgram-engine-flux` | `api-flux.toml.tmpl` → `engine-flux.toml` | Flux streaming STT | +| `deepgram-api-tts` | `deepgram-engine-tts` | `api-tts.toml.tmpl` → *(Deepgram-provided)* | Aura/Aura-2 TTS | + +- **API services** — Each handles requests for one workload type only, routes to its paired Engine, and returns results. All are public HTTPS endpoints. Scale horizontally without GPU. +- **STT Engine** — Runs nova-2/nova-3 inference. Supports up to 5 concurrent model files per L4 GPU instance. +- **Flux Engine** — Runs Flux turn-based streaming STT. Must be fully isolated; no other models may share this instance. Requires `release-251015` or later. +- **TTS Engine** — Runs Aura/Aura-2 text-to-speech. Requires 2x L4 GPUs and dedicated hardware. Requires language-specific `IMPELLER_*` UUIDs from Deepgram. +- **Models** — Stored in GCS buckets and mounted read-only via [GCS FUSE](https://cloud.google.com/run/docs/storage/gcs-fuse). Use separate buckets per workload type to prevent accidental model mixing. +- **Secrets** — API key, TOML configs, and TTS model UUIDs stored in [Secret Manager](https://cloud.google.com/secret-manager) and mounted at container startup. + +## Prerequisites + +| Requirement | Notes | +|-------------|-------| +| [gcloud CLI](https://cloud.google.com/sdk/docs/install) | Authenticated with `gcloud auth login` | +| [gettext](https://www.gnu.org/software/gettext/) (`envsubst`) | `brew install gettext` (macOS) / `apt-get install gettext` (Linux) | +| GCP project with billing enabled | GPU quota must be requested separately | +| NVIDIA L4 GPU quota | See [Request GPU quota](#1-request-gpu-quota) | +| Deepgram self-hosted API key | From [Deepgram console](https://developers.deepgram.com/docs/self-hosted-self-service-tutorial#create-a-self-hosted-api-key) | +| Model files in a GCS bucket | See [Upload models](#2-upload-models-to-gcs) | + +## Deployment Guide + +### 1. Request GPU quota + +Cloud Run GPU instances require explicit quota approval. This step is done once per project. + +1. Open the [GCP Quotas console](https://console.cloud.google.com/iam-admin/quotas). +2. Search for **`NVIDIA_L4_GPU`** under Cloud Run. +3. Request the total number of GPUs you need across all Engine services: + - **STT Engine**: 1 GPU per replica × `maxScale` replicas + - **Flux Engine**: 1 GPU per replica × `maxScale` replicas (if deploying Flux) + - **TTS Engine**: **2 GPUs per replica** × `maxScale` replicas (if deploying Aura TTS) + + Example: 3 STT replicas + 2 Flux replicas + 1 TTS replica = 3 + 2 + 2 = **7 L4 GPUs** + +Quota approval typically takes minutes to hours. + +GPU availability by region: + +| Region | L4 | A100 40GB | A100 80GB | H100 | +|--------|----|-----------|-----------| -----| +| `us-central1` | ✓ | ✓ | ✓ | ✓ | +| `us-east4` | ✓ | | | | +| `us-east1` | ✓ | | | | +| `europe-west1` | ✓ | | | | +| `asia-northeast1` | ✓ | | | | + +L4 is the recommended starting point for most inference workloads. To use a different GPU, change `run.googleapis.com/gpu-type` in `services/engine.service.yaml` and update `maxReplicas` to stay within your quota. + +### 2. Upload models to GCS + +The Engine container reads models from a GCS bucket mounted at `/models` via GCS FUSE. The bucket must be populated before deployment. + +```bash +# Create the bucket (if it does not already exist) +gsutil mb -p YOUR_PROJECT_ID -l us-central1 gs://YOUR_MODELS_BUCKET + +# Upload your model files +# Model files are typically provided by Deepgram as .dg archives or directories. +gsutil -m cp -r /local/path/to/models/* gs://YOUR_MODELS_BUCKET/ +``` + +The `engine.toml` config sets `search_paths = ["/models"]`, so place model files at the top level of the bucket or in subdirectories — the Engine will scan recursively. + +Example bucket layout for the STT (nova) engine: +``` +gs://YOUR_MODELS_BUCKET/ +├── nova-2-general/ +│ └── model.dg +└── nova-3-general/ + └── model.dg +``` + +Use **separate GCS buckets** per workload type to prevent models from being accidentally loaded on the wrong engine: + +| Workload | Recommended bucket name | Contents | +|----------|------------------------|----------| +| STT (nova) | `your-project-stt-models` | nova-2, nova-3 model files | +| Flux | `your-project-flux-models` | Flux model file only | +| TTS (Aura) | `your-project-tts-models` | Aura/Aura-2 model files | + +An L4 GPU can host **up to 5 nova-2/nova-3 model files** concurrently. Set `max_concurrently_loaded_models = 5` in `engine.toml` to enforce this limit. + +### 3. Configure environment + +```bash +cd cloud-run/ +cp .env.example .env +$EDITOR .env +``` + +The minimum required values in `.env`: + +```bash +GCP_PROJECT_ID=your-project-id +DEEPGRAM_MODELS_BUCKET=your-models-bucket +DEEPGRAM_API_KEY=your_deepgram_api_key +``` + +Optional values (defaults are shown in `.env.example`): + +| Variable | Default | Description | +|----------|---------|-------------| +| `GCP_REGION` | `us-central1` | Deployment region (must support L4 GPU) | +| `IMAGE_TAG` | `release-260305` | Deepgram container image tag | +| `SERVICE_ACCOUNT_NAME` | `deepgram-cloud-run` | GCP service account name | +| `ENGINE_SERVICE_NAME` | `deepgram-engine` | Cloud Run Engine service name | +| `API_SERVICE_NAME` | `deepgram-api` | Cloud Run API service name | + +### 4. (Optional) Customize configuration + +Edit `config/engine.toml` and `config/api.toml.tmpl` before deploying to adjust model settings, concurrency limits, or feature flags. + +Key Engine settings to review: + +```toml +# Limit concurrent requests per GPU instance. +# For STT streaming on an L4, start with 8–16 and tune based on load. +# max_active_requests = 16 + +# Pre-load models at startup so the first request has no cold-start latency. +# blocking = true means Cloud Run will not send traffic until models are loaded. +[preload_models] +blocking = true +``` + +Key API settings to review: + +```toml +# Limit concurrent requests per API replica to avoid memory pressure. +[concurrency_limit] +# active_requests = 100 +``` + +`config/api.toml.tmpl` contains the placeholder `ENGINE_SERVICE_URL` which `deploy.sh` replaces automatically with the Engine's Cloud Run URL. + +### 5. Deploy + +```bash +chmod +x deploy.sh +./deploy.sh +``` + +The script performs these steps automatically: + +1. Enables required GCP APIs (`run`, `secretmanager`, `storage`, `iam`) +2. Creates a service account with the minimum required permissions +3. Stores your Deepgram API key and TOML configs in Secret Manager +4. Deploys the **Engine** Cloud Run service (GPU, internal ingress) +5. Retrieves the Engine's HTTPS URL and substitutes it into `api.toml.tmpl` +6. Uploads the generated `api.toml` to Secret Manager +7. Deploys the **API** Cloud Run service (public ingress) +8. Prints the public API endpoint and sample curl command + +Deployment takes approximately **5–10 minutes** on the first run (model preloading and GCS FUSE initialization dominate startup time). Subsequent deployments are faster. + +### 6. Verify the deployment + +```bash +# Substitute the URL printed by deploy.sh +API_URL=https://deepgram-api-XXXX-uc.a.run.app + +# Check API status +curl "${API_URL}/v1/status" + +# Transcribe a WAV file +curl -X POST "${API_URL}/v1/listen?model=nova-2" \ + -H "Authorization: Token ${DEEPGRAM_API_KEY}" \ + -H "Content-Type: audio/wav" \ + --data-binary @audio.wav +``` + +## Updating configuration + +To update `engine.toml` or `api.toml` without redeploying the full stack: + +```bash +# Update engine config +gcloud secrets versions add deepgram-engine-config \ + --data-file=config/engine.toml + +# Force a new Engine revision to pick up the new config +gcloud run services update deepgram-engine \ + --region=us-central1 \ + --update-env-vars=CONFIG_RELOAD="$(date +%s)" +``` + +Or re-run `./deploy.sh` — it is idempotent and will update all secrets and redeploy both services. + +## Deploying Flux (dedicated STT) + +Flux requires its own dedicated Cloud Run service and must not share GPU instances with other models. + +**Requirements (from Deepgram docs):** +- Container image `release-251015` or later (current default satisfies this) +- 1x NVIDIA L4 GPU per instance, fully isolated — no nova, TTS, or supplementary models +- `flux.max_streams` explicitly set in `engine-flux.toml` (contact Deepgram for L4 value) +- `max_active_requests` does **not** apply to Flux; use `flux.max_streams` instead + +**Steps:** + +1. Create a separate GCS bucket for the Flux model file only. + +2. Store the Flux engine config in Secret Manager: + ```bash + gcloud secrets create deepgram-engine-flux-config \ + --data-file=config/engine-flux.toml \ + --project=${GCP_PROJECT_ID} + ``` + +3. Deploy the Flux Engine service: + ```bash + envsubst < services/engine-flux.service.yaml | \ + gcloud run services replace - --region=${GCP_REGION} --project=${GCP_PROJECT_ID} + ``` + +4. Retrieve the Flux Engine URL and generate the Flux API config: + ```bash + FLUX_ENGINE_URL=$(gcloud run services describe deepgram-engine-flux \ + --platform managed --region=${GCP_REGION} --project=${GCP_PROJECT_ID} \ + --format='value(status.url)') + + sed "s|FLUX_ENGINE_SERVICE_URL|${FLUX_ENGINE_URL}|g" \ + config/api-flux.toml.tmpl > /tmp/api-flux.toml + + gcloud secrets create deepgram-api-flux-config \ + --data-file=/tmp/api-flux.toml \ + --project=${GCP_PROJECT_ID} + ``` + +5. Deploy the Flux API service: + ```bash + envsubst < services/api-flux.service.yaml | \ + gcloud run services replace - --region=${GCP_REGION} --project=${GCP_PROJECT_ID} + ``` + +**Monitor Flux capacity** via the Engine metrics endpoint: +``` +flux_used_streams — active streams +flux_max_streams — configured maximum +flux_fraction_streams — scale out when this approaches 0.8 +``` + +--- + +## Deploying Aura TTS (dedicated TTS) + +Aura TTS requires its own dedicated Cloud Run service with 2x NVIDIA L4 GPUs per instance. + +**Requirements (from Deepgram docs):** +- 2x NVIDIA L4 GPUs (32 GB GPU RAM total) per instance +- 8+ CPU cores and 64 GiB RAM per instance (reflected as 16 vCPU / 64 GiB in the service YAML) +- `CUDA_VISIBLE_DEVICES=0,1` to expose both GPUs to the TTS engine +- Language-specific `IMPELLER_AURA2_T2C_UUID` and `IMPELLER_AURA2_C2A_UUID` env vars (provided by Deepgram) +- Dedicated hardware — do not mix with STT or Flux workloads + +**Steps:** + +1. Obtain your Aura-2 model UUIDs from your Deepgram Account Representative. + +2. Store the UUIDs in Secret Manager: + ```bash + echo -n "YOUR_T2C_UUID" | gcloud secrets create deepgram-tts-t2c-uuid \ + --data-file=- --project=${GCP_PROJECT_ID} + + echo -n "YOUR_C2A_UUID" | gcloud secrets create deepgram-tts-c2a-uuid \ + --data-file=- --project=${GCP_PROJECT_ID} + ``` + +3. Store the TTS engine config (provided by Deepgram) in Secret Manager: + ```bash + gcloud secrets create deepgram-engine-tts-config \ + --data-file=path/to/engine-tts.toml \ + --project=${GCP_PROJECT_ID} + ``` + +4. Grant the service account access to the new secrets: + ```bash + for SECRET in deepgram-tts-t2c-uuid deepgram-tts-c2a-uuid deepgram-engine-tts-config; do + gcloud secrets add-iam-policy-binding ${SECRET} \ + --project=${GCP_PROJECT_ID} \ + --member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" \ + --role="roles/secretmanager.secretAccessor" + done + ``` + +5. Request **2x** NVIDIA_L4_GPU quota per TTS replica in the GCP console. + +6. Deploy the TTS Engine service: + ```bash + envsubst < services/engine-tts.service.yaml | \ + gcloud run services replace - --region=${GCP_REGION} --project=${GCP_PROJECT_ID} + ``` + +7. Retrieve the TTS Engine URL and generate the TTS API config: + ```bash + TTS_ENGINE_URL=$(gcloud run services describe deepgram-engine-tts \ + --platform managed --region=${GCP_REGION} --project=${GCP_PROJECT_ID} \ + --format='value(status.url)') + + sed "s|TTS_ENGINE_SERVICE_URL|${TTS_ENGINE_URL}|g" \ + config/api-tts.toml.tmpl > /tmp/api-tts.toml + + gcloud secrets create deepgram-api-tts-config \ + --data-file=/tmp/api-tts.toml \ + --project=${GCP_PROJECT_ID} + ``` + +8. Deploy the TTS API service: + ```bash + envsubst < services/api-tts.service.yaml | \ + gcloud run services replace - --region=${GCP_REGION} --project=${GCP_PROJECT_ID} + ``` + +For full TTS deployment details, see the [Deepgram Deploy TTS Services guide](https://developers.deepgram.com/docs/deploy-tts-services). + +--- + +## Scaling + +### Engine replicas + +Edit the autoscaling annotations in `services/engine.service.yaml`: + +```yaml +autoscaling.knative.dev/minScale: "1" # Warm GPU instances (0 = scale to zero, cold starts ~2–5 min) +autoscaling.knative.dev/maxScale: "3" # Must be within your L4 GPU quota +``` + +Cloud Run scales the Engine on incoming request concurrency. Tune `max_active_requests` in `engine.toml` to control how many requests each Engine pod accepts before Cloud Run routes to a new replica. + +### API replicas + +The API service scales on request concurrency automatically. It does not require GPU and scales quickly. Adjust `maxScale` in `services/api.service.yaml` if you need more API throughput. + +### Switching GPU types + +To use A100 or H100 instead of L4: + +1. Verify GPU availability in your region (see the table above). +2. Change the annotation in `services/engine.service.yaml`: + ```yaml + run.googleapis.com/gpu-type: nvidia-a100-80gb + ``` +3. Request the appropriate quota and adjust `cpu`/`memory` limits accordingly. +4. Re-run `./deploy.sh`. + +## Cost considerations + +- **GPU instances** are the dominant cost. An NVIDIA L4 on Cloud Run is billed per second of request processing (when `minScale: "0"`) or per-instance-hour (when `minScale: "1"`). +- Setting `minScale: "0"` on the Engine eliminates idle cost but introduces a **cold start of 2–5 minutes** (GPU allocation + model loading). +- Setting `minScale: "1"` keeps one warm GPU instance running at all times for immediate response, at the cost of continuous GPU billing. +- The API service is cheap — a few vCPUs with no GPU. + +Estimate your costs using the [Cloud Run pricing calculator](https://cloud.google.com/products/calculator). + +## Limitations + +| Limitation | Detail | +|------------|--------| +| **Request size** | Cloud Run's HTTP load balancer accepts up to 32 MB per request body. For larger audio files, use the `url` query parameter to fetch audio from a remote URL, or use the `callback` parameter for async transcription. | +| **Streaming WebSockets** | Cloud Run supports WebSocket connections (HTTP/2). Streaming STT over WebSocket works, but individual connections are limited to the service `timeoutSeconds` (default: 3600s). | +| **Metrics port** | The Engine metrics server runs on port 9991, which is not exposed by Cloud Run's load balancer. To scrape metrics, deploy a Cloud Run sidecar container or use Cloud Run's built-in metrics in Cloud Monitoring. | +| **Model cold start** | If `preload_models.blocking = true`, Cloud Run will not route requests to a new Engine instance until all models are loaded (~1–3 min). This is safer than allowing requests during loading. | +| **Voice agent** | Voice agent (`agent.enabled`) requires persistent bidirectional state. Cloud Run can handle WebSocket-based agent sessions but has not been validated for production voice agent workloads. | + +## Logs and monitoring + +```bash +# Tail API logs +gcloud run services logs tail deepgram-api --region=us-central1 + +# Tail Engine logs (GPU instance) +gcloud run services logs tail deepgram-engine --region=us-central1 + +# View all logs in Cloud Logging +gcloud logging read \ + 'resource.type="cloud_run_revision" resource.labels.service_name="deepgram-engine"' \ + --limit=100 --format=json +``` + +Cloud Run automatically exports metrics (request count, latency, instance count) to [Cloud Monitoring](https://console.cloud.google.com/monitoring). GPU utilization metrics require [DCGM](https://developer.nvidia.com/dcgm) which is not currently available in Cloud Run. + +## Teardown + +```bash +# Remove Cloud Run services and secrets (keeps GCS bucket and service account) +./teardown.sh + +# Remove everything including the service account +./teardown.sh --all +``` + +The GCS models bucket is never deleted automatically. Delete it manually when no longer needed: + +```bash +gsutil -m rm -r gs://YOUR_MODELS_BUCKET +``` + +## Directory structure + +``` +cloud-run/ +├── README.md # This file +├── deploy.sh # Deployment script (STT / nova baseline) +├── teardown.sh # Cleanup script +├── .env.example # Environment variable template +├── config/ +│ ├── engine.toml # STT Engine config (nova-2 / nova-3) +│ ├── engine-flux.toml # Flux Engine config (dedicated) +│ ├── api.toml.tmpl # STT API config template (ENGINE_SERVICE_URL) +│ ├── api-flux.toml.tmpl # Flux API config template (FLUX_ENGINE_SERVICE_URL) +│ └── api-tts.toml.tmpl # TTS API config template (TTS_ENGINE_SERVICE_URL) +└── services/ + ├── api.service.yaml # Cloud Run API — STT (nova) + ├── api-flux.service.yaml # Cloud Run API — Flux (dedicated) + ├── api-tts.service.yaml # Cloud Run API — Aura TTS (dedicated) + ├── engine.service.yaml # Cloud Run Engine — STT (1x GPU, nova) + ├── engine-flux.service.yaml # Cloud Run Engine — Flux (1x GPU, dedicated) + └── engine-tts.service.yaml # Cloud Run Engine — TTS (2x GPU, Aura, dedicated) +``` + +## Getting help + +See the [Getting Help](../README.md#getting-help) section in the root of this repository for a list of resources to help you troubleshoot and resolve issues. diff --git a/cloud/google/cloud-run/config/api-flux.toml.tmpl b/cloud/google/cloud-run/config/api-flux.toml.tmpl new file mode 100644 index 0000000..ef6a8cf --- /dev/null +++ b/cloud/google/cloud-run/config/api-flux.toml.tmpl @@ -0,0 +1,101 @@ +### Deepgram API configuration for Google Cloud Run — Flux (dedicated) +### This file is a template. The deploy.sh script substitutes FLUX_ENGINE_SERVICE_URL +### before uploading to Secret Manager. +### +### This API instance routes requests exclusively to the Flux Engine. +### It must NOT be pointed at an STT or TTS engine. +### +### STT (nova) and TTS requests each require their own API instance: +### config/api.toml.tmpl — routes to deepgram-engine (nova STT) +### config/api-tts.toml.tmpl — routes to deepgram-engine-tts (Aura TTS) +### +### Keep in mind that all paths are in-container paths, and do not need to exist +### on the host machine. + + +### Configure license validation by passing in a DEEPGRAM_API_KEY environment variable. +### See https://developers.deepgram.com/docs/deploy-deepgram-services#credentials +[license] +server_url = ["https://license.deepgram.com"] + + +### Configure how the API will listen for your requests +[server] +### The base URL (prefix) for requests to the API. +base_url = "/v1" +### Listen on all interfaces inside the container. +host = "0.0.0.0" +### Cloud Run expects the container to listen on port 8080. +port = 8080 + +### How long to wait for a connection to a callback URL. +callback_conn_timeout = "1s" +### How long to wait for a response to a callback URL. +callback_timeout = "10s" + +### How long to wait for a connection to a fetch URL. +fetch_conn_timeout = "1s" +### How long to wait for a response to a fetch URL. +fetch_timeout = "60s" + + +### By default, the API listens over HTTP. Cloud Run terminates TLS at the load +### balancer, so no certificate configuration is needed here. +[server.https] +# cert_file = "/path/to/cert.pem" +# key_file = "/path/to/key.pem" + + +### Specify custom DNS resolution options. +[resolver] +### Specify custom domain name server(s). +### Format is "{IP} {PORT} {PROTOCOL (tcp or udp)}" +# nameservers = ["127.0.0.1 53 udp"] + +### If specifying a custom DNS nameserver, set the DNS TTL value. +# max_ttl = 10 + + +### Limit the number of active requests handled by a single API container. +### For Flux, use flux.max_streams in engine-flux.toml to control Engine-side +### concurrency. Set active_requests here to guard the API container itself. +[concurrency_limit] +# active_requests = + + +### Enable ancillary features +[features] +### Flux does not support diarization, entity detection, NER, or audio intelligence +### features. Only keep features that are compatible with Flux. +topic_detection = false +summarization = false +entity_detection = false +entity_redaction = false +format_entity_tags = false + +### TTS is not served by this API instance. +speak_streaming = false + +### Toggles usage data redaction; set to false to disable redaction of usage data; defaults to true if not present +# redact_usage = true # or false + +### Enable Flux turn-based streaming STT (v2 listen endpoint). +### This must be true for Flux requests to be accepted. +listen_v2 = true + + +### Configure the backend pool of speech engines. +### FLUX_ENGINE_SERVICE_URL is substituted at deploy time with the Flux Engine URL. +[[driver_pool.standard]] +### Cloud Run services are always served over HTTPS. +url = "FLUX_ENGINE_SERVICE_URL" +### Factor to increase the timeout by for each additional retry (exponential backoff). +timeout_backoff = 1.2 + +### Before attempting a retry, sleep for this long. +retry_sleep = "2s" +### Factor to increase the retry sleep by for each additional retry (exponential backoff). +retry_backoff = 1.6 + +### Maximum response to deserialize from Driver (in bytes). +max_response_size = 1073741824 # 1GB diff --git a/cloud/google/cloud-run/config/api-tts.toml.tmpl b/cloud/google/cloud-run/config/api-tts.toml.tmpl new file mode 100644 index 0000000..fc626be --- /dev/null +++ b/cloud/google/cloud-run/config/api-tts.toml.tmpl @@ -0,0 +1,99 @@ +### Deepgram API configuration for Google Cloud Run — Aura TTS (dedicated) +### This file is a template. The deploy.sh script substitutes TTS_ENGINE_SERVICE_URL +### before uploading to Secret Manager. +### +### This API instance routes requests exclusively to the TTS Engine (Aura/Aura-2). +### It must NOT be pointed at an STT or Flux engine. +### +### STT (nova) and Flux requests each require their own API instance: +### config/api.toml.tmpl — routes to deepgram-engine (nova STT) +### config/api-flux.toml.tmpl — routes to deepgram-engine-flux (Flux) +### +### Keep in mind that all paths are in-container paths, and do not need to exist +### on the host machine. + + +### Configure license validation by passing in a DEEPGRAM_API_KEY environment variable. +### See https://developers.deepgram.com/docs/deploy-deepgram-services#credentials +[license] +server_url = ["https://license.deepgram.com"] + + +### Configure how the API will listen for your requests +[server] +### The base URL (prefix) for requests to the API. +base_url = "/v1" +### Listen on all interfaces inside the container. +host = "0.0.0.0" +### Cloud Run expects the container to listen on port 8080. +port = 8080 + +### How long to wait for a connection to a callback URL. +callback_conn_timeout = "1s" +### How long to wait for a response to a callback URL. +callback_timeout = "10s" + +### How long to wait for a connection to a fetch URL. +fetch_conn_timeout = "1s" +### How long to wait for a response to a fetch URL. +fetch_timeout = "60s" + + +### By default, the API listens over HTTP. Cloud Run terminates TLS at the load +### balancer, so no certificate configuration is needed here. +[server.https] +# cert_file = "/path/to/cert.pem" +# key_file = "/path/to/key.pem" + + +### Specify custom DNS resolution options. +[resolver] +### Specify custom domain name server(s). +### Format is "{IP} {PORT} {PROTOCOL (tcp or udp)}" +# nameservers = ["127.0.0.1 53 udp"] + +### If specifying a custom DNS nameserver, set the DNS TTL value. +# max_ttl = 10 + + +### Limit the number of active requests handled by a single API container. +### If additional requests beyond the limit are sent, API will return +### a 429 HTTP status code. Default is no limit. +[concurrency_limit] +# active_requests = + + +### Enable ancillary features +[features] +### STT audio intelligence features are not applicable to TTS workloads. +topic_detection = false +summarization = false +entity_detection = false +entity_redaction = false +format_entity_tags = false + +### Enables streaming TTS via WebSocket (/v1/speak endpoint). +speak_streaming = true + +### Toggles usage data redaction; set to false to disable redaction of usage data; defaults to true if not present +# redact_usage = true # or false + +### Flux is not served by this API instance. +listen_v2 = false + + +### Configure the backend pool of speech engines. +### TTS_ENGINE_SERVICE_URL is substituted at deploy time with the TTS Engine URL. +[[driver_pool.standard]] +### Cloud Run services are always served over HTTPS. +url = "TTS_ENGINE_SERVICE_URL" +### Factor to increase the timeout by for each additional retry (exponential backoff). +timeout_backoff = 1.2 + +### Before attempting a retry, sleep for this long. +retry_sleep = "2s" +### Factor to increase the retry sleep by for each additional retry (exponential backoff). +retry_backoff = 1.6 + +### Maximum response to deserialize from Driver (in bytes). +max_response_size = 1073741824 # 1GB diff --git a/cloud/google/cloud-run/config/api.toml.tmpl b/cloud/google/cloud-run/config/api.toml.tmpl new file mode 100644 index 0000000..968c2a4 --- /dev/null +++ b/cloud/google/cloud-run/config/api.toml.tmpl @@ -0,0 +1,118 @@ +### Deepgram API configuration for Google Cloud Run — STT (nova-2 / nova-3) +### This file is a template. The deploy.sh script substitutes ENGINE_SERVICE_URL +### before uploading to Secret Manager. +### +### This API instance routes requests to the STT Engine (nova-2/nova-3) only. +### Flux and Aura TTS each require their own dedicated API instance: +### config/api-flux.toml.tmpl — routes to deepgram-engine-flux +### config/api-tts.toml.tmpl — routes to deepgram-engine-tts +### +### Keep in mind that all paths are in-container paths, and do not need to exist +### on the host machine. + + +### Configure license validation by passing in a DEEPGRAM_API_KEY environment variable. +### See https://developers.deepgram.com/docs/deploy-deepgram-services#credentials +[license] +server_url = ["https://license.deepgram.com"] + + +### Configure how the API will listen for your requests +[server] +### The base URL (prefix) for requests to the API. +base_url = "/v1" +### Listen on all interfaces inside the container. +host = "0.0.0.0" +### Cloud Run expects the container to listen on port 8080. +port = 8080 + +### How long to wait for a connection to a callback URL. +callback_conn_timeout = "1s" +### How long to wait for a response to a callback URL. +callback_timeout = "10s" + +### How long to wait for a connection to a fetch URL. +fetch_conn_timeout = "1s" +### How long to wait for a response to a fetch URL. +fetch_timeout = "60s" + + +### By default, the API listens over HTTP. Cloud Run terminates TLS at the load +### balancer, so no certificate configuration is needed here. +[server.https] +# cert_file = "/path/to/cert.pem" +# key_file = "/path/to/key.pem" + + +### Specify custom DNS resolution options. +[resolver] +### Specify custom domain name server(s). +### Format is "{IP} {PORT} {PROTOCOL (tcp or udp)}" +# nameservers = ["127.0.0.1 53 udp"] + +### If specifying a custom DNS nameserver, set the DNS TTL value. +# max_ttl = 10 + + +### Limit the number of active requests handled by a single API container. +### If additional requests beyond the limit are sent, API will return +### a 429 HTTP status code. Default is no limit. +[concurrency_limit] +# active_requests = + + +### Enable ancillary features +[features] +### Enables topic detection *if* a valid topic detection model is available +topic_detection = true # or false + +### Enables summarization *if* a valid summarization model is available +summarization = true # or false + +### Enables pre-recorded entity detection *if* a valid entity detection model is available +entity_detection = false # or true + +### Enables pre-recorded entity-based redaction *if* a valid entity detection model is available +entity_redaction = true # or false + +### Enables pre-recorded entity formatting *if* a valid NER model is available +format_entity_tags = true # or false + +### If API is receiving requests faster than Engine can process them, a request +### queue will form. By default, this queue is stored in memory. Under high load, +### the queue may grow too large and cause Out-Of-Memory errors. To avoid this, +### set a disk_buffer_path to buffer the overflow on the request queue to disk. +### +### WARN: This is only to temporarily buffer requests during high load. +### If there is not enough Engine capacity to process the queued requests over time, +### the queue (and response time) will grow indefinitely. +# disk_buffer_path = "/path/to/disk/buffer/directory" + +### Streaming TTS is disabled on the STT API instance. +### Use api-tts.toml.tmpl / deepgram-api-tts for Aura TTS requests. +speak_streaming = false + +### Toggles usage data redaction; set to false to disable redaction of usage data; defaults to true if not present +# redact_usage = true # or false + +### Flux is disabled on the STT API instance. +### Use api-flux.toml.tmpl / deepgram-api-flux for Flux requests. +listen_v2 = false + + +### Configure the backend pool of speech engines. +### The API will load-balance among drivers in the standard pool. +### ENGINE_SERVICE_URL is substituted by deploy.sh with the Engine Cloud Run service URL. +[[driver_pool.standard]] +### Cloud Run services are always served over HTTPS. +url = "ENGINE_SERVICE_URL" +### Factor to increase the timeout by for each additional retry (exponential backoff). +timeout_backoff = 1.2 + +### Before attempting a retry, sleep for this long. +retry_sleep = "2s" +### Factor to increase the retry sleep by for each additional retry (exponential backoff). +retry_backoff = 1.6 + +### Maximum response to deserialize from Driver (in bytes). +max_response_size = 1073741824 # 1GB diff --git a/cloud/google/cloud-run/config/engine-flux.toml b/cloud/google/cloud-run/config/engine-flux.toml new file mode 100644 index 0000000..5ab8819 --- /dev/null +++ b/cloud/google/cloud-run/config/engine-flux.toml @@ -0,0 +1,88 @@ +### Deepgram Engine configuration for Google Cloud Run — Flux (dedicated) +### +### This configuration is exclusively for Flux turn-based streaming STT. +### DO NOT add nova, TTS, diarizer, NER, or any other models to this engine. +### Flux allocates all GPU memory for streams at startup; mixing models causes +### CUDA out-of-memory errors and crashes. +### +### Requires container image release-251015 or later. +### See: https://developers.deepgram.com/docs/flux-self-hosted +### +### Keep in mind that all paths are in-container paths and do not need to exist +### on the host machine. + +### max_active_requests does NOT apply to Flux. Flux concurrency is controlled +### exclusively by flux.max_streams below. + + +### Configure license validation by passing in a DEEPGRAM_API_KEY environment variable. +### See https://developers.deepgram.com/docs/deploy-deepgram-services#credentials +[license] +server_url = ["https://license.deepgram.com"] + + +### Configure the server to listen for requests from the API. +[server] +### Listen on all interfaces inside the container. +host = "0.0.0.0" +### Cloud Run expects the container to listen on port 8080. +port = 8080 + + +### Metrics server for Prometheus-compatible scraping. +### Note: In Cloud Run, port 9991 is not exposed by the load balancer. +### Metrics are accessible only within the container or via Cloud Run sidecar patterns. +[metrics_server] +host = "0.0.0.0" +port = 9991 + + +[model_manager] +### Flux must be the only model loaded. Do not configure additional models here. +### The Flux model file must be present in the models directory at startup. +search_paths = ["/models"] + + +### Enable Flux turn-based streaming STT +[flux] +enabled = true + +### REQUIRED for production: set max_streams explicitly based on GPU type. +### Flux automatically derives a value from available GPU memory, but the +### auto-calculated value may be too high under sustained load, leading to: +### - Voice agents stopping or experiencing long delays +### - Calls dropped after sustained concurrent usage +### - "audio_window_end increased by more than 3 frames" errors in API logs +### +### Contact your Deepgram Account Representative for the recommended value +### for your GPU type (e.g. NVIDIA L4). +### +### Monitor stream utilization via Engine metrics: +### flux_used_streams — active Flux streams +### flux_max_streams — configured maximum +### flux_fraction_streams — fraction of capacity in use (scale out at ~0.8) +# max_streams = 0 + + +### Size of audio chunks to process in seconds. +[chunking.batch] +# min_duration = +# max_duration = +[chunking.streaming] +# min_duration = +# max_duration = + + +### Pre-load the Flux model into GPU memory at startup. Flux pre-allocates GPU +### memory for all configured streams on startup regardless of this setting. +### blocking = true prevents Cloud Run from routing requests until the model is +### ready, which is strongly recommended to avoid in-flight failures. +[preload_models] +blocking = true + + +[health] +### Fail on startup if no GPU is detected. +### In Cloud Run GPU deployments this should always be true. +### For Flux specifically, a CPU fallback produces OOM errors and high latency. +gpu_required = true diff --git a/cloud/google/cloud-run/config/engine.toml b/cloud/google/cloud-run/config/engine.toml new file mode 100644 index 0000000..6fbf82e --- /dev/null +++ b/cloud/google/cloud-run/config/engine.toml @@ -0,0 +1,109 @@ +### Deepgram Engine configuration for Google Cloud Run — STT (nova-2 / nova-3) +### +### This configuration is for Speech-to-Text workloads only. +### An NVIDIA L4 GPU can host up to 5 concurrent nova-2/nova-3 model files. +### +### For Flux or Aura TTS workloads, use their dedicated config files: +### config/engine-flux.toml — Flux turn-based streaming STT +### (TTS engine config is provided by your Deepgram Account Representative) +### +### Keep in mind that all paths are in-container paths and do not need to exist +### on the host machine. + +### Limit the number of active requests handled by a single Engine container. +### Engine will reject additional requests from API beyond this limit, and the +### API container will continue with the retry logic configured in api.toml. +### +### For an NVIDIA L4 GPU, a reasonable starting value is 8–16 for STT streaming. +### Tune this based on your model and workload. +### NOTE: This setting does NOT apply to Flux streams — use flux.max_streams instead. +# max_active_requests = + + +### Configure license validation by passing in a DEEPGRAM_API_KEY environment variable. +### See https://developers.deepgram.com/docs/deploy-deepgram-services#credentials +[license] +server_url = ["https://license.deepgram.com"] + + +### Configure the server to listen for requests from the API. +[server] +### Listen on all interfaces inside the container. +host = "0.0.0.0" +### Cloud Run expects the container to listen on port 8080. +port = 8080 + + +### Metrics server for Prometheus-compatible scraping. +### Note: In Cloud Run, port 9991 is not exposed by the load balancer. +### Metrics are accessible only within the container or via Cloud Run sidecar patterns. +[metrics_server] +host = "0.0.0.0" +port = 9991 + + +[model_manager] +### The number of models to have concurrently loaded in system memory. +### If managing a deployment with dozens of models this setting will +### help prevent instances where models consume too much memory and +### offload the models to disk as needed on a least-recently-used basis. +### +### For an NVIDIA L4 GPU running nova-2/nova-3, up to 5 models can be loaded +### concurrently on the same GPU. Set this to 5 or fewer to prevent OOM errors +### when hosting multiple nova variants. +### +### The default is no limit. +# max_concurrently_loaded_models = 5 + +### Inference model search path — matches the GCS FUSE mount point set in engine.service.yaml. +search_paths = ["/models"] + + +### Enable ancillary features +[features] +### Allow multichannel requests by setting this to true, set to false to disable +multichannel = true # or false +### Enables language detection *if* a valid language detection model is available +language_detection = true # or false +### Enables streaming entity formatting *if* a valid NER model is available +streaming_ner = true # or false +### Enables use of 36-language detection model +use_v2_language_detection = false + + +### Flux must NOT be enabled in this STT engine configuration. +### +### Flux requires a fully dedicated Engine instance with no other models present. +### Enabling Flux alongside nova-2/nova-3 models causes CUDA out-of-memory errors +### and will crash the Engine. Use config/engine-flux.toml with the dedicated +### engine-flux.service.yaml Cloud Run service instead. +### +### See: https://developers.deepgram.com/docs/flux-self-hosted +[flux] +enabled = false + + +### Size of audio chunks to process in seconds. +[chunking.batch] +# min_duration = +# max_duration = +[chunking.streaming] +# min_duration = +# max_duration = + + +### How often to return interim results, in seconds. Default is 1.0s. +# step = 1.0 + + +### Pre-load models into GPU memory at startup to reduce request latency. +### Blocking startup on preload prevents Cloud Run from sending traffic until models +### are ready, which is recommended for GPU deployments. +[preload_models] +blocking = true + + +[health] +### Fail on startup if no GPU is detected. +### In Cloud Run GPU deployments this should always be true. +gpu_required = true # or false diff --git a/cloud/google/cloud-run/deploy.sh b/cloud/google/cloud-run/deploy.sh new file mode 100755 index 0000000..d05f972 --- /dev/null +++ b/cloud/google/cloud-run/deploy.sh @@ -0,0 +1,520 @@ +#!/usr/bin/env bash +# deploy.sh — Deploy Deepgram self-hosted to Google Cloud Run with NVIDIA GPU +# +# Usage: +# cp .env.example .env && $EDITOR .env +# ./deploy.sh +# +# Prerequisites: +# - gcloud CLI installed and authenticated (gcloud auth login) +# - envsubst installed (part of gettext; brew install gettext on macOS) +# - Models uploaded to a GCS bucket (see README.md) + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ── Load .env if present ─────────────────────────────────────────────────────── +if [[ -f "${SCRIPT_DIR}/.env" ]]; then + # shellcheck disable=SC1091 + set -a; source "${SCRIPT_DIR}/.env"; set +a +fi + +# ── Required variables ───────────────────────────────────────────────────────── +: "${GCP_PROJECT_ID:?'GCP_PROJECT_ID is required. Set it in .env or export it.'}" +: "${DEEPGRAM_MODELS_BUCKET:?'DEEPGRAM_MODELS_BUCKET is required. Set it in .env or export it.'}" +: "${DEEPGRAM_API_KEY:?'DEEPGRAM_API_KEY is required. Set it in .env or export it.'}" + +# ── Defaults ─────────────────────────────────────────────────────────────────── +GCP_REGION="${GCP_REGION:-us-central1}" +IMAGE_TAG="${IMAGE_TAG:-release-260305}" +# Registry hosting Deepgram container images. When QUAY_USERNAME is provided, +# deploy.sh creates an Artifact Registry remote repo that proxies quay.io with +# credentials and sets IMAGE_REGISTRY to the AR path automatically. +# You may also set IMAGE_REGISTRY directly to skip AR repo creation. +IMAGE_REGISTRY="${IMAGE_REGISTRY:-}" +AR_REPO_NAME="${AR_REPO_NAME:-deepgram-quay}" +QUAY_PASSWORD_SECRET="${QUAY_PASSWORD_SECRET:-deepgram-quay-password}" +SERVICE_ACCOUNT_NAME="${SERVICE_ACCOUNT_NAME:-deepgram-cloud-run}" +DEEPGRAM_API_KEY_SECRET="${DEEPGRAM_API_KEY_SECRET:-deepgram-api-key}" +ENGINE_CONFIG_SECRET="deepgram-engine-config" +API_CONFIG_SECRET="deepgram-api-config" +ENGINE_SERVICE_NAME="${ENGINE_SERVICE_NAME:-deepgram-engine}" +API_SERVICE_NAME="${API_SERVICE_NAME:-deepgram-api}" +# VPC network for Cloud Run Direct VPC Egress. +# Set VPC_NETWORK to the name of an existing VPC to skip creation. +# Set VPC_SUBNETWORK to the name of an existing subnet in GCP_REGION to skip creation. +VPC_NETWORK="${VPC_NETWORK:-deepgram-vpc}" +VPC_SUBNETWORK="${VPC_SUBNETWORK:-deepgram-subnet}" +VPC_SUBNET_RANGE="${VPC_SUBNET_RANGE:-10.8.0.0/24}" +# Proxy-only subnet required by the internal HTTP load balancer. +VPC_PROXY_SUBNET="${VPC_PROXY_SUBNET:-deepgram-proxy-subnet}" +VPC_PROXY_SUBNET_RANGE="${VPC_PROXY_SUBNET_RANGE:-10.8.1.0/24}" +# Internal HTTP load balancer in front of the Engine. +ENGINE_LB_NAME="${ENGINE_LB_NAME:-deepgram-engine-lb}" +ENGINE_LB_IP_NAME="${ENGINE_LB_IP_NAME:-deepgram-engine-lb-ip}" + +SERVICE_ACCOUNT_EMAIL="${SERVICE_ACCOUNT_NAME}@${GCP_PROJECT_ID}.iam.gserviceaccount.com" + +# Export for envsubst +export GCP_PROJECT_ID GCP_REGION IMAGE_TAG IMAGE_REGISTRY MODELS_BUCKET="${DEEPGRAM_MODELS_BUCKET}" +export SERVICE_ACCOUNT_EMAIL VPC_NETWORK VPC_SUBNETWORK + +# ── Helpers ──────────────────────────────────────────────────────────────────── +log() { echo " [deploy] $*"; } +step() { echo; echo "▶ $*"; } +die() { echo "✖ ERROR: $*" >&2; exit 1; } + +require_cmd() { + command -v "$1" >/dev/null 2>&1 || die "'$1' is not installed. $2" +} + +secret_exists() { + gcloud secrets describe "$1" \ + --project="${GCP_PROJECT_ID}" \ + --quiet 2>/dev/null +} + +create_or_update_secret() { + local secret_name="$1" + local secret_value="$2" + + if secret_exists "${secret_name}"; then + log "Updating existing secret: ${secret_name}" + echo -n "${secret_value}" | gcloud secrets versions add "${secret_name}" \ + --project="${GCP_PROJECT_ID}" \ + --data-file=- + else + log "Creating new secret: ${secret_name}" + echo -n "${secret_value}" | gcloud secrets create "${secret_name}" \ + --project="${GCP_PROJECT_ID}" \ + --replication-policy=automatic \ + --data-file=- + fi +} + +create_or_update_secret_from_file() { + local secret_name="$1" + local file_path="$2" + + if secret_exists "${secret_name}"; then + log "Updating existing secret: ${secret_name}" + gcloud secrets versions add "${secret_name}" \ + --project="${GCP_PROJECT_ID}" \ + --data-file="${file_path}" + else + log "Creating new secret: ${secret_name}" + gcloud secrets create "${secret_name}" \ + --project="${GCP_PROJECT_ID}" \ + --replication-policy=automatic \ + --data-file="${file_path}" + fi +} + +# ── Preflight checks ─────────────────────────────────────────────────────────── +step "Checking prerequisites" +require_cmd gcloud "Install from https://cloud.google.com/sdk/docs/install" +require_cmd envsubst "Install gettext: brew install gettext (macOS) or apt-get install gettext (Linux)" + +gcloud config set project "${GCP_PROJECT_ID}" --quiet +log "Project: ${GCP_PROJECT_ID}" +log "Region: ${GCP_REGION}" +log "Models bucket: gs://${DEEPGRAM_MODELS_BUCKET}" + +# Verify the models bucket exists +if ! gsutil ls "gs://${DEEPGRAM_MODELS_BUCKET}" &>/dev/null; then + die "GCS bucket 'gs://${DEEPGRAM_MODELS_BUCKET}' does not exist or is not accessible. + Create it and upload your models before deploying. See README.md for guidance." +fi + +# ── Quay.io credentials ──────────────────────────────────────────────────────── +# Prompt for quay.io credentials if not already set in the environment. +# These are used to authenticate the Artifact Registry remote repo so it can +# pull private Deepgram images from quay.io on behalf of Cloud Run. +if [[ -z "${QUAY_USERNAME:-}" ]]; then + read -r -p " quay.io username: " QUAY_USERNAME +fi +if [[ -z "${QUAY_PASSWORD:-}" ]]; then + read -r -s -p " quay.io password / token: " QUAY_PASSWORD + echo "" +fi +[[ -n "${QUAY_USERNAME}" ]] || die "quay.io username is required." +[[ -n "${QUAY_PASSWORD}" ]] || die "quay.io password is required." + +# ── Enable required APIs ─────────────────────────────────────────────────────── +step "Enabling required Google Cloud APIs" +gcloud services enable \ + run.googleapis.com \ + secretmanager.googleapis.com \ + storage.googleapis.com \ + iam.googleapis.com \ + artifactregistry.googleapis.com \ + compute.googleapis.com \ + --project="${GCP_PROJECT_ID}" \ + --quiet +log "APIs enabled." + +# ── Service account ──────────────────────────────────────────────────────────── +step "Setting up service account: ${SERVICE_ACCOUNT_EMAIL}" +if ! gcloud iam service-accounts describe "${SERVICE_ACCOUNT_EMAIL}" \ + --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + gcloud iam service-accounts create "${SERVICE_ACCOUNT_NAME}" \ + --project="${GCP_PROJECT_ID}" \ + --display-name="Deepgram Cloud Run service account" \ + --quiet + log "Service account created." +else + log "Service account already exists." +fi + +# Wait for service account to propagate before applying IAM bindings. +# GCP can take a few seconds to make a newly created SA visible to IAM. +for i in $(seq 1 10); do + if gcloud iam service-accounts describe "${SERVICE_ACCOUNT_EMAIL}" \ + --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + break + fi + log "Waiting for service account to propagate (attempt ${i}/10)..." + sleep 5 +done + +# Grant Secret Manager access (read secrets at runtime) +gcloud projects add-iam-policy-binding "${GCP_PROJECT_ID}" \ + --member="serviceAccount:${SERVICE_ACCOUNT_EMAIL}" \ + --role="roles/secretmanager.secretAccessor" \ + --quiet >/dev/null + +# Grant GCS read access (models bucket via GCS FUSE) +gsutil iam ch \ + "serviceAccount:${SERVICE_ACCOUNT_EMAIL}:roles/storage.objectViewer" \ + "gs://${DEEPGRAM_MODELS_BUCKET}" + +log "IAM permissions configured." + +# ── VPC network ──────────────────────────────────────────────────────────────── +step "Configuring VPC network: ${VPC_NETWORK}" + +if gcloud compute networks describe "${VPC_NETWORK}" \ + --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "Using existing VPC: ${VPC_NETWORK}" +else + log "Creating VPC: ${VPC_NETWORK}" + gcloud compute networks create "${VPC_NETWORK}" \ + --subnet-mode=custom \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +if gcloud compute networks subnets describe "${VPC_SUBNETWORK}" \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "Using existing subnet: ${VPC_SUBNETWORK} (${GCP_REGION})" +else + log "Creating subnet: ${VPC_SUBNETWORK} (${GCP_REGION}, ${VPC_SUBNET_RANGE})" + gcloud compute networks subnets create "${VPC_SUBNETWORK}" \ + --network="${VPC_NETWORK}" \ + --region="${GCP_REGION}" \ + --range="${VPC_SUBNET_RANGE}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +log "VPC ready: ${VPC_NETWORK} / ${VPC_SUBNETWORK}" + +# ── Artifact Registry remote repo (quay.io proxy) ───────────────────────────── +step "Configuring Artifact Registry remote repo: ${AR_REPO_NAME}" + +# Store the quay.io password in Secret Manager so AR can use it to authenticate. +create_or_update_secret "${QUAY_PASSWORD_SECRET}" "${QUAY_PASSWORD}" +log "quay.io password stored in secret: ${QUAY_PASSWORD_SECRET}" + +QUAY_PASSWORD_SECRET_VERSION="projects/${GCP_PROJECT_ID}/secrets/${QUAY_PASSWORD_SECRET}/versions/latest" +AR_REPO_HOST="${GCP_REGION}-docker.pkg.dev" +AR_REPO_PATH="${AR_REPO_HOST}/${GCP_PROJECT_ID}/${AR_REPO_NAME}/deepgram" + +# Grant Artifact Registry the ability to read the quay.io password secret. +PROJECT_NUMBER="$(gcloud projects describe "${GCP_PROJECT_ID}" --format='value(projectNumber)')" +AR_SERVICE_AGENT="service-${PROJECT_NUMBER}@gcp-sa-artifactregistry.iam.gserviceaccount.com" +gcloud secrets add-iam-policy-binding "${QUAY_PASSWORD_SECRET}" \ + --project="${GCP_PROJECT_ID}" \ + --member="serviceAccount:${AR_SERVICE_AGENT}" \ + --role="roles/secretmanager.secretAccessor" \ + --quiet >/dev/null + +# Create or update the AR remote repo pointing at quay.io with credentials. +if gcloud artifacts repositories describe "${AR_REPO_NAME}" \ + --location="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "Updating existing AR remote repo with quay.io credentials..." + gcloud artifacts repositories update "${AR_REPO_NAME}" \ + --location="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --remote-username="${QUAY_USERNAME}" \ + --remote-password-secret-version="${QUAY_PASSWORD_SECRET_VERSION}" \ + --quiet +else + log "Creating AR remote repo: ${AR_REPO_NAME}..." + gcloud artifacts repositories create "${AR_REPO_NAME}" \ + --repository-format=docker \ + --location="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --description="Remote proxy for quay.io/deepgram" \ + --mode=remote-repository \ + --remote-repo-config-desc="quay.io" \ + --remote-docker-repo="https://quay.io" \ + --remote-username="${QUAY_USERNAME}" \ + --remote-password-secret-version="${QUAY_PASSWORD_SECRET_VERSION}" \ + --quiet +fi + +# Grant Cloud Run's service agent read access to pull images from the AR repo. +CR_SERVICE_AGENT="service-${PROJECT_NUMBER}@serverless-robot-prod.iam.gserviceaccount.com" +gcloud artifacts repositories add-iam-policy-binding "${AR_REPO_NAME}" \ + --location="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --member="serviceAccount:${CR_SERVICE_AGENT}" \ + --role="roles/artifactregistry.reader" \ + --quiet >/dev/null + +# Set IMAGE_REGISTRY to the AR remote repo path for use by envsubst in service YAMLs. +IMAGE_REGISTRY="${AR_REPO_PATH}" +export IMAGE_REGISTRY +log "Image registry: ${IMAGE_REGISTRY}" + +# ── Secrets ──────────────────────────────────────────────────────────────────── +step "Storing secrets in Secret Manager" + +# Deepgram API key +create_or_update_secret "${DEEPGRAM_API_KEY_SECRET}" "${DEEPGRAM_API_KEY}" +log "API key stored in secret: ${DEEPGRAM_API_KEY_SECRET}" + +# Engine config — upload the static engine.toml directly +create_or_update_secret_from_file \ + "${ENGINE_CONFIG_SECRET}" \ + "${SCRIPT_DIR}/config/engine.toml" +log "Engine config stored in secret: ${ENGINE_CONFIG_SECRET}" + +# ── Deploy Engine service ────────────────────────────────────────────────────── +step "Deploying Engine service (GPU): ${ENGINE_SERVICE_NAME}" +log "Substituting template variables in engine.service.yaml..." +ENGINE_YAML="$(envsubst < "${SCRIPT_DIR}/services/engine.service.yaml")" + +log "Applying Engine service definition..." +echo "${ENGINE_YAML}" | gcloud run services replace - \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet + +# Allow unauthenticated invocations within internal ingress. +# The Engine is only reachable by other Cloud Run services in the same project. +gcloud run services add-iam-policy-binding "${ENGINE_SERVICE_NAME}" \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --member="allUsers" \ + --role="roles/run.invoker" \ + --quiet + +log "Engine service deployed." + +# ── Internal HTTP load balancer for Engine ───────────────────────────────────── +step "Configuring internal HTTP load balancer: ${ENGINE_LB_NAME}" + +ENGINE_LB_NEG="${ENGINE_LB_NAME}-neg" +ENGINE_LB_BACKEND="${ENGINE_LB_NAME}-backend" +ENGINE_LB_URL_MAP="${ENGINE_LB_NAME}-url-map" +ENGINE_LB_HTTP_PROXY="${ENGINE_LB_NAME}-http-proxy" + +# Proxy-only subnet — required by INTERNAL_MANAGED HTTP load balancers. +if gcloud compute networks subnets describe "${VPC_PROXY_SUBNET}" \ + --region="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "Proxy-only subnet exists: ${VPC_PROXY_SUBNET}" +else + log "Creating proxy-only subnet: ${VPC_PROXY_SUBNET} (${VPC_PROXY_SUBNET_RANGE})" + gcloud compute networks subnets create "${VPC_PROXY_SUBNET}" \ + --purpose=REGIONAL_MANAGED_PROXY \ + --role=ACTIVE \ + --network="${VPC_NETWORK}" \ + --region="${GCP_REGION}" \ + --range="${VPC_PROXY_SUBNET_RANGE}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +# Reserve a static internal IP for the LB forwarding rule. +if gcloud compute addresses describe "${ENGINE_LB_IP_NAME}" \ + --region="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "LB IP already reserved: ${ENGINE_LB_IP_NAME}" +else + log "Reserving internal IP: ${ENGINE_LB_IP_NAME}" + gcloud compute addresses create "${ENGINE_LB_IP_NAME}" \ + --region="${GCP_REGION}" \ + --subnet="${VPC_SUBNETWORK}" \ + --purpose=SHARED_LOADBALANCER_VIP \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +ENGINE_LB_IP="$(gcloud compute addresses describe "${ENGINE_LB_IP_NAME}" \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --format='value(address)')" +log "Engine LB IP: ${ENGINE_LB_IP}" + +# Serverless NEG — points directly at the Engine Cloud Run service. +if gcloud compute network-endpoint-groups describe "${ENGINE_LB_NEG}" \ + --region="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "Serverless NEG exists: ${ENGINE_LB_NEG}" +else + log "Creating serverless NEG: ${ENGINE_LB_NEG}" + gcloud compute network-endpoint-groups create "${ENGINE_LB_NEG}" \ + --region="${GCP_REGION}" \ + --network-endpoint-type=serverless \ + --cloud-run-service="${ENGINE_SERVICE_NAME}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +# Backend service (HTTP, internal). +if gcloud compute backend-services describe "${ENGINE_LB_BACKEND}" \ + --region="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "Backend service exists: ${ENGINE_LB_BACKEND}" +else + log "Creating backend service: ${ENGINE_LB_BACKEND}" + gcloud compute backend-services create "${ENGINE_LB_BACKEND}" \ + --load-balancing-scheme=INTERNAL_MANAGED \ + --protocol=HTTP \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet + gcloud compute backend-services add-backend "${ENGINE_LB_BACKEND}" \ + --region="${GCP_REGION}" \ + --network-endpoint-group="${ENGINE_LB_NEG}" \ + --network-endpoint-group-region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +# URL map. +if gcloud compute url-maps describe "${ENGINE_LB_URL_MAP}" \ + --region="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "URL map exists: ${ENGINE_LB_URL_MAP}" +else + log "Creating URL map: ${ENGINE_LB_URL_MAP}" + gcloud compute url-maps create "${ENGINE_LB_URL_MAP}" \ + --default-service="${ENGINE_LB_BACKEND}" \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +# HTTP target proxy — no TLS, plain HTTP only. +if gcloud compute target-http-proxies describe "${ENGINE_LB_HTTP_PROXY}" \ + --region="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "HTTP proxy exists: ${ENGINE_LB_HTTP_PROXY}" +else + log "Creating HTTP target proxy: ${ENGINE_LB_HTTP_PROXY}" + gcloud compute target-http-proxies create "${ENGINE_LB_HTTP_PROXY}" \ + --url-map="${ENGINE_LB_URL_MAP}" \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +# Forwarding rule — binds the reserved internal IP on port 80. +if gcloud compute forwarding-rules describe "${ENGINE_LB_NAME}" \ + --region="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "Forwarding rule exists: ${ENGINE_LB_NAME}" +else + log "Creating forwarding rule: ${ENGINE_LB_NAME} -> ${ENGINE_LB_IP}:80" + gcloud compute forwarding-rules create "${ENGINE_LB_NAME}" \ + --load-balancing-scheme=INTERNAL_MANAGED \ + --network="${VPC_NETWORK}" \ + --subnet="${VPC_SUBNETWORK}" \ + --address="${ENGINE_LB_IP_NAME}" \ + --region="${GCP_REGION}" \ + --target-http-proxy="${ENGINE_LB_HTTP_PROXY}" \ + --target-http-proxy-region="${GCP_REGION}" \ + --ports=80 \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +# Firewall rule — allow VPC subnet traffic to reach the LB on port 80. +if gcloud compute firewall-rules describe deepgram-allow-internal-http \ + --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + log "Firewall rule exists: deepgram-allow-internal-http" +else + log "Creating firewall rule: allow TCP:80 from ${VPC_SUBNET_RANGE}" + gcloud compute firewall-rules create deepgram-allow-internal-http \ + --network="${VPC_NETWORK}" \ + --allow=tcp:80 \ + --source-ranges="${VPC_SUBNET_RANGE}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet +fi + +log "Engine LB ready: http://${ENGINE_LB_IP}" + +# The API talks to the Engine via the internal HTTP LB, not the Cloud Run URL. +ENGINE_SERVICE_URL="http://${ENGINE_LB_IP}" +log "Engine URL (for api.toml): ${ENGINE_SERVICE_URL}" + +# ── Generate and store API config ────────────────────────────────────────────── +step "Generating API configuration" +API_TOML="$(sed "s|ENGINE_SERVICE_URL|${ENGINE_SERVICE_URL}|g" \ + "${SCRIPT_DIR}/config/api.toml.tmpl")" + +# Write to a temp file so we can pass it to the secret command +TMPFILE="$(mktemp /tmp/api.toml.XXXXXX)" +trap 'rm -f "${TMPFILE}"' EXIT +echo "${API_TOML}" > "${TMPFILE}" + +create_or_update_secret_from_file "${API_CONFIG_SECRET}" "${TMPFILE}" +log "API config stored in secret: ${API_CONFIG_SECRET} (Engine URL: ${ENGINE_SERVICE_URL})" + +# ── Deploy API service ───────────────────────────────────────────────────────── +step "Deploying API service: ${API_SERVICE_NAME}" +log "Substituting template variables in api.service.yaml..." +API_YAML="$(envsubst < "${SCRIPT_DIR}/services/api.service.yaml")" + +log "Applying API service definition..." +echo "${API_YAML}" | gcloud run services replace - \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet + +# Allow unauthenticated public access to the API service +gcloud run services add-iam-policy-binding "${API_SERVICE_NAME}" \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --member="allUsers" \ + --role="roles/run.invoker" \ + --quiet + +log "API service deployed." + +# Retrieve the API service URL +API_SERVICE_URL="$(gcloud run services describe "${API_SERVICE_NAME}" \ + --platform managed \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --format='value(status.url)')" + +# ── Summary ──────────────────────────────────────────────────────────────────── +echo +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Deployment complete!" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo +echo " API endpoint: ${API_SERVICE_URL}/v1" +echo " Engine endpoint: ${ENGINE_SERVICE_URL} (internal only)" +echo +echo " Test with:" +echo " curl -X POST \"${API_SERVICE_URL}/v1/listen\" \\" +echo " -H \"Authorization: Token \${DEEPGRAM_API_KEY}\" \\" +echo " -H \"Content-Type: audio/wav\" \\" +echo " --data-binary @audio.wav" +echo +echo " Logs:" +echo " gcloud run services logs read ${API_SERVICE_NAME} --region=${GCP_REGION}" +echo " gcloud run services logs read ${ENGINE_SERVICE_NAME} --region=${GCP_REGION}" +echo diff --git a/cloud/google/cloud-run/services/api-flux.service.yaml b/cloud/google/cloud-run/services/api-flux.service.yaml new file mode 100644 index 0000000..30a0cf4 --- /dev/null +++ b/cloud/google/cloud-run/services/api-flux.service.yaml @@ -0,0 +1,100 @@ +# Deepgram API — Cloud Run service definition (Flux / dedicated) +# +# This API service routes requests exclusively to the Flux Engine. +# It must NOT be shared with STT (nova) or TTS workloads — each workload +# type requires its own dedicated API+Engine pair: +# services/api.service.yaml — STT nova-2/nova-3 +# services/api-tts.service.yaml — Aura TTS +# +# This file is a template. Variables in ${VAR} notation are substituted +# by deploy.sh using envsubst before being applied with: +# gcloud run services replace - --region=${REGION} --project=${PROJECT_ID} + +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: deepgram-api-flux + annotations: + # Allow public internet traffic. Clients send Flux streaming requests here. + run.googleapis.com/ingress: all +spec: + template: + metadata: + annotations: + run.googleapis.com/execution-environment: gen2 + + # The API is CPU-bound. Keep at least one warm instance to avoid + # connection delays for latency-sensitive Flux voice agent traffic. + autoscaling.knative.dev/minScale: "1" + + # Maximum number of API replicas. Scale based on concurrent Flux connection volume. + autoscaling.knative.dev/maxScale: "10" + + run.googleapis.com/network-interfaces: '[{"network":"${VPC_NETWORK}","subnetwork":"${VPC_SUBNETWORK}"}]' + run.googleapis.com/vpc-access-egress: private-ranges-only + spec: + # Service account used to read config from Secret Manager. + serviceAccountName: ${SERVICE_ACCOUNT_EMAIL} + + # Match the Engine timeout to allow long-running Flux sessions to complete. + timeoutSeconds: 3600 + + containers: + - image: "${IMAGE_REGISTRY}/self-hosted-api:${IMAGE_TAG}" + args: ["-v", "serve", "/config/api.toml"] + + ports: + - name: h2c + containerPort: 8080 + + env: + # Deepgram self-hosted API key, read from Secret Manager at runtime. + - name: DEEPGRAM_API_KEY + valueFrom: + secretKeyRef: + name: deepgram-api-key + key: latest + + - name: DEEPGRAM_DEPLOYMENT_ORCHESTRATOR + value: cloud-run + + resources: + limits: + # The API is lightweight. Adjust based on observed memory usage. + cpu: "2" + memory: 4Gi + + volumeMounts: + # api-flux.toml is mounted from Secret Manager into /config/api.toml. + # The config already contains the Flux Engine service URL, substituted + # by deploy.sh when the secret was created. + - name: api-config + mountPath: /config + readOnly: true + + startupProbe: + httpGet: + path: /v1/status + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 12 + timeoutSeconds: 3 + + livenessProbe: + httpGet: + path: /v1/status + port: 8080 + periodSeconds: 30 + failureThreshold: 3 + timeoutSeconds: 5 + + volumes: + # Mount api-flux.toml from Secret Manager. + # Create this secret with: gcloud secrets create deepgram-api-flux-config + - name: api-config + secret: + secretName: deepgram-api-flux-config + items: + - key: latest + path: api.toml diff --git a/cloud/google/cloud-run/services/api-tts.service.yaml b/cloud/google/cloud-run/services/api-tts.service.yaml new file mode 100644 index 0000000..5a23919 --- /dev/null +++ b/cloud/google/cloud-run/services/api-tts.service.yaml @@ -0,0 +1,100 @@ +# Deepgram API — Cloud Run service definition (Aura TTS / dedicated) +# +# This API service routes requests exclusively to the TTS Engine (Aura/Aura-2). +# It must NOT be shared with STT (nova) or Flux workloads — each workload +# type requires its own dedicated API+Engine pair: +# services/api.service.yaml — STT nova-2/nova-3 +# services/api-flux.service.yaml — Flux turn-based streaming STT +# +# This file is a template. Variables in ${VAR} notation are substituted +# by deploy.sh using envsubst before being applied with: +# gcloud run services replace - --region=${REGION} --project=${PROJECT_ID} + +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: deepgram-api-tts + annotations: + # Allow public internet traffic. Clients send TTS synthesis requests here. + run.googleapis.com/ingress: all +spec: + template: + metadata: + annotations: + run.googleapis.com/execution-environment: gen2 + + # The API is CPU-bound. Keep at least one warm instance to avoid + # cold-start latency on real-time TTS voicebot traffic. + autoscaling.knative.dev/minScale: "1" + + # Maximum number of API replicas. Scale based on concurrent TTS request volume. + autoscaling.knative.dev/maxScale: "10" + + run.googleapis.com/network-interfaces: '[{"network":"${VPC_NETWORK}","subnetwork":"${VPC_SUBNETWORK}"}]' + run.googleapis.com/vpc-access-egress: private-ranges-only + spec: + # Service account used to read config from Secret Manager. + serviceAccountName: ${SERVICE_ACCOUNT_EMAIL} + + # Match the Engine timeout to allow long TTS synthesis jobs to complete. + timeoutSeconds: 3600 + + containers: + - image: "${IMAGE_REGISTRY}/self-hosted-api:${IMAGE_TAG}" + args: ["-v", "serve", "/config/api.toml"] + + ports: + - name: h2c + containerPort: 8080 + + env: + # Deepgram self-hosted API key, read from Secret Manager at runtime. + - name: DEEPGRAM_API_KEY + valueFrom: + secretKeyRef: + name: deepgram-api-key + key: latest + + - name: DEEPGRAM_DEPLOYMENT_ORCHESTRATOR + value: cloud-run + + resources: + limits: + # The API is lightweight. Adjust based on observed memory usage. + cpu: "2" + memory: 4Gi + + volumeMounts: + # api-tts.toml is mounted from Secret Manager into /config/api.toml. + # The config already contains the TTS Engine service URL, substituted + # by deploy.sh when the secret was created. + - name: api-config + mountPath: /config + readOnly: true + + startupProbe: + httpGet: + path: /v1/status + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 12 + timeoutSeconds: 3 + + livenessProbe: + httpGet: + path: /v1/status + port: 8080 + periodSeconds: 30 + failureThreshold: 3 + timeoutSeconds: 5 + + volumes: + # Mount api-tts.toml from Secret Manager. + # Create this secret with: gcloud secrets create deepgram-api-tts-config + - name: api-config + secret: + secretName: deepgram-api-tts-config + items: + - key: latest + path: api.toml diff --git a/cloud/google/cloud-run/services/api.service.yaml b/cloud/google/cloud-run/services/api.service.yaml new file mode 100644 index 0000000..770323b --- /dev/null +++ b/cloud/google/cloud-run/services/api.service.yaml @@ -0,0 +1,102 @@ +# Deepgram API — Cloud Run service definition (STT / Nova-2 / Nova-3) +# +# This API service routes requests to the STT Engine (nova-2/nova-3) only. +# Each workload type requires its own dedicated API+Engine pair: +# services/api-flux.service.yaml — Flux turn-based streaming STT +# services/api-tts.service.yaml — Aura TTS +# +# This file is a template. Variables in ${VAR} notation are substituted +# by deploy.sh using envsubst before being applied with: +# gcloud run services replace - --region=${REGION} --project=${PROJECT_ID} + +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: deepgram-api + annotations: + # Allow public internet traffic. Clients send transcription requests here. + run.googleapis.com/ingress: all +spec: + template: + metadata: + annotations: + run.googleapis.com/execution-environment: gen2 + + # The API is CPU-bound; scale from zero when idle to save cost. + # Raise minScale if you require zero-latency API responses at all times. + autoscaling.knative.dev/minScale: "1" + + # Maximum number of API replicas. Each replica load-balances across all + # Engine instances in the driver pool. + autoscaling.knative.dev/maxScale: "10" + + # Attach the API to the Deepgram VPC via Direct VPC Egress so that + # calls to the Engine stay on the private network. + run.googleapis.com/network-interfaces: '[{"network":"${VPC_NETWORK}","subnetwork":"${VPC_SUBNETWORK}"}]' + run.googleapis.com/vpc-access-egress: private-ranges-only + spec: + # Service account used to read config from Secret Manager. + serviceAccountName: ${SERVICE_ACCOUNT_EMAIL} + + # Match the Engine timeout to allow long-running requests to complete. + timeoutSeconds: 3600 + + containers: + - image: "${IMAGE_REGISTRY}/self-hosted-api:${IMAGE_TAG}" + # Override the container CMD. The api config is mounted from Secret Manager. + args: ["-vvv", "serve", "/config/api.toml"] + + ports: + - containerPort: 8080 + + env: + # Deepgram self-hosted API key, read from Secret Manager at runtime. + - name: DEEPGRAM_API_KEY + valueFrom: + secretKeyRef: + name: deepgram-api-key + key: latest + + - name: DEEPGRAM_DEPLOYMENT_ORCHESTRATOR + value: cloud-run + + resources: + limits: + # The API is lightweight. Adjust based on observed memory usage. + cpu: "2" + memory: 4Gi + + volumeMounts: + # api.toml is mounted from Secret Manager into /config/api.toml. + # The config already contains the Engine service URL, substituted + # by deploy.sh when the secret was created. + - name: api-config + mountPath: /config + readOnly: true + + startupProbe: + httpGet: + path: /v1/status + port: 8080 + initialDelaySeconds: 5 + periodSeconds: 5 + failureThreshold: 12 + timeoutSeconds: 3 + + livenessProbe: + httpGet: + path: /v1/status + port: 8080 + periodSeconds: 30 + failureThreshold: 3 + timeoutSeconds: 5 + + volumes: + # Mount api.toml from Secret Manager. + # The secret value is written to /config/api.toml inside the container. + - name: api-config + secret: + secretName: deepgram-api-config + items: + - key: latest + path: api.toml diff --git a/cloud/google/cloud-run/services/engine-flux.service.yaml b/cloud/google/cloud-run/services/engine-flux.service.yaml new file mode 100644 index 0000000..0d81e34 --- /dev/null +++ b/cloud/google/cloud-run/services/engine-flux.service.yaml @@ -0,0 +1,146 @@ +# Deepgram Engine — Cloud Run service definition (Flux / dedicated STT) +# +# Use this file ONLY for Flux turn-based streaming STT workloads. +# Flux must run in complete isolation — no other Deepgram models (nova, TTS, +# diarizer, NER, etc.) may be present on the same Engine instance. Mixing +# models causes CUDA out-of-memory errors and request failures. +# +# Flux requires container images from October 2025 or later (release-251015+). +# The current IMAGE_TAG default satisfies this requirement. +# +# Concurrency for Flux is controlled exclusively by flux.max_streams in +# engine-flux.toml. The max_active_requests setting does not apply to Flux. +# +# For optimal streaming performance, Deepgram recommends a separate API service +# routing Flux traffic to this engine. You may reuse the standard deepgram-api +# service for low-to-moderate Flux traffic. +# +# See: https://developers.deepgram.com/docs/flux-self-hosted +# +# This file is a template. Variables in ${VAR} notation are substituted +# by deploy.sh using envsubst before being applied with: +# gcloud run services replace - --region=${REGION} --project=${PROJECT_ID} +# +# GPU regions: https://cloud.google.com/run/docs/configuring/services/gpu + +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: deepgram-engine-flux + annotations: + # Restrict to internal Cloud Run traffic only — the API service calls this + # endpoint directly using its Cloud Run URL. No public access is granted. + run.googleapis.com/ingress: internal +spec: + template: + metadata: + annotations: + # Second-generation execution environment is required for GPU support. + run.googleapis.com/execution-environment: gen2 + + # Attach one NVIDIA L4 GPU per instance. Flux allocates all GPU memory + # for streams at startup; no other models may share this GPU. + run.googleapis.com/gpu-type: nvidia-l4 + + # Disable CPU throttling. GPU workloads must not have their CPU throttled + # or model loading and inference will stall. + run.googleapis.com/cpu-throttling: "false" + + # Boost CPU during startup to accelerate model loading from GCS. + run.googleapis.com/startup-cpu-boost: "true" + + # Disable GPU zonal redundancy. Required unless you have explicit quota + # for GPU zonal redundancy (g.co/cloudrun/gpu-redundancy-help). + run.googleapis.com/gpu-zonal-redundancy-disabled: "true" + + run.googleapis.com/network-interfaces: '[{"network":"${VPC_NETWORK}","subnetwork":"${VPC_SUBNETWORK}"}]' + run.googleapis.com/vpc-access-egress: private-ranges-only + + # Keep at least one warm Flux instance running to eliminate cold-start + # latency. Flux pre-allocates GPU memory on startup, so cold starts are + # especially disruptive for real-time voice agent traffic. + autoscaling.knative.dev/minScale: "1" + + # Maximum number of Flux GPU instances. Each instance requires its own + # L4 GPU quota. Flux stream capacity per instance is controlled by + # flux.max_streams in engine-flux.toml — scale out instances when + # flux_fraction_streams consistently approaches 1.0. + autoscaling.knative.dev/maxScale: "3" + spec: + # Service account used to read models from GCS and secrets from Secret Manager. + serviceAccountName: ${SERVICE_ACCOUNT_EMAIL} + + # Extend the request timeout to support long-lived Flux streaming sessions. + # Cloud Run gen2 maximum is 3600s. + timeoutSeconds: 3600 + + containers: + - image: "${IMAGE_REGISTRY}/self-hosted-engine:${IMAGE_TAG}" + # Mount the Flux-specific engine config from Secret Manager. + args: ["-v", "serve", "/config/engine.toml"] + + ports: + - name: h2c + containerPort: 8080 + + env: + # Deepgram self-hosted API key, read from Secret Manager at runtime. + - name: DEEPGRAM_API_KEY + valueFrom: + secretKeyRef: + name: deepgram-api-key + key: latest + + - name: DEEPGRAM_DEPLOYMENT_ORCHESTRATOR + value: cloud-run + + resources: + limits: + # 8 vCPU and 32 GiB RAM are the recommended minimums for an L4 GPU + # Flux workload. Flux is memory-intensive at startup because it + # pre-allocates GPU memory for all configured streams. + cpu: "8" + memory: 32Gi + nvidia.com/gpu: "1" + + volumeMounts: + # engine-flux.toml is mounted from Secret Manager into /config/engine.toml + - name: engine-config + mountPath: /config + readOnly: true + + # Models are served from a GCS bucket via GCS FUSE. + # Only the Flux model file should be present — no nova or TTS models. + - name: models + mountPath: /models + readOnly: true + + # Startup probe: Flux pre-loads its model and allocates GPU memory + # during startup. Allow up to 10 minutes for large GPU allocations. + startupProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 60 + timeoutSeconds: 5 + + volumes: + # Mount engine-flux.toml from Secret Manager. + # Create this secret with: gcloud secrets create deepgram-engine-flux-config + - name: engine-config + secret: + secretName: deepgram-engine-flux-config + items: + - key: latest + path: engine.toml + + # Mount the Flux models GCS bucket via GCS FUSE (gen2 only). + # The bucket should contain only the Flux model file to prevent + # accidental loading of incompatible models. + - name: models + csi: + driver: gcsfuse.run.googleapis.com + readOnly: true + volumeAttributes: + bucketName: ${MODELS_BUCKET} diff --git a/cloud/google/cloud-run/services/engine-tts.service.yaml b/cloud/google/cloud-run/services/engine-tts.service.yaml new file mode 100644 index 0000000..7cdb5a4 --- /dev/null +++ b/cloud/google/cloud-run/services/engine-tts.service.yaml @@ -0,0 +1,164 @@ +# Deepgram Engine — Cloud Run service definition (Aura TTS / dedicated) +# +# Use this file ONLY for Text-to-Speech workloads using Aura or Aura-2 models. +# Deepgram strongly recommends dedicating each node to a single service type. +# Running TTS alongside STT or Flux models causes resource contention and +# unpredictable latency. +# +# Aura TTS requires 2x NVIDIA L4 GPUs per instance. Cloud Run supports up to +# 2 L4 GPUs on gen2 instances; verify quota in your region before deploying. +# Request 2x NVIDIA_L4_GPU quota per Engine TTS replica in the GCP console. +# +# Before deploying, store the language-specific Aura-2 model UUIDs in Secret +# Manager (see env section below). UUIDs are provided by your Deepgram Account +# Representative. See: https://developers.deepgram.com/docs/deploy-tts-services +# +# This file is a template. Variables in ${VAR} notation are substituted +# by deploy.sh using envsubst before being applied with: +# gcloud run services replace - --region=${REGION} --project=${PROJECT_ID} +# +# GPU regions: https://cloud.google.com/run/docs/configuring/services/gpu + +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: deepgram-engine-tts + annotations: + # Restrict to internal Cloud Run traffic only — the API service calls this + # endpoint directly using its Cloud Run URL. No public access is granted. + run.googleapis.com/ingress: internal +spec: + template: + metadata: + annotations: + # Second-generation execution environment is required for GPU support. + run.googleapis.com/execution-environment: gen2 + + # Aura TTS requires 2 NVIDIA L4 GPUs per instance (32 GB GPU RAM total). + # CUDA_VISIBLE_DEVICES=0,1 (set below) pins the TTS engine to both GPUs. + run.googleapis.com/gpu-type: nvidia-l4 + + # Disable CPU throttling. TTS synthesis is compute-intensive and must + # not be throttled or first-byte latency will degrade significantly. + run.googleapis.com/cpu-throttling: "false" + + # Boost CPU during startup to accelerate model loading from GCS. + run.googleapis.com/startup-cpu-boost: "true" + + # Disable GPU zonal redundancy. Required unless you have explicit quota + # for GPU zonal redundancy (g.co/cloudrun/gpu-redundancy-help). + run.googleapis.com/gpu-zonal-redundancy-disabled: "true" + + run.googleapis.com/network-interfaces: '[{"network":"${VPC_NETWORK}","subnetwork":"${VPC_SUBNETWORK}"}]' + run.googleapis.com/vpc-access-egress: private-ranges-only + + # Keep at least one warm TTS instance running to avoid cold-start + # latency (~2–5 min for TTS model loading). + autoscaling.knative.dev/minScale: "1" + + # Maximum number of TTS GPU instances. Each instance requires 2x L4 + # GPU quota. Request quota before raising this value. + autoscaling.knative.dev/maxScale: "2" + spec: + # Service account used to read models from GCS and secrets from Secret Manager. + serviceAccountName: ${SERVICE_ACCOUNT_EMAIL} + + # Extend timeout to support long TTS synthesis jobs. + timeoutSeconds: 3600 + + containers: + - image: "${IMAGE_REGISTRY}/self-hosted-engine:${IMAGE_TAG}" + args: ["-v", "serve", "/config/engine.toml"] + + ports: + - name: h2c + containerPort: 8080 + + env: + # Deepgram self-hosted API key, read from Secret Manager at runtime. + - name: DEEPGRAM_API_KEY + valueFrom: + secretKeyRef: + name: deepgram-api-key + key: latest + + - name: DEEPGRAM_DEPLOYMENT_ORCHESTRATOR + value: cloud-run + + # Pin the TTS engine to both GPUs. Aura requires both GPUs to be + # visible; removing this causes the engine to use only one GPU and + # fail to initialize the TTS pipeline. + - name: CUDA_VISIBLE_DEVICES + value: "0,1" + + # Aura-2 model UUIDs — language-specific values provided by Deepgram. + # Store these in Secret Manager and create the secrets before deploying. + # To create: gcloud secrets create deepgram-tts-t2c-uuid --data-file=- + - name: IMPELLER_AURA2_T2C_UUID + valueFrom: + secretKeyRef: + name: deepgram-tts-t2c-uuid + key: latest + + - name: IMPELLER_AURA2_C2A_UUID + valueFrom: + secretKeyRef: + name: deepgram-tts-c2a-uuid + key: latest + + # Maximum batch size for TTS synthesis. 8 is the recommended default + # for L4 GPUs. Tune based on latency vs. throughput requirements. + - name: IMPELLER_AURA2_MAX_BATCH_SIZE + value: "8" + + resources: + limits: + # Aura TTS requires 2x NVIDIA L4 GPUs (32 GB GPU RAM total), + # 8+ CPU cores, and 64 GiB RAM per instance. + # See: https://developers.deepgram.com/docs/self-hosted-deployment-environments + cpu: "16" + memory: 64Gi + nvidia.com/gpu: "2" + + volumeMounts: + # engine.toml is mounted from Secret Manager into /config/engine.toml. + # Use a TTS-specific engine config (provided by Deepgram). + - name: engine-config + mountPath: /config + readOnly: true + + # TTS models are served from a GCS bucket via GCS FUSE. + # The bucket should contain only Aura model files. + - name: models + mountPath: /models + readOnly: true + + # TTS model loading is slower than STT. Allow up to 10 minutes for + # both GPUs to initialize and models to load. + startupProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 60 + periodSeconds: 10 + failureThreshold: 60 + timeoutSeconds: 5 + + volumes: + # Mount TTS engine.toml from Secret Manager. + # Create this secret with the TTS-specific config provided by Deepgram: + # gcloud secrets create deepgram-engine-tts-config --data-file=engine-tts.toml + - name: engine-config + secret: + secretName: deepgram-engine-tts-config + items: + - key: latest + path: engine.toml + + # Mount the Aura models GCS bucket via GCS FUSE (gen2 only). + # The bucket should contain only TTS model files. + - name: models + csi: + driver: gcsfuse.run.googleapis.com + readOnly: true + volumeAttributes: + bucketName: ${MODELS_BUCKET} diff --git a/cloud/google/cloud-run/services/engine.service.yaml b/cloud/google/cloud-run/services/engine.service.yaml new file mode 100644 index 0000000..ea92d64 --- /dev/null +++ b/cloud/google/cloud-run/services/engine.service.yaml @@ -0,0 +1,138 @@ +# Deepgram Engine — Cloud Run service definition (STT / Nova-2 / Nova-3) +# +# Use this file for Speech-to-Text workloads using nova-2 or nova-3 models. +# An NVIDIA L4 GPU can host up to 5 concurrent nova-2/nova-3 model files. +# +# DO NOT enable [flux] in engine.toml for this service — Flux must run on its +# own dedicated Cloud Run service (see engine-flux.service.yaml). Enabling Flux +# alongside nova models causes CUDA out-of-memory errors. +# Aura TTS also requires its own dedicated service (see engine-tts.service.yaml). +# +# This file is a template. Variables in ${VAR} notation are substituted +# by deploy.sh using envsubst before being applied with: +# gcloud run services replace - --region=${REGION} --project=${PROJECT_ID} +# +# GPU regions: https://cloud.google.com/run/docs/configuring/services/gpu + +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: deepgram-engine + annotations: + # Restrict to internal Cloud Run traffic only — the API service calls this + # endpoint directly using its Cloud Run URL. No public access is granted. + run.googleapis.com/ingress: internal +spec: + template: + metadata: + annotations: + # Second-generation execution environment is required for GPU support. + run.googleapis.com/execution-environment: gen2 + + # Attach one NVIDIA L4 GPU per instance. L4 is available in more regions + # than A100/H100 and is well-suited for inference workloads. + # To use a different GPU type, change this value (e.g. nvidia-a100-80gb). + run.googleapis.com/gpu-type: nvidia-l4 + + # Disable CPU throttling. GPU workloads must not have their CPU throttled + # or model loading and inference will stall. + run.googleapis.com/cpu-throttling: "false" + + # Boost CPU during startup to accelerate model loading from GCS. + run.googleapis.com/startup-cpu-boost: "true" + + # Disable GPU zonal redundancy. Required unless you have explicit quota + # for GPU zonal redundancy (g.co/cloudrun/gpu-redundancy-help). + run.googleapis.com/gpu-zonal-redundancy-disabled: "true" + + # Attach the Engine to the Deepgram VPC via Direct VPC Egress so that + # intra-service traffic stays on the private network. + run.googleapis.com/network-interfaces: '[{"network":"${VPC_NETWORK}","subnetwork":"${VPC_SUBNETWORK}"}]' + run.googleapis.com/vpc-access-egress: private-ranges-only + + # Keep at least one warm instance running at all times to eliminate + # GPU cold-start latency (~2–5 min). Set to "0" only if cost is a + # higher priority than availability. + autoscaling.knative.dev/minScale: "1" + + # Maximum number of GPU instances. Each instance requires its own L4 + # GPU quota. Request additional quota in the GCP console before raising + # this value. + autoscaling.knative.dev/maxScale: "3" + spec: + # Service account used to read models from GCS and secrets from Secret Manager. + serviceAccountName: ${SERVICE_ACCOUNT_EMAIL} + + # Extend the request timeout to support long batch audio processing jobs. + # Cloud Run gen2 maximum is 3600s. + timeoutSeconds: 3600 + + containers: + - image: "${IMAGE_REGISTRY}/self-hosted-engine:${IMAGE_TAG}" + # Override the container CMD. The engine config is mounted from Secret Manager. + args: ["-v", "serve", "/config/engine.toml"] + + ports: + - containerPort: 8080 + + env: + # Deepgram self-hosted API key, read from Secret Manager at runtime. + - name: DEEPGRAM_API_KEY + valueFrom: + secretKeyRef: + name: deepgram-api-key + key: latest + + - name: DEEPGRAM_DEPLOYMENT_ORCHESTRATOR + value: cloud-run + + resources: + limits: + # 8 vCPU and 32 GiB RAM are the recommended minimums for an L4 GPU + # STT inference workload. A single L4 GPU can host up to 5 concurrent + # nova-2/nova-3 model files; increase memory if loading many large models. + cpu: "8" + memory: 32Gi + nvidia.com/gpu: "1" + + volumeMounts: + # engine.toml is mounted from Secret Manager into /config/engine.toml + - name: engine-config + mountPath: /config + readOnly: true + + # Models are served from a GCS bucket via GCS FUSE. + # The bucket is read-only; no writes are performed by the Engine. + - name: models + mountPath: /models + readOnly: true + + # Startup probe: wait up to 5 minutes for the Engine to become ready. + # Model preloading (blocking = true in engine.toml) can take several + # minutes on first start depending on model size and GCS read speed. + startupProbe: + tcpSocket: + port: 8080 + initialDelaySeconds: 30 + periodSeconds: 10 + failureThreshold: 30 + timeoutSeconds: 5 + + volumes: + # Mount engine.toml from Secret Manager. + # The secret value is written to /config/engine.toml inside the container. + - name: engine-config + secret: + secretName: deepgram-engine-config + items: + - key: latest + path: engine.toml + + # Mount the models GCS bucket via GCS FUSE (gen2 only). + # The service account must have roles/storage.objectViewer on this bucket. + - name: models + csi: + driver: gcsfuse.run.googleapis.com + readOnly: true + volumeAttributes: + bucketName: ${MODELS_BUCKET} diff --git a/cloud/google/cloud-run/teardown.sh b/cloud/google/cloud-run/teardown.sh new file mode 100755 index 0000000..e3db131 --- /dev/null +++ b/cloud/google/cloud-run/teardown.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# teardown.sh — Remove Deepgram Cloud Run services and associated resources +# +# Usage: +# ./teardown.sh +# +# By default this deletes Cloud Run services and Secret Manager secrets. +# The GCS models bucket and service account are NOT deleted automatically +# because they may be shared or expensive to recreate. +# Pass --all to also remove the service account. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# ── Load .env if present ─────────────────────────────────────────────────────── +if [[ -f "${SCRIPT_DIR}/.env" ]]; then + # shellcheck disable=SC1091 + set -a; source "${SCRIPT_DIR}/.env"; set +a +fi + +# ── Required variables ───────────────────────────────────────────────────────── +: "${GCP_PROJECT_ID:?'GCP_PROJECT_ID is required.'}" + +# ── Defaults (must match deploy.sh) ─────────────────────────────────────────── +GCP_REGION="${GCP_REGION:-us-central1}" +SERVICE_ACCOUNT_NAME="${SERVICE_ACCOUNT_NAME:-deepgram-cloud-run}" +DEEPGRAM_API_KEY_SECRET="${DEEPGRAM_API_KEY_SECRET:-deepgram-api-key}" +ENGINE_CONFIG_SECRET="deepgram-engine-config" +API_CONFIG_SECRET="deepgram-api-config" +ENGINE_SERVICE_NAME="${ENGINE_SERVICE_NAME:-deepgram-engine}" +API_SERVICE_NAME="${API_SERVICE_NAME:-deepgram-api}" +SERVICE_ACCOUNT_EMAIL="${SERVICE_ACCOUNT_NAME}@${GCP_PROJECT_ID}.iam.gserviceaccount.com" + +DELETE_SA=false +if [[ "${1:-}" == "--all" ]]; then + DELETE_SA=true +fi + +log() { echo " [teardown] $*"; } +step() { echo; echo "▶ $*"; } + +# ── Confirmation ─────────────────────────────────────────────────────────────── +echo +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " This will DELETE the following resources in project: ${GCP_PROJECT_ID}" +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Cloud Run services: ${API_SERVICE_NAME}, ${ENGINE_SERVICE_NAME}" +echo " Secrets: ${DEEPGRAM_API_KEY_SECRET}, ${ENGINE_CONFIG_SECRET}, ${API_CONFIG_SECRET}" +if [[ "${DELETE_SA}" == "true" ]]; then + echo " Service account: ${SERVICE_ACCOUNT_EMAIL}" +fi +echo " GCS models bucket: NOT deleted (delete manually if desired)" +echo + +read -rp " Proceed? [y/N] " confirm +if [[ "${confirm}" != "y" && "${confirm}" != "Y" ]]; then + echo "Aborted." + exit 0 +fi + +gcloud config set project "${GCP_PROJECT_ID}" --quiet + +# ── Delete Cloud Run services ────────────────────────────────────────────────── +step "Deleting Cloud Run services" +for svc in "${API_SERVICE_NAME}" "${ENGINE_SERVICE_NAME}"; do + if gcloud run services describe "${svc}" \ + --region="${GCP_REGION}" --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + gcloud run services delete "${svc}" \ + --region="${GCP_REGION}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet + log "Deleted service: ${svc}" + else + log "Service not found (skipping): ${svc}" + fi +done + +# ── Delete secrets ───────────────────────────────────────────────────────────── +step "Deleting Secret Manager secrets" +for secret in "${DEEPGRAM_API_KEY_SECRET}" "${ENGINE_CONFIG_SECRET}" "${API_CONFIG_SECRET}"; do + if gcloud secrets describe "${secret}" \ + --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + gcloud secrets delete "${secret}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet + log "Deleted secret: ${secret}" + else + log "Secret not found (skipping): ${secret}" + fi +done + +# ── Optionally delete service account ───────────────────────────────────────── +if [[ "${DELETE_SA}" == "true" ]]; then + step "Deleting service account: ${SERVICE_ACCOUNT_EMAIL}" + if gcloud iam service-accounts describe "${SERVICE_ACCOUNT_EMAIL}" \ + --project="${GCP_PROJECT_ID}" --quiet 2>/dev/null; then + gcloud iam service-accounts delete "${SERVICE_ACCOUNT_EMAIL}" \ + --project="${GCP_PROJECT_ID}" \ + --quiet + log "Deleted service account." + else + log "Service account not found (skipping)." + fi +fi + +echo +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Teardown complete." +echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" +echo " Note: the GCS models bucket was not deleted." +echo " To delete it: gsutil -m rm -r gs://${DEEPGRAM_MODELS_BUCKET:-YOUR_BUCKET}" +echo