diff --git a/.github/actions/build-flavor/action.yml b/.github/actions/build-flavor/action.yml new file mode 100644 index 0000000..da2271a --- /dev/null +++ b/.github/actions/build-flavor/action.yml @@ -0,0 +1,150 @@ +name: Build one image flavor +description: > + Builds one flavor of one cell for a single architecture and pushes it by digest. + Push-by-digest leaves the manifest untagged; the merge-flavor action is what + assembles the tagged manifest list, so no consumer-facing tag ever points at a + half-published or unsigned image. + +inputs: + cell: + description: One build entry emitted by scripts/build_pgedge_images.py (PGEDGE_EMIT_MATRIX=1) + required: true + repo: + description: Image repository to push to + required: true + registry_token: + description: > + Token for the container registry. A composite action has no secrets + context, so this has to be passed in by the calling workflow. + required: true + dry_run: + description: When true, resolve and print the build without pushing + required: false + default: "false" + no_cache: + description: When true, build without cache + required: false + default: "false" + +outputs: + digest: + description: Digest of the pushed per-architecture manifest + value: ${{ steps.build.outputs.digest }} + +runs: + using: composite + steps: + # No QEMU: the calling job selects a runner native to the target + # architecture, so nothing here is emulated. + # buildx-init only registers the builder; buildx selects it via the + # BUILDX_BUILDER environment variable. Without this the build lands on the + # default docker driver, which rejects the bake file's attestations. + - name: Setup Docker Buildx + shell: bash + run: | + set -o errexit -o pipefail + make buildx-init + echo "BUILDX_BUILDER=$(make -s print-buildx-builder)" >> "$GITHUB_ENV" + + - name: Login to the container registry + if: ${{ inputs.dry_run != 'true' }} + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.registry_token }} + + - name: Set up Go + if: ${{ inputs.dry_run != 'true' }} + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + with: + go-version: '1.25' + cache-dependency-path: tests/go.sum + + - name: Build and push by digest + id: build + shell: bash + env: + CELL: ${{ inputs.cell }} + REPO: ${{ inputs.repo }} + DRY_RUN: ${{ inputs.dry_run }} + NO_CACHE: ${{ inputs.no_cache }} + run: | + set -o errexit -o pipefail -o nounset + + target=$(jq -r '.target' <<< "$CELL") + arch=$(jq -r '.arch' <<< "$CELL") + parent=$(jq -r '.parent_build_tag' <<< "$CELL") + build_tag=$(jq -r '.build_tag' <<< "$CELL") + + args=( + --file pgedge.docker-bake.hcl + --set "default.platform=linux/${arch}" + --set "default.tags=" + --metadata-file metadata.json + ) + if [[ "$NO_CACHE" == "true" ]]; then + args+=(--no-cache) + fi + if [[ "$DRY_RUN" == "true" ]]; then + args+=(--print) + else + args+=(--set "default.output=type=image,name=${REPO},push-by-digest=true,name-canonical=true,push=true") + fi + + # Every packagelist ARG comes from the cell, so a new flavor needs no + # change here -- only the Dockerfile and the driver's maps. + envs=( + "PACKAGE_RELEASE_CHANNEL=$(jq -r '.package_release_channel' <<< "$CELL")" + "POSTGRES_MAJOR_VERSION=$(jq -r '.postgres_major' <<< "$CELL")" + "TARGET=${target}" + "TAG=${REPO}" + ) + while IFS= read -r kv; do + envs+=("$kv") + done < <(jq -r '.package_list_args | to_entries[] | "\(.key)=\(.value)"' <<< "$CELL") + + # A chained flavor starts FROM the image the previous wave published. + parent_arg=$(jq -r '.parent_image_arg' <<< "$CELL") + if [[ -n "$parent" && -n "$parent_arg" ]]; then + envs+=("${parent_arg}=${REPO}:${parent}") + fi + + env "${envs[@]}" docker buildx bake "${args[@]}" + + if [[ "$DRY_RUN" != "true" ]]; then + digest=$(jq -r '.["default"]["containerimage.digest"] // .["containerimage.digest"]' metadata.json) + test -n "$digest" && test "$digest" != "null" + echo "digest=${digest}" >> "$GITHUB_OUTPUT" + mkdir -p digests + echo -n "$digest" > "digests/${arch}" + echo "::notice::${target} ${build_tag} ${arch} -> ${digest}" + fi + + # Runs against the bytes just pushed, on a runner native to their + # architecture, so a broken image never reaches the merge step. The digest is + # re-tagged locally first: the harness derives the expected spock major from + # the tag and silently skips that assertion for a bare digest reference. + - name: Test the pushed image + if: ${{ inputs.dry_run != 'true' }} + shell: bash + env: + CELL: ${{ inputs.cell }} + REPO: ${{ inputs.repo }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + set -o errexit -o pipefail -o nounset + target=$(jq -r '.target' <<< "$CELL") + build_tag=$(jq -r '.build_tag' <<< "$CELL") + docker pull -q "${REPO}@${DIGEST}" + docker tag "${REPO}@${DIGEST}" "${REPO}:${build_tag}" + make test-image IMAGE="${REPO}:${build_tag}" FLAVOR="${target}" + + - name: Upload the digest for the merge job + if: ${{ inputs.dry_run != 'true' }} + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: digest-${{ fromJSON(inputs.cell).target }}-${{ fromJSON(inputs.cell).build_tag }}-${{ fromJSON(inputs.cell).arch }} + path: digests/* + retention-days: 1 + if-no-files-found: error diff --git a/.github/actions/merge-flavor/action.yml b/.github/actions/merge-flavor/action.yml new file mode 100644 index 0000000..2b334be --- /dev/null +++ b/.github/actions/merge-flavor/action.yml @@ -0,0 +1,115 @@ +name: Merge one image flavor +description: > + Assembles one flavor's per-architecture digests into a manifest list, signs the + index, and only then applies the mutable tags. Ordering matters: the per-arch + manifests pushed by the build jobs are untagged, and the immutable epoch tag is + created here from an index that is signed before any other tag points at it, so + no consumer-facing tag is ever live without a signature. + +inputs: + cell: + description: One merge entry emitted by scripts/build_pgedge_images.py (PGEDGE_EMIT_MATRIX=1) + required: true + repo: + description: Image repository to publish to + required: true + registry_token: + description: > + Token for the container registry. A composite action has no secrets + context, so this has to be passed in by the calling workflow. + required: true + dry_run: + description: When true, print what would be published and exit + required: false + default: "false" + +runs: + using: composite + steps: + - name: Install cosign + if: ${{ inputs.dry_run != 'true' }} + uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 + + # buildx-init only registers the builder; buildx selects it via the + # BUILDX_BUILDER environment variable. Without this the build lands on the + # default docker driver, which rejects the bake file's attestations. + - name: Setup Docker Buildx + shell: bash + run: | + set -o errexit -o pipefail + make buildx-init + echo "BUILDX_BUILDER=$(make -s print-buildx-builder)" >> "$GITHUB_ENV" + + - name: Login to the container registry + if: ${{ inputs.dry_run != 'true' }} + uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ inputs.registry_token }} + + - name: Collect the per-architecture digests + if: ${{ inputs.dry_run != 'true' && fromJSON(inputs.cell).needs_build }} + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: digest-*-${{ fromJSON(inputs.cell).build_tag }}-* + path: digests + merge-multiple: true + + - name: Create, sign, then tag + if: ${{ inputs.dry_run != 'true' }} + shell: bash + env: + CELL: ${{ inputs.cell }} + REPO: ${{ inputs.repo }} + run: | + set -o errexit -o pipefail -o nounset + + build_tag=$(jq -r '.build_tag' <<< "$CELL") + + # needs_build is false when the immutable tag was already published and + # republish was not requested. The index is then left exactly as it is -- + # not reassembled, not re-signed -- and only the mutable tags are + # refreshed, so a re-dispatch repairs tag drift without rebuilding. + if [[ "$(jq -r '.needs_build' <<< "$CELL")" == "true" ]]; then + # One source per architecture, addressed by digest. + sources=() + while read -r arch; do + digest=$(cat "digests/${arch}") + test -n "$digest" + sources+=("${REPO}@${digest}") + done < <(jq -r '.arches[]' <<< "$CELL") + test "${#sources[@]}" -gt 0 + + echo "Assembling ${REPO}:${build_tag} from ${#sources[@]} platform manifest(s)" + docker buildx imagetools create --tag "${REPO}:${build_tag}" "${sources[@]}" + + # cosign signs the index. Its children are content-addressed by the + # index, so verifying a tag transitively covers every platform. + index_digest=$(docker buildx imagetools inspect "${REPO}:${build_tag}" \ + --format '{{ printf "%s" .Manifest.Digest }}') + cosign sign --yes "${REPO}@${index_digest}" + else + echo "${REPO}:${build_tag} is already published; refreshing tags only" + fi + + # Mutable tags last, so each points at an already-signed index. + while read -r tag; do + echo "Tagging ${REPO}:${tag}" + docker buildx imagetools create --tag "${REPO}:${tag}" "${REPO}:${build_tag}" + done < <(jq -r '.extra_tags[]' <<< "$CELL") + + - name: Dry run summary + if: ${{ inputs.dry_run == 'true' }} + shell: bash + env: + CELL: ${{ inputs.cell }} + REPO: ${{ inputs.repo }} + run: | + if [[ "$(jq -r '.needs_build' <<< "$CELL")" == "true" ]]; then + echo "would assemble and sign ${REPO}:$(jq -r '.build_tag' <<< "$CELL")" + echo " from arches : $(jq -r '.arches | join(", ")' <<< "$CELL")" + else + echo "${REPO}:$(jq -r '.build_tag' <<< "$CELL") already published; tags only" + fi + echo " mutable tags: $(jq -r '.extra_tags | join(", ")' <<< "$CELL")" diff --git a/.github/workflows/build_images.yaml b/.github/workflows/build_images.yaml index ef9c409..eb75275 100644 --- a/.github/workflows/build_images.yaml +++ b/.github/workflows/build_images.yaml @@ -1,3 +1,21 @@ +# Per-flavor wave build. +# +# Each flavor is a wave of native single-platform builds -- one job per +# (postgres major, spock version, architecture) -- followed by a merge round +# that assembles the manifest list, signs it, and applies the mutable tags. +# A wave builds FROM the image the previous wave published, so a chained stage +# is never rebuilt on a second runner: rebuilding it would re-run its unpinned +# "dnf update -y" and produce layers that differ from the published parent's. +# +# Why waves rather than one job per cell: +# * a flavor's failure cannot unpublish the flavors beneath it; +# * every layer is built exactly once, so the published images provably share +# them instead of relying on a warm build cache; +# * arm64 builds run on native runners instead of under QEMU. +# +# The matrix comes from scripts/build_pgedge_images.py, so the cell list cannot +# drift away from the image definitions it already owns. + name: build-images on: @@ -9,7 +27,7 @@ on: default: "ghcr.io/pgedge/pgedge-postgres-internal" required: false pgedge_image_republish: - description: "Republish images? (true/false)" + description: "Republish images that are already published? (true/false)" type: boolean default: false pgedge_image_dry_run: @@ -19,7 +37,7 @@ on: pgedge_image_no_cache: description: "Build without cache? (true/false)" type: boolean - default: true + default: false pgedge_image_only_postgres_version: description: "Build only this Postgres version (leave blank for all)" type: string @@ -50,22 +68,29 @@ env: PGEDGE_IMAGE_ONLY_ARCH: ${{ inputs.pgedge_image_only_arch }} jobs: - build-images: - runs-on: ubuntu-latest + plan: + name: Plan waves + runs-on: ubuntu-24.04 + outputs: + postgres_build: ${{ steps.matrix.outputs.postgres_build }} + postgres_merge: ${{ steps.matrix.outputs.postgres_merge }} + minimal_build: ${{ steps.matrix.outputs.minimal_build }} + minimal_merge: ${{ steps.matrix.outputs.minimal_merge }} + standard_build: ${{ steps.matrix.outputs.standard_build }} + standard_merge: ${{ steps.matrix.outputs.standard_merge }} + coldfront_build: ${{ steps.matrix.outputs.coldfront_build }} + coldfront_merge: ${{ steps.matrix.outputs.coldfront_merge }} steps: - name: Checkout repository uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - - name: Setup QEMU - uses: docker/setup-qemu-action@c7c53464625b32c7a7e944ae62b3e17d2b600130 # v3.7.0 - - - name: Install cosign - uses: sigstore/cosign-installer@7e8b541eb2e61bf99390e1afd4be13a184e9ebc5 # v3.10.1 - - - name: Setup Docker Buildx - run: | - make buildx-init + - name: Set up python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: '3.x' + # The plan reads the registry to decide what is already published, so it + # needs credentials even though it publishes nothing itself. - name: Login to GitHub Container Registry uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 with: @@ -73,11 +98,245 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: '3.x' - - - name: Build images + - name: Emit the wave matrices + id: matrix run: | - make pgedge-images \ No newline at end of file + set -o errexit -o pipefail + PGEDGE_EMIT_MATRIX=1 ./scripts/build_pgedge_images.py > waves.json + for flavor in postgres minimal standard coldfront; do + for kind in build merge; do + value=$(python3 -c "import json;print(json.dumps(json.load(open('waves.json'))['$flavor']['$kind']))") + echo "${flavor}_${kind}=${value}" >> "$GITHUB_OUTPUT" + done + done + + - name: Show the plan + run: python3 -m json.tool waves.json + + build-postgres: + name: "postgres ${{ matrix.cell.name }}" + needs: [plan] + if: ${{ fromJSON(needs.plan.outputs.postgres_build)[0] != null }} + runs-on: ${{ matrix.cell.runner }} + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.plan.outputs.postgres_build) }} + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Build and push by digest + uses: ./.github/actions/build-flavor + with: + cell: ${{ toJSON(matrix.cell) }} + repo: ${{ inputs.pgedge_image_repo }} + registry_token: ${{ secrets.GITHUB_TOKEN }} + dry_run: ${{ inputs.pgedge_image_dry_run }} + no_cache: ${{ inputs.pgedge_image_no_cache }} + + merge-postgres: + name: "merge postgres ${{ matrix.cell.name }}" + # An empty matrix skips a job, and a skipped "needs" fails the implicit + # success() of everything downstream. So each job below lists its whole + # upstream chain and blocks only on a real failure. Every upstream job is + # named rather than just the previous one, because a failure reaches the + # next job as a *skip*: checking only the immediate predecessor would let a + # later wave build on top of a broken earlier one. + needs: [plan, build-postgres] + if: >- + ${{ !cancelled() + && needs.plan.result == 'success' + && needs.build-postgres.result != 'failure' + && fromJSON(needs.plan.outputs.postgres_merge)[0] != null }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.plan.outputs.postgres_merge) }} + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Merge, sign and tag + uses: ./.github/actions/merge-flavor + with: + cell: ${{ toJSON(matrix.cell) }} + repo: ${{ inputs.pgedge_image_repo }} + registry_token: ${{ secrets.GITHUB_TOKEN }} + dry_run: ${{ inputs.pgedge_image_dry_run }} + + build-minimal: + name: "minimal ${{ matrix.cell.name }}" + needs: [plan, build-postgres, merge-postgres] + if: >- + ${{ !cancelled() + && needs.plan.result == 'success' + && needs.build-postgres.result != 'failure' + && needs.merge-postgres.result != 'failure' + && fromJSON(needs.plan.outputs.minimal_build)[0] != null }} + runs-on: ${{ matrix.cell.runner }} + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.plan.outputs.minimal_build) }} + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Build and push by digest + uses: ./.github/actions/build-flavor + with: + cell: ${{ toJSON(matrix.cell) }} + repo: ${{ inputs.pgedge_image_repo }} + registry_token: ${{ secrets.GITHUB_TOKEN }} + dry_run: ${{ inputs.pgedge_image_dry_run }} + no_cache: ${{ inputs.pgedge_image_no_cache }} + + merge-minimal: + name: "merge minimal ${{ matrix.cell.name }}" + needs: [plan, build-postgres, merge-postgres, build-minimal] + if: >- + ${{ !cancelled() + && needs.plan.result == 'success' + && needs.build-postgres.result != 'failure' + && needs.merge-postgres.result != 'failure' + && needs.build-minimal.result != 'failure' + && fromJSON(needs.plan.outputs.minimal_merge)[0] != null }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.plan.outputs.minimal_merge) }} + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Merge, sign and tag + uses: ./.github/actions/merge-flavor + with: + cell: ${{ toJSON(matrix.cell) }} + repo: ${{ inputs.pgedge_image_repo }} + registry_token: ${{ secrets.GITHUB_TOKEN }} + dry_run: ${{ inputs.pgedge_image_dry_run }} + + build-standard: + name: "standard ${{ matrix.cell.name }}" + needs: [plan, build-postgres, merge-postgres, build-minimal, merge-minimal] + if: >- + ${{ !cancelled() + && needs.plan.result == 'success' + && needs.build-postgres.result != 'failure' + && needs.merge-postgres.result != 'failure' + && needs.build-minimal.result != 'failure' + && needs.merge-minimal.result != 'failure' + && fromJSON(needs.plan.outputs.standard_build)[0] != null }} + runs-on: ${{ matrix.cell.runner }} + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.plan.outputs.standard_build) }} + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Build and push by digest + uses: ./.github/actions/build-flavor + with: + cell: ${{ toJSON(matrix.cell) }} + repo: ${{ inputs.pgedge_image_repo }} + registry_token: ${{ secrets.GITHUB_TOKEN }} + dry_run: ${{ inputs.pgedge_image_dry_run }} + no_cache: ${{ inputs.pgedge_image_no_cache }} + + merge-standard: + name: "merge standard ${{ matrix.cell.name }}" + needs: [plan, build-postgres, merge-postgres, build-minimal, merge-minimal, build-standard] + if: >- + ${{ !cancelled() + && needs.plan.result == 'success' + && needs.build-postgres.result != 'failure' + && needs.merge-postgres.result != 'failure' + && needs.build-minimal.result != 'failure' + && needs.merge-minimal.result != 'failure' + && needs.build-standard.result != 'failure' + && fromJSON(needs.plan.outputs.standard_merge)[0] != null }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.plan.outputs.standard_merge) }} + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Merge, sign and tag + uses: ./.github/actions/merge-flavor + with: + cell: ${{ toJSON(matrix.cell) }} + repo: ${{ inputs.pgedge_image_repo }} + registry_token: ${{ secrets.GITHUB_TOKEN }} + dry_run: ${{ inputs.pgedge_image_dry_run }} + + # A failure here leaves this run's earlier flavors published. + build-coldfront: + name: "coldfront ${{ matrix.cell.name }}" + needs: [plan, build-postgres, merge-postgres, build-minimal, merge-minimal, build-standard, merge-standard] + if: >- + ${{ !cancelled() + && needs.plan.result == 'success' + && needs.build-postgres.result != 'failure' + && needs.merge-postgres.result != 'failure' + && needs.build-minimal.result != 'failure' + && needs.merge-minimal.result != 'failure' + && needs.build-standard.result != 'failure' + && needs.merge-standard.result != 'failure' + && fromJSON(needs.plan.outputs.coldfront_build)[0] != null }} + runs-on: ${{ matrix.cell.runner }} + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.plan.outputs.coldfront_build) }} + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Build and push by digest + uses: ./.github/actions/build-flavor + with: + cell: ${{ toJSON(matrix.cell) }} + repo: ${{ inputs.pgedge_image_repo }} + registry_token: ${{ secrets.GITHUB_TOKEN }} + dry_run: ${{ inputs.pgedge_image_dry_run }} + no_cache: ${{ inputs.pgedge_image_no_cache }} + + merge-coldfront: + name: "merge coldfront ${{ matrix.cell.name }}" + needs: [plan, build-postgres, merge-postgres, build-minimal, merge-minimal, build-standard, merge-standard, build-coldfront] + if: >- + ${{ !cancelled() + && needs.plan.result == 'success' + && needs.build-postgres.result != 'failure' + && needs.merge-postgres.result != 'failure' + && needs.build-minimal.result != 'failure' + && needs.merge-minimal.result != 'failure' + && needs.build-standard.result != 'failure' + && needs.merge-standard.result != 'failure' + && needs.build-coldfront.result != 'failure' + && fromJSON(needs.plan.outputs.coldfront_merge)[0] != null }} + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + cell: ${{ fromJSON(needs.plan.outputs.coldfront_merge) }} + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + + - name: Merge, sign and tag + uses: ./.github/actions/merge-flavor + with: + cell: ${{ toJSON(matrix.cell) }} + repo: ${{ inputs.pgedge_image_repo }} + registry_token: ${{ secrets.GITHUB_TOKEN }} + dry_run: ${{ inputs.pgedge_image_dry_run }} diff --git a/.github/workflows/pr_test_latest.yaml b/.github/workflows/pr_test_latest.yaml index 802dd0a..1ebce6c 100644 --- a/.github/workflows/pr_test_latest.yaml +++ b/.github/workflows/pr_test_latest.yaml @@ -65,10 +65,14 @@ jobs: tag=$(echo "$tag" | xargs) # trim whitespace # Determine flavor from tag - if [[ "$tag" == *"-minimal"* ]]; then + if [[ "$tag" == *"-postgres"* ]]; then + flavor="postgres" + elif [[ "$tag" == *"-minimal"* ]]; then flavor="minimal" elif [[ "$tag" == *"-standard"* ]]; then flavor="standard" + elif [[ "$tag" == *"-coldfront"* ]]; then + flavor="coldfront" else # Default to standard if not specified flavor="standard" diff --git a/.github/workflows/test_images.yaml b/.github/workflows/test_images.yaml index ff3eab5..7ffc8f0 100644 --- a/.github/workflows/test_images.yaml +++ b/.github/workflows/test_images.yaml @@ -50,10 +50,14 @@ jobs: tag=$(echo "$tag" | xargs) # trim whitespace # Determine flavor from tag - if [[ "$tag" == *"-minimal"* ]]; then + if [[ "$tag" == *"-postgres"* ]]; then + flavor="postgres" + elif [[ "$tag" == *"-minimal"* ]]; then flavor="minimal" elif [[ "$tag" == *"-standard"* ]]; then flavor="standard" + elif [[ "$tag" == *"-coldfront"* ]]; then + flavor="coldfront" else # Default to standard if not specified flavor="standard" diff --git a/.gitignore b/.gitignore index 44b7acb..05a3223 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,6 @@ __pycache__/ # Go binaries tests/tests +# Scratch written by the image build (bake metadata, per-arch digests) +digests/ +metadata.json diff --git a/Dockerfile b/Dockerfile index 0293c4f..e99a902 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,14 @@ # base image for all flavors # ############################## +# What each chained stage builds FROM. Declared here because an ARG used in a +# FROM must precede the first FROM. The default names the stage above it, giving +# one build graph; a registry reference instead starts that stage from an +# already-published image, which is what a per-flavor CI wave needs. +ARG POSTGRES_IMAGE=postgres +ARG MINIMAL_IMAGE=minimal +ARG STANDARD_IMAGE=standard + FROM rockylinux/rockylinux:9-ubi AS base ARG PACKAGE_RELEASE_CHANNEL="" @@ -29,17 +37,19 @@ mkdir /docker-entrypoint-initdb.d EOF -########################## -# minimal-flavored image # -########################## +############################# +# PostgreSQL-only base image # +############################# +# +# Spock-independent: built once per major, shared across spock lines. -FROM base AS minimal +FROM base AS postgres -ARG PACKAGE_LIST_FILE +ARG POSTGRES_PACKAGE_LIST_FILE ARG TARGETARCH ARG POSTGRES_MAJOR_VERSION -COPY packagelists/${TARGETARCH}/${PACKAGE_LIST_FILE} /usr/share/pgedge/packages.txt +COPY packagelists/${TARGETARCH}/${POSTGRES_PACKAGE_LIST_FILE} /usr/share/pgedge/packages.txt RUN < "$out" + chmod 0600 "$out" + + while IFS='|' read -r section key var; do + [ -n "$section" ] || continue + value=${!var-} + [ -n "$value" ] || continue + # Fatal, not skipped: skipping would surface as the tool's "no config + # file" instead of naming the variable at fault. + case $value in + *$'\n'*) + echo "coldfront: ${var} must not contain a newline" >&2 + exit 1 + ;; + esac + if [ "$section" != "$emitted" ]; then + printf '%s:\n' "$section" >> "$out" + emitted=$section + fi + case $value in + true | false) printf ' %s: %s\n' "$key" "$value" ;; + *) printf " %s: '%s'\n" "$key" "${value//\'/\'\'}" ;; + esac >> "$out" + done <<< "$_CF_FIELDS" + + [ -s "$out" ] +} + +case ${1:-} in +archiver | partitioner | compactor) + _cf_tool=$1 + shift + + _cf_flagged=0 + for _cf_arg; do + case $_cf_arg in -config | -config=* | --config | --config=*) _cf_flagged=1 ;; esac + done + + if [ "$_cf_flagged" = 0 ]; then + # COLDFRONT_CONFIG names a file to READ, never to write, and outranks + # rendering: COLDFRONT_WAREHOUSE and COLDFRONT_LAKEKEEPER are read by + # the postgres path too, so they are set even when a file is supplied. + _cf_rendered=${COLDFRONT_RENDER_CONFIG_TO:-/var/lib/pgedge/coldfront/config.yaml} + if [ -n "${COLDFRONT_CONFIG:-}" ]; then + set -- -config "$COLDFRONT_CONFIG" "$@" + elif _cf_render_config "$_cf_rendered"; then + echo "coldfront: rendered ${_cf_rendered} from the environment" >&2 + set -- -config "$_cf_rendered" "$@" + fi + fi + + exec "/usr/bin/${_cf_tool}" "$@" + ;; +esac + +# libpq keyword/value quoting for the loopback DSN: single-quote the value and +# backslash-escape backslashes and quotes, so a space or quote in a role or +# database name cannot split the DSN or end a value early. +_cf_dsn_quote() { + printf "'%s'" "$(printf '%s' "$1" | sed "s/[\\\\']/\\\\&/g")" +} + +# docker-entrypoint.sh turns a leading option into "postgres $@", but only after +# this wrapper has run. Normalising first keeps `run -c work_mem=...` on +# the branch below instead of silently starting without the preloads. +case ${1:-} in +-*) set -- postgres "$@" ;; +esac + +if [ "${1:-}" = "postgres" ]; then + shift + + # coldfront.so installs DML-routing hooks and an XactCallback, pg_duckdb + # installs planner and executor hooks, so both need preloading. autoinstall + # is off because the RPM ships all four loadable extensions and httpfs is + # compiled into libduckdb -- with allow_unsigned also on, leaving it enabled + # would mean fetching unsigned code at runtime. + args=( + -c shared_preload_libraries="${COLDFRONT_PRELOAD}" + -c duckdb.extension_directory="${COLDFRONT_EXTENSION_DIR}" + -c duckdb.autoinstall_known_extensions=false + -c duckdb.autoload_known_extensions=true + -c duckdb.allow_unsigned_extensions=true + ) + + # Unset rather than defaulted when absent: coldfront reads these with + # current_setting(..., true), so the image still starts as a plain node with + # the extension present but idle. + if [ -n "${COLDFRONT_WAREHOUSE:-}" ]; then + args+=(-c coldfront.warehouse="${COLDFRONT_WAREHOUSE}") + fi + if [ -n "${COLDFRONT_LAKEKEEPER:-}" ]; then + args+=(-c coldfront.lakekeeper_endpoint="${COLDFRONT_LAKEKEEPER}") + fi + + # Loopback DSN for coldfront.ensure_pg_attached(). Derived from the same + # variables pgEdge's entrypoint uses for the role and database, following + # its defaulting (POSTGRES_DB falls back to POSTGRES_USER). Override where + # the socket lives elsewhere -- CNPG forces /controller/run. + if [ -z "${COLDFRONT_LOCAL_PG_DSN:-}" ]; then + _cf_user="${POSTGRES_USER:-postgres}" + COLDFRONT_LOCAL_PG_DSN="host=$(_cf_dsn_quote "${COLDFRONT_SOCKET_DIR:-/var/run/postgresql}")" + COLDFRONT_LOCAL_PG_DSN+=" dbname=$(_cf_dsn_quote "${POSTGRES_DB:-${_cf_user}}")" + COLDFRONT_LOCAL_PG_DSN+=" user=$(_cf_dsn_quote "${_cf_user}")" + COLDFRONT_LOCAL_PG_DSN+=" application_name=coldfront_pglocal" + fi + args+=(-c coldfront.local_pg_dsn="${COLDFRONT_LOCAL_PG_DSN}") + + # pg_duckdb gates DuckDB on membership of this role; unset keeps its stock + # superuser-only default. + if [ -n "${COLDFRONT_DUCKDB_ROLE:-}" ]; then + args+=(-c duckdb.postgres_role="${COLDFRONT_DUCKDB_ROLE}") + fi + + # Operator arguments last, so an explicit -c wins. + set -- postgres "${args[@]}" "$@" +fi + +exec /usr/local/bin/docker-entrypoint.sh "$@" diff --git a/packagelists/amd64/pg16.15-postgres.txt b/packagelists/amd64/pg16.15-postgres.txt new file mode 100644 index 0000000..e0b60e3 --- /dev/null +++ b/packagelists/amd64/pg16.15-postgres.txt @@ -0,0 +1,2 @@ +pgedge-postgresql16-16.15-1.el9 +pgedge-postgresql16-server-16.15-1.el9 diff --git a/packagelists/amd64/pg16.15-spock5.0.11-coldfront.txt b/packagelists/amd64/pg16.15-spock5.0.11-coldfront.txt new file mode 100644 index 0000000..85c9410 --- /dev/null +++ b/packagelists/amd64/pg16.15-spock5.0.11-coldfront.txt @@ -0,0 +1,2 @@ +pgedge-coldfront_16-1.0.0-beta2_1.el9 +pgedge-coldfront-1.0.0-beta2_1.el9 diff --git a/packagelists/amd64/pg16.15-spock5.0.11-minimal.txt b/packagelists/amd64/pg16.15-spock5.0.11-minimal.txt index 52266aa..7bf91a4 100644 --- a/packagelists/amd64/pg16.15-spock5.0.11-minimal.txt +++ b/packagelists/amd64/pg16.15-spock5.0.11-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql16-16.15-1.el9 pgedge-spock50_16-5.0.11-1.el9 pgedge-snowflake_16-2.6.0-1.el9 pgedge-lolor_16-1.2.2-1.el9 diff --git a/packagelists/amd64/pg16.15-spock5.0.11-standard.txt b/packagelists/amd64/pg16.15-spock5.0.11-standard.txt index 210dbde..da0b6c2 100644 --- a/packagelists/amd64/pg16.15-spock5.0.11-standard.txt +++ b/packagelists/amd64/pg16.15-spock5.0.11-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql16-16.15-1.el9 -pgedge-spock50_16-5.0.11-1.el9 -pgedge-snowflake_16-2.6.0-1.el9 -pgedge-lolor_16-1.2.2-1.el9 pgedge-pgaudit_16-16.1-1.el9 pgedge-postgis36_16-3.6.4-1.el9 pgedge-pgvector_16-0.8.5-1.el9 diff --git a/packagelists/amd64/pg16.15-spock6.0.0-beta1-minimal.txt b/packagelists/amd64/pg16.15-spock6.0.0-beta1-minimal.txt index 1a4c084..87fc969 100644 --- a/packagelists/amd64/pg16.15-spock6.0.0-beta1-minimal.txt +++ b/packagelists/amd64/pg16.15-spock6.0.0-beta1-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql16-16.15-1.el9 pgedge-spock60_16-6.0.0-beta1_1.el9 pgedge-snowflake_16-2.6.0-1.el9 pgedge-lolor_16-1.2.2-1.el9 diff --git a/packagelists/amd64/pg16.15-spock6.0.0-beta1-standard.txt b/packagelists/amd64/pg16.15-spock6.0.0-beta1-standard.txt index 902bcc7..da0b6c2 100644 --- a/packagelists/amd64/pg16.15-spock6.0.0-beta1-standard.txt +++ b/packagelists/amd64/pg16.15-spock6.0.0-beta1-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql16-16.15-1.el9 -pgedge-spock60_16-6.0.0-beta1_1.el9 -pgedge-snowflake_16-2.6.0-1.el9 -pgedge-lolor_16-1.2.2-1.el9 pgedge-pgaudit_16-16.1-1.el9 pgedge-postgis36_16-3.6.4-1.el9 pgedge-pgvector_16-0.8.5-1.el9 diff --git a/packagelists/amd64/pg17.11-postgres.txt b/packagelists/amd64/pg17.11-postgres.txt new file mode 100644 index 0000000..e6b6f11 --- /dev/null +++ b/packagelists/amd64/pg17.11-postgres.txt @@ -0,0 +1,2 @@ +pgedge-postgresql17-17.11-1.el9 +pgedge-postgresql17-server-17.11-1.el9 diff --git a/packagelists/amd64/pg17.11-spock5.0.11-coldfront.txt b/packagelists/amd64/pg17.11-spock5.0.11-coldfront.txt new file mode 100644 index 0000000..a5b55a8 --- /dev/null +++ b/packagelists/amd64/pg17.11-spock5.0.11-coldfront.txt @@ -0,0 +1,2 @@ +pgedge-coldfront_17-1.0.0-beta2_1.el9 +pgedge-coldfront-1.0.0-beta2_1.el9 diff --git a/packagelists/amd64/pg17.11-spock5.0.11-minimal.txt b/packagelists/amd64/pg17.11-spock5.0.11-minimal.txt index a185cb9..920688d 100644 --- a/packagelists/amd64/pg17.11-spock5.0.11-minimal.txt +++ b/packagelists/amd64/pg17.11-spock5.0.11-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql17-17.11-1.el9 pgedge-spock50_17-5.0.11-1.el9 pgedge-snowflake_17-2.6.0-1.el9 pgedge-lolor_17-1.2.2-1.el9 diff --git a/packagelists/amd64/pg17.11-spock5.0.11-standard.txt b/packagelists/amd64/pg17.11-spock5.0.11-standard.txt index 3d3df4d..6098198 100644 --- a/packagelists/amd64/pg17.11-spock5.0.11-standard.txt +++ b/packagelists/amd64/pg17.11-spock5.0.11-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql17-17.11-1.el9 -pgedge-spock50_17-5.0.11-1.el9 -pgedge-snowflake_17-2.6.0-1.el9 -pgedge-lolor_17-1.2.2-1.el9 pgedge-pgaudit_17-17.1-1.el9 pgedge-postgis36_17-3.6.4-1.el9 pgedge-pgvector_17-0.8.5-1.el9 diff --git a/packagelists/amd64/pg17.11-spock6.0.0-beta1-minimal.txt b/packagelists/amd64/pg17.11-spock6.0.0-beta1-minimal.txt index 3ad4040..bdad999 100644 --- a/packagelists/amd64/pg17.11-spock6.0.0-beta1-minimal.txt +++ b/packagelists/amd64/pg17.11-spock6.0.0-beta1-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql17-17.11-1.el9 pgedge-spock60_17-6.0.0-beta1_1.el9 pgedge-snowflake_17-2.6.0-1.el9 pgedge-lolor_17-1.2.2-1.el9 diff --git a/packagelists/amd64/pg17.11-spock6.0.0-beta1-standard.txt b/packagelists/amd64/pg17.11-spock6.0.0-beta1-standard.txt index 925c82f..6098198 100644 --- a/packagelists/amd64/pg17.11-spock6.0.0-beta1-standard.txt +++ b/packagelists/amd64/pg17.11-spock6.0.0-beta1-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql17-17.11-1.el9 -pgedge-spock60_17-6.0.0-beta1_1.el9 -pgedge-snowflake_17-2.6.0-1.el9 -pgedge-lolor_17-1.2.2-1.el9 pgedge-pgaudit_17-17.1-1.el9 pgedge-postgis36_17-3.6.4-1.el9 pgedge-pgvector_17-0.8.5-1.el9 diff --git a/packagelists/amd64/pg18.6-postgres.txt b/packagelists/amd64/pg18.6-postgres.txt new file mode 100644 index 0000000..c363ba2 --- /dev/null +++ b/packagelists/amd64/pg18.6-postgres.txt @@ -0,0 +1,2 @@ +pgedge-postgresql18-18.6-1.el9 +pgedge-postgresql18-server-18.6-1.el9 diff --git a/packagelists/amd64/pg18.6-spock5.0.11-coldfront.txt b/packagelists/amd64/pg18.6-spock5.0.11-coldfront.txt new file mode 100644 index 0000000..f242c11 --- /dev/null +++ b/packagelists/amd64/pg18.6-spock5.0.11-coldfront.txt @@ -0,0 +1,2 @@ +pgedge-coldfront_18-1.0.0-beta2_1.el9 +pgedge-coldfront-1.0.0-beta2_1.el9 diff --git a/packagelists/amd64/pg18.6-spock5.0.11-minimal.txt b/packagelists/amd64/pg18.6-spock5.0.11-minimal.txt index 2f1fa16..a0a7793 100644 --- a/packagelists/amd64/pg18.6-spock5.0.11-minimal.txt +++ b/packagelists/amd64/pg18.6-spock5.0.11-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql18-18.6-1.el9 pgedge-spock50_18-5.0.11-1.el9 pgedge-snowflake_18-2.6.0-1.el9 pgedge-lolor_18-1.2.2-1.el9 diff --git a/packagelists/amd64/pg18.6-spock5.0.11-standard.txt b/packagelists/amd64/pg18.6-spock5.0.11-standard.txt index dddbfaf..82ece0c 100644 --- a/packagelists/amd64/pg18.6-spock5.0.11-standard.txt +++ b/packagelists/amd64/pg18.6-spock5.0.11-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql18-18.6-1.el9 -pgedge-spock50_18-5.0.11-1.el9 -pgedge-snowflake_18-2.6.0-1.el9 -pgedge-lolor_18-1.2.2-1.el9 pgedge-pgaudit_18-18.0-1.el9 pgedge-postgis36_18-3.6.4-1.el9 pgedge-pgvector_18-0.8.5-1.el9 diff --git a/packagelists/amd64/pg18.6-spock6.0.0-beta1-minimal.txt b/packagelists/amd64/pg18.6-spock6.0.0-beta1-minimal.txt index 4f57c19..ab432d1 100644 --- a/packagelists/amd64/pg18.6-spock6.0.0-beta1-minimal.txt +++ b/packagelists/amd64/pg18.6-spock6.0.0-beta1-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql18-18.6-1.el9 pgedge-spock60_18-6.0.0-beta1_1.el9 pgedge-snowflake_18-2.6.0-1.el9 pgedge-lolor_18-1.2.2-1.el9 diff --git a/packagelists/amd64/pg18.6-spock6.0.0-beta1-standard.txt b/packagelists/amd64/pg18.6-spock6.0.0-beta1-standard.txt index 444748b..82ece0c 100644 --- a/packagelists/amd64/pg18.6-spock6.0.0-beta1-standard.txt +++ b/packagelists/amd64/pg18.6-spock6.0.0-beta1-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql18-18.6-1.el9 -pgedge-spock60_18-6.0.0-beta1_1.el9 -pgedge-snowflake_18-2.6.0-1.el9 -pgedge-lolor_18-1.2.2-1.el9 pgedge-pgaudit_18-18.0-1.el9 pgedge-postgis36_18-3.6.4-1.el9 pgedge-pgvector_18-0.8.5-1.el9 diff --git a/packagelists/arm64/pg16.15-postgres.txt b/packagelists/arm64/pg16.15-postgres.txt new file mode 100644 index 0000000..e0b60e3 --- /dev/null +++ b/packagelists/arm64/pg16.15-postgres.txt @@ -0,0 +1,2 @@ +pgedge-postgresql16-16.15-1.el9 +pgedge-postgresql16-server-16.15-1.el9 diff --git a/packagelists/arm64/pg16.15-spock5.0.11-coldfront.txt b/packagelists/arm64/pg16.15-spock5.0.11-coldfront.txt new file mode 100644 index 0000000..85c9410 --- /dev/null +++ b/packagelists/arm64/pg16.15-spock5.0.11-coldfront.txt @@ -0,0 +1,2 @@ +pgedge-coldfront_16-1.0.0-beta2_1.el9 +pgedge-coldfront-1.0.0-beta2_1.el9 diff --git a/packagelists/arm64/pg16.15-spock5.0.11-minimal.txt b/packagelists/arm64/pg16.15-spock5.0.11-minimal.txt index 52266aa..7bf91a4 100644 --- a/packagelists/arm64/pg16.15-spock5.0.11-minimal.txt +++ b/packagelists/arm64/pg16.15-spock5.0.11-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql16-16.15-1.el9 pgedge-spock50_16-5.0.11-1.el9 pgedge-snowflake_16-2.6.0-1.el9 pgedge-lolor_16-1.2.2-1.el9 diff --git a/packagelists/arm64/pg16.15-spock5.0.11-standard.txt b/packagelists/arm64/pg16.15-spock5.0.11-standard.txt index 210dbde..da0b6c2 100644 --- a/packagelists/arm64/pg16.15-spock5.0.11-standard.txt +++ b/packagelists/arm64/pg16.15-spock5.0.11-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql16-16.15-1.el9 -pgedge-spock50_16-5.0.11-1.el9 -pgedge-snowflake_16-2.6.0-1.el9 -pgedge-lolor_16-1.2.2-1.el9 pgedge-pgaudit_16-16.1-1.el9 pgedge-postgis36_16-3.6.4-1.el9 pgedge-pgvector_16-0.8.5-1.el9 diff --git a/packagelists/arm64/pg16.15-spock6.0.0-beta1-minimal.txt b/packagelists/arm64/pg16.15-spock6.0.0-beta1-minimal.txt index 1a4c084..87fc969 100644 --- a/packagelists/arm64/pg16.15-spock6.0.0-beta1-minimal.txt +++ b/packagelists/arm64/pg16.15-spock6.0.0-beta1-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql16-16.15-1.el9 pgedge-spock60_16-6.0.0-beta1_1.el9 pgedge-snowflake_16-2.6.0-1.el9 pgedge-lolor_16-1.2.2-1.el9 diff --git a/packagelists/arm64/pg16.15-spock6.0.0-beta1-standard.txt b/packagelists/arm64/pg16.15-spock6.0.0-beta1-standard.txt index 902bcc7..da0b6c2 100644 --- a/packagelists/arm64/pg16.15-spock6.0.0-beta1-standard.txt +++ b/packagelists/arm64/pg16.15-spock6.0.0-beta1-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql16-16.15-1.el9 -pgedge-spock60_16-6.0.0-beta1_1.el9 -pgedge-snowflake_16-2.6.0-1.el9 -pgedge-lolor_16-1.2.2-1.el9 pgedge-pgaudit_16-16.1-1.el9 pgedge-postgis36_16-3.6.4-1.el9 pgedge-pgvector_16-0.8.5-1.el9 diff --git a/packagelists/arm64/pg17.11-postgres.txt b/packagelists/arm64/pg17.11-postgres.txt new file mode 100644 index 0000000..e6b6f11 --- /dev/null +++ b/packagelists/arm64/pg17.11-postgres.txt @@ -0,0 +1,2 @@ +pgedge-postgresql17-17.11-1.el9 +pgedge-postgresql17-server-17.11-1.el9 diff --git a/packagelists/arm64/pg17.11-spock5.0.11-coldfront.txt b/packagelists/arm64/pg17.11-spock5.0.11-coldfront.txt new file mode 100644 index 0000000..a5b55a8 --- /dev/null +++ b/packagelists/arm64/pg17.11-spock5.0.11-coldfront.txt @@ -0,0 +1,2 @@ +pgedge-coldfront_17-1.0.0-beta2_1.el9 +pgedge-coldfront-1.0.0-beta2_1.el9 diff --git a/packagelists/arm64/pg17.11-spock5.0.11-minimal.txt b/packagelists/arm64/pg17.11-spock5.0.11-minimal.txt index a185cb9..920688d 100644 --- a/packagelists/arm64/pg17.11-spock5.0.11-minimal.txt +++ b/packagelists/arm64/pg17.11-spock5.0.11-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql17-17.11-1.el9 pgedge-spock50_17-5.0.11-1.el9 pgedge-snowflake_17-2.6.0-1.el9 pgedge-lolor_17-1.2.2-1.el9 diff --git a/packagelists/arm64/pg17.11-spock5.0.11-standard.txt b/packagelists/arm64/pg17.11-spock5.0.11-standard.txt index 3d3df4d..6098198 100644 --- a/packagelists/arm64/pg17.11-spock5.0.11-standard.txt +++ b/packagelists/arm64/pg17.11-spock5.0.11-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql17-17.11-1.el9 -pgedge-spock50_17-5.0.11-1.el9 -pgedge-snowflake_17-2.6.0-1.el9 -pgedge-lolor_17-1.2.2-1.el9 pgedge-pgaudit_17-17.1-1.el9 pgedge-postgis36_17-3.6.4-1.el9 pgedge-pgvector_17-0.8.5-1.el9 diff --git a/packagelists/arm64/pg17.11-spock6.0.0-beta1-minimal.txt b/packagelists/arm64/pg17.11-spock6.0.0-beta1-minimal.txt index 3ad4040..bdad999 100644 --- a/packagelists/arm64/pg17.11-spock6.0.0-beta1-minimal.txt +++ b/packagelists/arm64/pg17.11-spock6.0.0-beta1-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql17-17.11-1.el9 pgedge-spock60_17-6.0.0-beta1_1.el9 pgedge-snowflake_17-2.6.0-1.el9 pgedge-lolor_17-1.2.2-1.el9 diff --git a/packagelists/arm64/pg17.11-spock6.0.0-beta1-standard.txt b/packagelists/arm64/pg17.11-spock6.0.0-beta1-standard.txt index 925c82f..6098198 100644 --- a/packagelists/arm64/pg17.11-spock6.0.0-beta1-standard.txt +++ b/packagelists/arm64/pg17.11-spock6.0.0-beta1-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql17-17.11-1.el9 -pgedge-spock60_17-6.0.0-beta1_1.el9 -pgedge-snowflake_17-2.6.0-1.el9 -pgedge-lolor_17-1.2.2-1.el9 pgedge-pgaudit_17-17.1-1.el9 pgedge-postgis36_17-3.6.4-1.el9 pgedge-pgvector_17-0.8.5-1.el9 diff --git a/packagelists/arm64/pg18.6-postgres.txt b/packagelists/arm64/pg18.6-postgres.txt new file mode 100644 index 0000000..c363ba2 --- /dev/null +++ b/packagelists/arm64/pg18.6-postgres.txt @@ -0,0 +1,2 @@ +pgedge-postgresql18-18.6-1.el9 +pgedge-postgresql18-server-18.6-1.el9 diff --git a/packagelists/arm64/pg18.6-spock5.0.11-coldfront.txt b/packagelists/arm64/pg18.6-spock5.0.11-coldfront.txt new file mode 100644 index 0000000..f242c11 --- /dev/null +++ b/packagelists/arm64/pg18.6-spock5.0.11-coldfront.txt @@ -0,0 +1,2 @@ +pgedge-coldfront_18-1.0.0-beta2_1.el9 +pgedge-coldfront-1.0.0-beta2_1.el9 diff --git a/packagelists/arm64/pg18.6-spock5.0.11-minimal.txt b/packagelists/arm64/pg18.6-spock5.0.11-minimal.txt index 2f1fa16..a0a7793 100644 --- a/packagelists/arm64/pg18.6-spock5.0.11-minimal.txt +++ b/packagelists/arm64/pg18.6-spock5.0.11-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql18-18.6-1.el9 pgedge-spock50_18-5.0.11-1.el9 pgedge-snowflake_18-2.6.0-1.el9 pgedge-lolor_18-1.2.2-1.el9 diff --git a/packagelists/arm64/pg18.6-spock5.0.11-standard.txt b/packagelists/arm64/pg18.6-spock5.0.11-standard.txt index dddbfaf..82ece0c 100644 --- a/packagelists/arm64/pg18.6-spock5.0.11-standard.txt +++ b/packagelists/arm64/pg18.6-spock5.0.11-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql18-18.6-1.el9 -pgedge-spock50_18-5.0.11-1.el9 -pgedge-snowflake_18-2.6.0-1.el9 -pgedge-lolor_18-1.2.2-1.el9 pgedge-pgaudit_18-18.0-1.el9 pgedge-postgis36_18-3.6.4-1.el9 pgedge-pgvector_18-0.8.5-1.el9 diff --git a/packagelists/arm64/pg18.6-spock6.0.0-beta1-minimal.txt b/packagelists/arm64/pg18.6-spock6.0.0-beta1-minimal.txt index 4f57c19..ab432d1 100644 --- a/packagelists/arm64/pg18.6-spock6.0.0-beta1-minimal.txt +++ b/packagelists/arm64/pg18.6-spock6.0.0-beta1-minimal.txt @@ -1,4 +1,3 @@ -pgedge-postgresql18-18.6-1.el9 pgedge-spock60_18-6.0.0-beta1_1.el9 pgedge-snowflake_18-2.6.0-1.el9 pgedge-lolor_18-1.2.2-1.el9 diff --git a/packagelists/arm64/pg18.6-spock6.0.0-beta1-standard.txt b/packagelists/arm64/pg18.6-spock6.0.0-beta1-standard.txt index 444748b..82ece0c 100644 --- a/packagelists/arm64/pg18.6-spock6.0.0-beta1-standard.txt +++ b/packagelists/arm64/pg18.6-spock6.0.0-beta1-standard.txt @@ -1,7 +1,3 @@ -pgedge-postgresql18-18.6-1.el9 -pgedge-spock60_18-6.0.0-beta1_1.el9 -pgedge-snowflake_18-2.6.0-1.el9 -pgedge-lolor_18-1.2.2-1.el9 pgedge-pgaudit_18-18.0-1.el9 pgedge-postgis36_18-3.6.4-1.el9 pgedge-pgvector_18-0.8.5-1.el9 diff --git a/pgedge.docker-bake.hcl b/pgedge.docker-bake.hcl index 70e1cda..12c81a3 100644 --- a/pgedge.docker-bake.hcl +++ b/pgedge.docker-bake.hcl @@ -21,6 +21,43 @@ variable "PACKAGE_LIST_FILE" { default = "" } +// Every stage in the chain needs its own packagelist variable. A chained flavor +// still triggers its ancestors' stages, and each of those consumes its own ARG, +// so one shared variable would make an ancestor install a descendant's list. +variable "POSTGRES_PACKAGE_LIST_FILE" { + type = string + default = "" +} + +variable "STANDARD_PACKAGE_LIST_FILE" { + type = string + default = "" +} + +variable "COLDFRONT_PACKAGE_LIST_FILE" { + type = string + default = "" +} + +// Select what each chained stage is built FROM. Empty keeps the in-Dockerfile +// default (the parent stage), which is the single-graph build. A registry +// reference switches that stage to start from an already-published image, which +// is what a per-flavor CI wave needs. +variable "POSTGRES_IMAGE" { + type = string + default = "postgres" +} + +variable "MINIMAL_IMAGE" { + type = string + default = "minimal" +} + +variable "STANDARD_IMAGE" { + type = string + default = "standard" +} + variable "TAG" { type = string default = "pgedge" @@ -31,9 +68,15 @@ target "default" { target = TARGET tags = [TAG] args = { - PACKAGE_RELEASE_CHANNEL = PACKAGE_RELEASE_CHANNEL - PACKAGE_LIST_FILE = PACKAGE_LIST_FILE - POSTGRES_MAJOR_VERSION = POSTGRES_MAJOR_VERSION + PACKAGE_RELEASE_CHANNEL = PACKAGE_RELEASE_CHANNEL + POSTGRES_PACKAGE_LIST_FILE = POSTGRES_PACKAGE_LIST_FILE + PACKAGE_LIST_FILE = PACKAGE_LIST_FILE + STANDARD_PACKAGE_LIST_FILE = STANDARD_PACKAGE_LIST_FILE + COLDFRONT_PACKAGE_LIST_FILE = COLDFRONT_PACKAGE_LIST_FILE + POSTGRES_IMAGE = POSTGRES_IMAGE + MINIMAL_IMAGE = MINIMAL_IMAGE + STANDARD_IMAGE = STANDARD_IMAGE + POSTGRES_MAJOR_VERSION = POSTGRES_MAJOR_VERSION } platforms = [ "linux/amd64", diff --git a/scripts/build_pgedge_images.py b/scripts/build_pgedge_images.py index 2a7a2f9..bccc9d7 100755 --- a/scripts/build_pgedge_images.py +++ b/scripts/build_pgedge_images.py @@ -34,6 +34,37 @@ def from_env() -> "Config": ) +# Flavors that are built FROM another flavor rather than FROM base. Building a +# chained flavor in one graph also runs every ancestor's stage, and each stage +# consumes its own packagelist ARG, so build() has to pass the whole ancestry -- +# see PgEdgeImage.package_list_args. +FLAVOR_PARENTS = {"minimal": "postgres", "standard": "minimal", "coldfront": "standard"} + +# Built once per major and shared by every spock line: a "-spock…-postgres" tag +# would claim a version the image does not contain. +SPOCK_INDEPENDENT_FLAVORS = {"postgres"} + +# The Dockerfile ARG that selects what a chained flavor is built FROM. Emitted +# with each cell so the workflow never has to know these names. +FLAVOR_IMAGE_ARGS = { + "minimal": "POSTGRES_IMAGE", + "standard": "MINIMAL_IMAGE", + "coldfront": "STANDARD_IMAGE", +} + +# The Dockerfile ARG each flavor's stage reads its packagelist from. +FLAVOR_LIST_ARGS = { + "postgres": "POSTGRES_PACKAGE_LIST_FILE", + "minimal": "PACKAGE_LIST_FILE", + "standard": "STANDARD_PACKAGE_LIST_FILE", + "coldfront": "COLDFRONT_PACKAGE_LIST_FILE", +} + +# Flavors built for every spock line. coldfront is opted in per image (its +# protocol is validated against spock 5 only); postgres is listed once per major. +DEFAULT_FLAVORS = ["minimal", "standard"] + + @dataclass class Tag: postgres_version: str @@ -72,16 +103,66 @@ def postgres_major(self) -> str: @property def spock_major(self) -> str: - return self.spock_version.split(".")[0] + return self.spock_version.split(".")[0] if self.spock_version else "" + + def _package_list_for(self, flavor: str) -> str: + filename = f"pg{self.postgres_version}" + + if flavor not in SPOCK_INDEPENDENT_FLAVORS: + filename += f"-spock{self.spock_version}" + + if flavor: + filename += f"-{flavor}" + + return filename + ".txt" @property def package_list(self) -> str: - filename = f"pg{self.postgres_version}-spock{self.spock_version}" + return self._package_list_for(self.flavor) - if self.flavor: - filename += f"-{self.flavor}" + @property + def parent_build_tag(self) -> str: + """Immutable tag of the flavor this one is chained FROM, or "" if none. + + The per-flavor wave model builds each flavor FROM the image the previous + wave published, so the stage is never rebuilt on a different runner. + """ + parent = FLAVOR_PARENTS.get(self.flavor) + if not parent: + return "" + return str( + Tag( + postgres_version=self.postgres_version, + flavor=parent, + spock_version=( + "" if parent in SPOCK_INDEPENDENT_FLAVORS else self.spock_version + ), + epoch=self.epoch, + ) + ) - return filename + ".txt" + @property + def ancestry(self) -> list[str]: + """This flavor and every flavor it is chained FROM, base-most first.""" + chain = [self.flavor] + while FLAVOR_PARENTS.get(chain[0]): + chain.insert(0, FLAVOR_PARENTS[chain[0]]) + return chain + + @property + def package_list_args(self) -> dict[str, str]: + """One packagelist build-arg per stage in this image's ancestry. + + A single-graph build of a chained flavor runs its ancestors' stages too, + and each reads its own ARG, so all of them have to be supplied. Args for + flavors outside the ancestry are sent empty so bake does not carry a + stale value over from another image. + """ + chain = self.ancestry + return { + arg: (self._package_list_for(flavor) if flavor in chain else "") + for flavor, arg in FLAVOR_LIST_ARGS.items() + } @property def build_tag(self) -> Tag: @@ -104,7 +185,7 @@ def extra_tags(self) -> list[Tag]: ) ] - if self.is_latest_for_spock_major: + if self.is_latest_for_spock_major and self.spock_version: # Mutable tag without spock minor/patch and epoch tags.append( Tag( @@ -123,6 +204,8 @@ def extra_tags(self) -> list[Tag]: spock_version=self.spock_major, ) ) + elif not self.spock_version and self.is_latest_for_pg_major: + tags.append(Tag(postgres_version=self.postgres_major, flavor=self.flavor)) return tags @@ -138,9 +221,10 @@ def make_all_flavor_images( is_latest_for_pg_major: bool = False, is_latest_for_spock_major: bool = False, package_release_channel: str = "", + flavors: list[str] = None, ) -> list[PgEdgeImage]: images: list[PgEdgeImage] = [] - for flavor in ["minimal", "standard"]: + for flavor in flavors if flavors is not None else DEFAULT_FLAVORS: images.append( PgEdgeImage( postgres_version=postgres_version, @@ -159,6 +243,19 @@ def make_all_flavor_images( # This is the list of all images that this script will build. Any new images should be # added to this list. all_images: list[PgEdgeImage] = [ + # PostgreSQL-only base, one per major; no spock segment. + PgEdgeImage( + postgres_version="16.15", spock_version="", epoch=2, flavor="postgres", + is_latest_for_pg_major=True, + ), + PgEdgeImage( + postgres_version="17.11", spock_version="", epoch=2, flavor="postgres", + is_latest_for_pg_major=True, + ), + PgEdgeImage( + postgres_version="18.6", spock_version="", epoch=2, flavor="postgres", + is_latest_for_pg_major=True, + ), # pg16 images *make_all_flavor_images( postgres_version="16.15", @@ -166,6 +263,7 @@ def make_all_flavor_images( epoch=2, is_latest_for_pg_major=True, is_latest_for_spock_major=True, + flavors=DEFAULT_FLAVORS + ["coldfront"], ), # pg17 images *make_all_flavor_images( @@ -174,6 +272,7 @@ def make_all_flavor_images( epoch=2, is_latest_for_pg_major=True, is_latest_for_spock_major=True, + flavors=DEFAULT_FLAVORS + ["coldfront"], ), # pg18 images *make_all_flavor_images( @@ -182,6 +281,7 @@ def make_all_flavor_images( epoch=2, is_latest_for_pg_major=True, is_latest_for_spock_major=True, + flavors=DEFAULT_FLAVORS + ["coldfront"], ), # pg16 spock60 images *make_all_flavor_images( @@ -210,6 +310,82 @@ def make_all_flavor_images( ] +# Runner label per architecture. arm64 builds go to a native runner rather than +# QEMU on an amd64 host: emulated dnf transactions dominate the build time. +ARCH_RUNNERS = {"amd64": "ubuntu-24.04", "arm64": "ubuntu-24.04-arm"} + +FLAVOR_WAVES = ["postgres", "minimal", "standard", "coldfront"] + + +def emit_matrix(config: "Config") -> None: + """Print the per-wave build and merge matrices as JSON. + + The workflow consumes this instead of hardcoding the cell list, so the + matrix and the image definitions above cannot drift apart. + + An image whose immutable tag is already published is left out of the build + matrix unless republish is set, but stays in the merge matrix with + needs_build false, so a re-dispatch repairs its mutable tags without + rebuilding anything. + """ + arches = [config.only_arch] if config.only_arch else list(ARCH_RUNNERS) + waves: dict = {} + + for flavor in FLAVOR_WAVES: + builds: list[dict] = [] + merges: list[dict] = [] + # Job label. A spock-independent flavor has no spock version to name. + cell = "{major}" if flavor in SPOCK_INDEPENDENT_FLAVORS else "{major}-spock{spock}" + + for image in all_images: + if image.flavor != flavor or _should_skip_image(image, config): + continue + + needs_build = config.republish or not published_digests( + config.repo, image.build_tag + ) + if not needs_build: + logging.info(f"{image.build_tag} is already published") + + merges.append( + { + "name": cell.format( + major=image.postgres_major, spock=image.spock_major + ), + "build_tag": str(image.build_tag), + "extra_tags": [str(t) for t in image.extra_tags], + "arches": arches, + "needs_build": needs_build, + } + ) + + if not needs_build: + continue + + for arch in arches: + builds.append( + { + "name": cell.format( + major=image.postgres_major, spock=image.spock_major + ) + + f"-{arch}", + "runner": ARCH_RUNNERS[arch], + "arch": arch, + "target": flavor, + "build_tag": str(image.build_tag), + "postgres_major": image.postgres_major, + "package_release_channel": image.package_release_channel, + "parent_build_tag": image.parent_build_tag, + "parent_image_arg": FLAVOR_IMAGE_ARGS.get(flavor, ""), + "package_list_args": image.package_list_args, + } + ) + + waves[flavor] = {"build": builds, "merge": merges} + + print(json.dumps(waves)) + + def validate_images(images: list[PgEdgeImage]): all_tags = set() @@ -284,7 +460,7 @@ def build( **os.environ.copy(), "PACKAGE_RELEASE_CHANNEL": image.package_release_channel, "POSTGRES_MAJOR_VERSION": image.postgres_major, - "PACKAGE_LIST_FILE": image.package_list, + **image.package_list_args, "TAG": f"{repo}:{image.build_tag}", "TARGET": image.flavor, }, @@ -339,7 +515,13 @@ def _log_config(config: "Config") -> None: def _should_skip_image(image: "PgEdgeImage", config: "Config") -> bool: if config.only_postgres_version and image.postgres_version != config.only_postgres_version: return True - if config.only_spock_version and image.spock_version != config.only_spock_version: + # Every spock line depends on the spock-independent base, so a spock filter + # must not exclude it. + if ( + config.only_spock_version + and image.spock_version + and image.spock_version != config.only_spock_version + ): return True return False @@ -380,6 +562,11 @@ def main(): print(",".join(get_latest_tags())) return + if os.getenv("PGEDGE_EMIT_MATRIX", "0") == "1": + validate_images(all_images) + emit_matrix(config) + return + _log_config(config) validate_images(all_images) diff --git a/tests/main.go b/tests/main.go index 7166e8f..f35a1be 100644 --- a/tests/main.go +++ b/tests/main.go @@ -27,7 +27,21 @@ type Test struct { Name string Cmd string ExpectedOutput func(exitCode int, output string) error - StandardOnly bool // Only run on standard flavor images + MinimalOnly bool // Only run on minimal-or-later (needs the pgEdge extensions) + StandardOnly bool // Only run on standard-or-later flavors (standard, coldfront) + ColdfrontOnly bool // Only run on the coldfront flavor +} + +// includesMinimal reports whether a flavor ships the pgEdge extensions minimal +// adds. postgres, the bare server, does not. +func includesMinimal(flavor string) bool { + return flavor == "minimal" || includesStandard(flavor) +} + +// includesStandard reports whether a flavor ships everything standard does. +// coldfront is chained FROM standard, so it is a superset. +func includesStandard(flavor string) bool { + return flavor == "standard" || flavor == "coldfront" } // TestRunner manages container lifecycle and test execution @@ -101,20 +115,20 @@ func spockMajorFromImage(image string) string { func parseFlags() (string, string) { image := flag.String("image", "", "Docker image to test (required)") - flavor := flag.String("flavor", "", "Image flavor: minimal or standard (required)") + flavor := flag.String("flavor", "", "Image flavor: postgres, minimal, standard or coldfront (required)") flag.Parse() if *image == "" || *flavor == "" { - fmt.Println("Usage: go run main.go -image -flavor ") + fmt.Println("Usage: go run main.go -image -flavor ") fmt.Println() fmt.Println("Arguments:") fmt.Println(" -image Docker image to test (e.g., ghcr.io/pgedge/pgedge-postgres:17-spock5-standard)") - fmt.Println(" -flavor Image flavor: 'minimal' or 'standard'") + fmt.Println(" -flavor Image flavor: 'postgres', 'minimal', 'standard' or 'coldfront'") os.Exit(1) } - if *flavor != "minimal" && *flavor != "standard" { - log.Fatalf("Invalid flavor '%s'. Must be 'minimal' or 'standard'", *flavor) + if *flavor != "postgres" && !includesMinimal(*flavor) { + log.Fatalf("Invalid flavor '%s'. Must be 'postgres', 'minimal', 'standard' or 'coldfront'", *flavor) } return *image, *flavor @@ -151,8 +165,8 @@ func runEntrypointTests(runner *DefaultEntrypointRunner, flavor string) int { } fmt.Println() - // Phase 2: Test Patroni entrypoint (standard only) - if flavor == "standard" { + // Phase 2: Test Patroni entrypoint (standard and the flavors chained from it) + if includesStandard(flavor) { printPhaseHeader("Phase 2: Patroni Entrypoint Test") if err := runner.TestPatroniEntrypoint(); err != nil { errorCount++ @@ -164,6 +178,19 @@ func runEntrypointTests(runner *DefaultEntrypointRunner, flavor string) int { fmt.Println() } + // Phase 2b: ColdFront's wrapper around that entrypoint + if flavor == "coldfront" { + printPhaseHeader("Phase 2b: ColdFront Entrypoint Test") + if err := runner.TestColdfrontEntrypoint(); err != nil { + errorCount++ + fmt.Printf(" ColdFront entrypoint test ❌\n") + log.Printf(" Error: %v", err) + } else { + fmt.Printf(" ColdFront entrypoint test ✅\n") + } + fmt.Println() + } + return errorCount } @@ -199,13 +226,20 @@ func printSummary(errorCount int, flavor, spockMajor string) { tests := buildTestSuite(spockMajor) extensionTests := 0 for _, t := range tests { - if !t.StandardOnly || flavor == "standard" { - extensionTests++ + if t.MinimalOnly && !includesMinimal(flavor) { + continue } + if t.StandardOnly && !includesStandard(flavor) { + continue + } + if t.ColdfrontOnly && flavor != "coldfront" { + continue + } + extensionTests++ } testsRun := 1 + extensionTests // default entrypoint + extensions - if flavor == "standard" { + if includesStandard(flavor) { testsRun++ // patroni entrypoint } @@ -341,6 +375,92 @@ patroni /tmp/patroni.yml`, patroniConfig) return resp.ID, nil } +// TestColdfrontEntrypoint starts the image the way an operator passing server +// options does -- a leading "-c" rather than an explicit "postgres" -- and with +// a space in the role and database names, so both ways the wrapper can lose +// ColdFront's settings are covered. +func (r *DefaultEntrypointRunner) TestColdfrontEntrypoint() error { + const ( + user = "cf user" + db = "cf db" + ) + + resp, err := r.cli.ContainerCreate(r.ctx, &container.Config{ + Image: r.image, + Env: []string{ + "POSTGRES_PASSWORD=testpassword", + "POSTGRES_USER=" + user, + "POSTGRES_DB=" + db, + "COLDFRONT_WAREHOUSE=wh", + }, + // docker-entrypoint.sh only turns this into "postgres -c ..." after the + // ColdFront wrapper has run, so the wrapper has to normalise it itself. + Cmd: []string{"-c", "work_mem=8MB"}, + }, &container.HostConfig{}, nil, nil, "") + if err != nil { + return fmt.Errorf("error creating container: %w", err) + } + defer r.cleanupContainer(resp.ID) + + if err := r.cli.ContainerStart(r.ctx, resp.ID, container.StartOptions{}); err != nil { + return fmt.Errorf("error starting container: %w", err) + } + + psql := func(sql string) (string, error) { + exitCode, out, err := execInContainer(r.cli, r.ctx, resp.ID, + []string{"psql", "-U", user, "-d", db, "-X", "-t", "-A", "-c", sql}) + if err != nil { + return "", err + } + if exitCode != 0 { + return "", fmt.Errorf("psql exited %d: %s", exitCode, strings.TrimSpace(out)) + } + return strings.TrimSpace(out), nil + } + + // A real query, not pg_isready: initdb's own bootstrap server answers before + // the postmaster this test is about is listening. + ready := false + for deadline := time.Now().Add(90 * time.Second); time.Now().Before(deadline); { + if _, err := psql("SELECT 1"); err == nil { + ready = true + break + } + time.Sleep(2 * time.Second) + } + if !ready { + return fmt.Errorf("timeout waiting for PostgreSQL to be ready") + } + + checks := []struct { + name, sql, want string + }{ + // Present only if the leading option did not bypass the wrapper. + {"shared_preload_libraries", "SHOW shared_preload_libraries", "pg_duckdb,coldfront"}, + // The operator's own argument is appended last and still wins. + {"work_mem", "SHOW work_mem", "8MB"}, + {"coldfront.warehouse", "SHOW coldfront.warehouse", "wh"}, + // Spaces have to be quoted, not split into further libpq keywords. + {"coldfront.local_pg_dsn", "SHOW coldfront.local_pg_dsn", + "host='/var/run/postgresql' dbname='" + db + "' user='" + user + "' application_name=coldfront_pglocal"}, + } + for _, c := range checks { + got, err := psql(c.sql) + if err != nil { + return fmt.Errorf("%s: %w", c.name, err) + } + if got != c.want { + return fmt.Errorf("%s = %q, want %q", c.name, got, c.want) + } + } + + // Connects back over that DSN, which a value split on its spaces could not do. + if _, err := psql("CREATE EXTENSION IF NOT EXISTS coldfront CASCADE; SELECT coldfront.ensure_pg_attached()"); err != nil { + return fmt.Errorf("ensure_pg_attached: %w", err) + } + return nil +} + func (r *DefaultEntrypointRunner) cleanupContainer(containerID string) { r.cli.ContainerStop(r.ctx, containerID, container.StopOptions{}) r.cli.ContainerRemove(r.ctx, containerID, container.RemoveOptions{}) @@ -407,10 +527,19 @@ func (r *TestRunner) Start() error { // Build shared_preload_libraries based on flavor // These extensions require preloading before they can be used // Note: We only include extensions that are guaranteed to be in all images - sharedLibs := "spock,snowflake" - if r.flavor == "standard" { + sharedLibs := "" + if includesMinimal(r.flavor) { + sharedLibs = "spock,snowflake" + } + if includesStandard(r.flavor) { sharedLibs = "spock,snowflake,pgaudit,supautils" } + // pg_duckdb and coldfront install hooks at postmaster start. The image's own + // entrypoint already passes them, but the -c built below is appended after it + // and would otherwise replace the value. + if r.flavor == "coldfront" { + sharedLibs += ",pg_duckdb,coldfront" + } // Build postgres command with required configuration // Note: We pass these as postgres arguments, which the entrypoint will handle @@ -421,7 +550,11 @@ func (r *TestRunner) Start() error { "-c", "track_commit_timestamp=on", "-c", "max_replication_slots=10", "-c", "max_wal_senders=10", - "-c", "snowflake.node=1", + } + // snowflake.node exists only once that extension is preloaded; passing it to + // the bare server makes the postmaster refuse to start. + if includesMinimal(r.flavor) { + cmd = append(cmd, "-c", "snowflake.node=1") } resp, err := r.cli.ContainerCreate(r.ctx, &container.Config{ @@ -595,7 +728,13 @@ func (r *TestRunner) exec(cmd string) (int, string, error) { return -1, "", fmt.Errorf("empty command") } - execID, err := r.cli.ContainerExecCreate(r.ctx, r.containerID, container.ExecOptions{ + return execInContainer(r.cli, r.ctx, r.containerID, cmdArgs) +} + +// execInContainer runs an already-parsed argv in a container and returns its +// exit code with stdout and stderr interleaved. +func execInContainer(cli *client.Client, ctx context.Context, containerID string, cmdArgs []string) (int, string, error) { + execID, err := cli.ContainerExecCreate(ctx, containerID, container.ExecOptions{ Cmd: cmdArgs, AttachStdout: true, AttachStderr: true, @@ -604,19 +743,18 @@ func (r *TestRunner) exec(cmd string) (int, string, error) { return -1, "", fmt.Errorf("error creating exec: %w", err) } - resp, err := r.cli.ContainerExecAttach(r.ctx, execID.ID, container.ExecAttachOptions{}) + resp, err := cli.ContainerExecAttach(ctx, execID.ID, container.ExecAttachOptions{}) if err != nil { return -1, "", fmt.Errorf("error attaching to exec: %w", err) } defer resp.Close() var outputBuf bytes.Buffer - _, err = stdcopy.StdCopy(&outputBuf, &outputBuf, resp.Reader) - if err != nil { + if _, err := stdcopy.StdCopy(&outputBuf, &outputBuf, resp.Reader); err != nil { return -1, "", fmt.Errorf("error copying output: %w", err) } - inspectResp, err := r.cli.ContainerExecInspect(r.ctx, execID.ID) + inspectResp, err := cli.ContainerExecInspect(ctx, execID.ID) if err != nil { return -1, "", fmt.Errorf("error inspecting exec: %w", err) } @@ -629,7 +767,13 @@ func (r *TestRunner) RunTests(tests []Test) int { for _, test := range tests { // Skip standard-only tests for minimal flavor - if test.StandardOnly && r.flavor != "standard" { + if test.MinimalOnly && !includesMinimal(r.flavor) { + continue + } + if test.StandardOnly && !includesStandard(r.flavor) { + continue + } + if test.ColdfrontOnly && r.flavor != "coldfront" { continue } @@ -664,6 +808,7 @@ func buildTestSuite(spockMajor string) []Test { // Runs after getCommonExtensionTests, which creates the spock extension. tests = append(tests, getSpockVersionTests(spockMajor)...) tests = append(tests, getStandardOnlyTests()...) + tests = append(tests, getColdfrontTests()...) return tests } @@ -678,8 +823,9 @@ func getSpockVersionTests(spockMajor string) []Test { } return []Test{ { - Name: fmt.Sprintf("Spock extension major version is %s", spockMajor), - Cmd: "psql -U postgres -d testdb -t -A -c \"SELECT extversion FROM pg_extension WHERE extname = 'spock';\"", + Name: fmt.Sprintf("Spock extension major version is %s", spockMajor), + MinimalOnly: true, + Cmd: "psql -U postgres -d testdb -t -A -c \"SELECT extversion FROM pg_extension WHERE extname = 'spock';\"", ExpectedOutput: func(exitCode int, output string) error { if exitCode != 0 { return fmt.Errorf("unexpected exit code: %d", exitCode) @@ -732,12 +878,14 @@ func getCommonExtensionTests() []Test { return []Test{ { Name: "Spock extension can be created", + MinimalOnly: true, Cmd: "psql -U postgres -d testdb -t -A -c \"CREATE EXTENSION IF NOT EXISTS spock; SELECT 1;\"", ExpectedOutput: expectSuccess, }, { - Name: "Spock subscription table accessible", - Cmd: "psql -U postgres -d testdb -t -A -c \"SELECT count(*) FROM spock.subscription;\"", + Name: "Spock subscription table accessible", + MinimalOnly: true, + Cmd: "psql -U postgres -d testdb -t -A -c \"SELECT count(*) FROM spock.subscription;\"", ExpectedOutput: func(exitCode int, output string) error { if exitCode != 0 { return fmt.Errorf("unexpected exit code: %d", exitCode) @@ -750,12 +898,14 @@ func getCommonExtensionTests() []Test { }, { Name: "LOLOR extension can be created", + MinimalOnly: true, Cmd: "psql -U postgres -d testdb -t -A -c \"CREATE EXTENSION IF NOT EXISTS lolor; SELECT 1;\"", ExpectedOutput: expectSuccess, }, { - Name: "LOLOR lo_create works", - Cmd: "psql -U postgres -d testdb -t -A -c \"SELECT lo_create(200000);\"", + Name: "LOLOR lo_create works", + MinimalOnly: true, + Cmd: "psql -U postgres -d testdb -t -A -c \"SELECT lo_create(200000);\"", ExpectedOutput: func(exitCode int, output string) error { if exitCode != 0 { return fmt.Errorf("unexpected exit code: %d", exitCode) @@ -768,12 +918,14 @@ func getCommonExtensionTests() []Test { }, { Name: "Snowflake extension can be created", + MinimalOnly: true, Cmd: "psql -U postgres -d testdb -t -A -c \"CREATE EXTENSION IF NOT EXISTS snowflake; SELECT 1;\"", ExpectedOutput: expectSuccess, }, { - Name: "Snowflake ID generation works", - Cmd: "psql -U postgres -d testdb -t -A -c \"SELECT snowflake.nextval() > 0;\"", + Name: "Snowflake ID generation works", + MinimalOnly: true, + Cmd: "psql -U postgres -d testdb -t -A -c \"SELECT snowflake.nextval() > 0;\"", ExpectedOutput: func(exitCode int, output string) error { if exitCode != 0 { return fmt.Errorf("unexpected exit code: %d", exitCode) @@ -787,6 +939,81 @@ func getCommonExtensionTests() []Test { } } +// rpmExtensionDir is where pgedge-coldfront-duckdb-extensions installs the +// DuckDB extension binaries. duckdb.extension_directory points here and +// duckdb.autoinstall_known_extensions is off, so a successful load proves the +// extensions are read from the read-only package path rather than fetched. +const rpmExtensionDir = "/usr/lib/pgedge/coldfront/duckdb-extensions" + +func expectTrimmed(want string) func(int, string) error { + return func(exitCode int, output string) error { + if exitCode != 0 { + return fmt.Errorf("unexpected exit code: %d", exitCode) + } + if got := strings.TrimSpace(output); got != want { + return fmt.Errorf("expected %q, got %q", want, got) + } + return nil + } +} + +func getColdfrontTests() []Test { + loadAll := `SELECT duckdb.load_extension('iceberg');` + + `SELECT duckdb.load_extension('avro');` + + `SELECT duckdb.load_extension('azure');` + + `SELECT duckdb.load_extension('postgres_scanner');` + + `SELECT * FROM duckdb.query('SELECT count(*) FROM duckdb_extensions() ` + + `WHERE loaded AND install_path LIKE ''` + rpmExtensionDir + `%''');` + + return []Test{ + { + Name: "pg_duckdb extension can be created", + ColdfrontOnly: true, + Cmd: `psql -U postgres -d testdb -t -A -c "CREATE EXTENSION IF NOT EXISTS pg_duckdb; SELECT 1;"`, + ExpectedOutput: expectSuccess, + }, + { + Name: "coldfront extension can be created", + ColdfrontOnly: true, + Cmd: `psql -U postgres -d testdb -t -A -c "CREATE EXTENSION IF NOT EXISTS coldfront CASCADE; SELECT 1;"`, + ExpectedOutput: expectSuccess, + }, + { + Name: "DuckDB executes a query", + ColdfrontOnly: true, + Cmd: `psql -U postgres -d testdb -t -A -c "SELECT * FROM duckdb.query('SELECT 42');"`, + ExpectedOutput: expectTrimmed("42"), + }, + { + Name: "duckdb.extension_directory points at the package path", + ColdfrontOnly: true, + Cmd: `psql -U postgres -d testdb -t -A -c "SHOW duckdb.extension_directory;"`, + ExpectedOutput: expectTrimmed(rpmExtensionDir), + }, + { + Name: "DuckDB extension autoinstall is disabled", + ColdfrontOnly: true, + Cmd: `psql -U postgres -d testdb -t -A -c "SHOW duckdb.autoinstall_known_extensions;"`, + ExpectedOutput: expectTrimmed("off"), + }, + { + Name: "all four DuckDB extensions load from the package path", + ColdfrontOnly: true, + Cmd: `psql -U postgres -d testdb -t -A -c "` + loadAll + `"`, + ExpectedOutput: func(exitCode int, output string) error { + if exitCode != 0 { + return fmt.Errorf("unexpected exit code: %d", exitCode) + } + fields := strings.Fields(strings.TrimSpace(output)) + if len(fields) == 0 || fields[len(fields)-1] != "4" { + return fmt.Errorf("expected 4 extensions loaded from %s, got: %s", rpmExtensionDir, output) + } + return nil + }, + }, + } +} + func getStandardOnlyTests() []Test { tests := append(getSystemStatsAndVectorTests(), getPostGISAuditBackrestTests()...) return append(tests, getSupautilsTests()...)