Refactor Dockerfiles to use ARG for base images and add ACR sync work…

.github/workflows/build_docker_images.yml

@@ -21,6 +21,23 @@ jobs:
   docker_build:
     name: Build images
     runs-on: ubuntu-latest
+    env:
+      PYTHON_BASE_IMAGE: >-
+        ${{ vars.ACR_BASE_IMAGE_PREFIX != '' &&
+        format('{0}/mirror/library/python:3.12-slim-bookworm', vars.ACR_BASE_IMAGE_PREFIX)
+        || 'python:3.12-slim-bookworm' }}
+      MAVEN_BASE_IMAGE: >-
+        ${{ vars.ACR_BASE_IMAGE_PREFIX != '' &&
+        format('{0}/mirror/library/maven:3.9-eclipse-temurin-17-alpine', vars.ACR_BASE_IMAGE_PREFIX)
+        || 'maven:3.9-eclipse-temurin-17-alpine' }}
+      GUACD_BASE_IMAGE: >-
+        ${{ vars.ACR_BASE_IMAGE_PREFIX != '' &&
+        format('{0}/mirror/guacamole/guacd:1.6.0', vars.ACR_BASE_IMAGE_PREFIX)
+        || 'guacamole/guacd:1.6.0' }}
+      GITEA_BASE_IMAGE_REPO: >-
+        ${{ vars.ACR_BASE_IMAGE_PREFIX != '' &&
+        format('{0}/mirror/gitea/gitea', vars.ACR_BASE_IMAGE_PREFIX)
+        || 'gitea/gitea' }}
     permissions:
       contents: read
       actions: write # For uploading artifacts
@@ -137,6 +154,8 @@ jobs:
         with:
           context: ./api_app/
           file: ./api_app/Dockerfile
+          build-args: |
+            PYTHON_BASE_IMAGE=${{ env.PYTHON_BASE_IMAGE }}
           outputs: type=local,dest=test-results
           target: test-results
           cache-from: type=gha
@@ -157,6 +176,8 @@ jobs:
         with:
           context: ./api_app/
           file: ./api_app/Dockerfile
+          build-args: |
+            PYTHON_BASE_IMAGE=${{ env.PYTHON_BASE_IMAGE }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -168,6 +189,8 @@ jobs:
         with:
           context: ./resource_processor
           file: ./resource_processor/vmss_porter/Dockerfile
+          build-args: |
+            PYTHON_BASE_IMAGE=${{ env.PYTHON_BASE_IMAGE }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -179,6 +202,9 @@ jobs:
         with:
           context: ./templates/workspace_services/guacamole/guacamole-server
           file: ./templates/workspace_services/guacamole/guacamole-server/docker/Dockerfile
+          build-args: |
+            MAVEN_BASE_IMAGE=${{ env.MAVEN_BASE_IMAGE }}
+            GUACD_BASE_IMAGE=${{ env.GUACD_BASE_IMAGE }}
           outputs: type=local,dest=test-results
           target: test-results
           cache-from: type=gha
@@ -199,6 +225,9 @@ jobs:
         with:
           context: ./templates/workspace_services/guacamole/guacamole-server
           file: ./templates/workspace_services/guacamole/guacamole-server/docker/Dockerfile
+          build-args: |
+            MAVEN_BASE_IMAGE=${{ env.MAVEN_BASE_IMAGE }}
+            GUACD_BASE_IMAGE=${{ env.GUACD_BASE_IMAGE }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
 
@@ -210,6 +239,8 @@ jobs:
         with:
           context: ./templates/shared_services/gitea/docker
           file: ./templates/shared_services/gitea/docker/Dockerfile
+          build-args: |
+            GITEA_BASE_IMAGE_REPO=${{ env.GITEA_BASE_IMAGE_REPO }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
 

.github/workflows/sync_acr_base_images.yml

@@ -0,0 +1,111 @@
+---
+name: Sync ACR Base Images
+
+on: # yamllint disable-line rule:truthy
+  schedule:
+    - cron: "23 2 * * *"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  id-token: write
+
+jobs:
+  sync:
+    name: Import newer upstream base images into ACR
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
+        with:
+          persist-credentials: false
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd
+
+      - name: Validate required secrets
+        run: |
+          if [ "${{ secrets.ACR_NAME }}" = '' ]; then
+            echo "Missing secret: ACR_NAME" && exit 1
+          fi
+          if [ "${{ secrets.AZURE_CLIENT_ID }}" = '' ]; then
+            echo "Missing secret: AZURE_CLIENT_ID" && exit 1
+          fi
+          if [ "${{ secrets.AZURE_TENANT_ID }}" = '' ]; then
+            echo "Missing secret: AZURE_TENANT_ID" && exit 1
+          fi
+          if [ "${{ secrets.AZURE_SUBSCRIPTION_ID }}" = '' ]; then
+            echo "Missing secret: AZURE_SUBSCRIPTION_ID" && exit 1
+          fi
+
+      - name: Azure Login
+        uses: azure/login@532459ea530d8321f2fb9bb10d1e0bcf23869a43 # v3.0.0
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+          environment: ${{ (vars.AZURE_ENVIRONMENT != '' && vars.AZURE_ENVIRONMENT) || 'AzureCloud' }}
+
+      - name: Login to ACR
+        run: az acr login --name "${{ secrets.ACR_NAME }}"
+
+      - name: Sync image mirrors
+        env:
+          ACR_NAME: ${{ secrets.ACR_NAME }}
+        run: |
+          set -euo pipefail
+
+          acr_domain_suffix=$(az cloud show --query suffixes.acrLoginServerEndpoint --output tsv)
+          ACR_LOGIN_SERVER="${ACR_NAME}${acr_domain_suffix}"
+
+          # source_repo|target_repo|tag
+          IMAGES=(
+            "docker.io/library/python|mirror/library/python|3.12-slim-bookworm"
+            "docker.io/library/maven|mirror/library/maven|3.9-eclipse-temurin-17-alpine"
+            "docker.io/guacamole/guacd|mirror/guacamole/guacd|1.6.0"
+            "docker.io/gitea/gitea|mirror/gitea/gitea|1.15"
+            "docker.io/gitea/gitea|mirror/gitea/gitea|1.17.3"
+            "docker.io/library/debian|mirror/library/debian|bookworm-slim"
+            "docker.io/library/debian|mirror/library/debian|bullseye-slim"
+          )
+
+          sync_image() {
+            local source_repo="$1"
+            local target_repo="$2"
+            local tag="$3"
+            local source_ref="${source_repo}:${tag}"
+            local target_ref="${ACR_LOGIN_SERVER}/${target_repo}:${tag}"
+            local digest_format='{{json .Manifest.Digest}}'
+
+            local source_digest
+            local target_digest
+
+            source_digest=$(docker buildx imagetools inspect "${source_ref}" \
+              --format "${digest_format}" 2>/dev/null | tr -d '"' || true)
+            target_digest=$(docker buildx imagetools inspect "${target_ref}" \
+              --format "${digest_format}" 2>/dev/null | tr -d '"' || true)
+
+            if [[ -n "${source_digest}" && -n "${target_digest}" && "${source_digest}" == "${target_digest}" ]]; then
+              echo "Up-to-date: ${target_repo}:${tag} (${source_digest})"
+              return 0
+            fi
+
+            echo "Importing ${source_ref} -> ${target_repo}:${tag}"
+            az acr import \
+              --name "${ACR_NAME}" \
+              --source "${source_ref}" \
+              --image "${target_repo}:${tag}" \
+              --force
+          }
+
+          for image in "${IMAGES[@]}"; do
+            IFS='|' read -r source_repo target_repo tag <<< "${image}"
+            sync_image "${source_repo}" "${target_repo}" "${tag}"
+          done
+
+      - name: Emit repository variable hint
+        env:
+          ACR_NAME: ${{ secrets.ACR_NAME }}
+        run: |
+          acr_domain_suffix=$(az cloud show --query suffixes.acrLoginServerEndpoint --output tsv)
+          echo "Set repository variable ACR_BASE_IMAGE_PREFIX to: ${ACR_NAME}${acr_domain_suffix}"

api_app/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:3.12-slim-bookworm AS base
+ARG PYTHON_BASE_IMAGE=python:3.12-slim-bookworm
+FROM ${PYTHON_BASE_IMAGE} AS base
 
 COPY requirements.txt /.
 RUN pip3 install --no-cache-dir -r requirements.txt

api_app/_version.py

@@ -1 +1 @@
-__version__ = "0.25.27"
+__version__ = "0.25.28"

docs/tre-admins/setup-instructions/cicd-pre-deployment-steps.md

@@ -95,6 +95,7 @@ Configure the following **variables** in your github environment:
 | ----------- | ----------- |
 | `AZURE_ENVIRONMENT` | Optional. The Azure cloud environment. Default is `AzureCloud`. Use `AzureUSGovernment` for US Government cloud |
 | `LOCATION` | The Azure location (region) for all resources. E.g. `westeurope` |
+| `ACR_BASE_IMAGE_PREFIX` | Optional. ACR login server prefix used by image build workflows to pull mirrored base images instead of directly pulling from Docker Hub. Example value: `myacr.azurecr.io`. |
 | `TERRAFORM_STATE_CONTAINER_NAME` | Optional. The name of the blob container to hold the Terraform state. Default value is `tfstate`. |
 | `CORE_ADDRESS_SPACE` | Optional. The address space for the Azure TRE core virtual network. Default value is `10.0.0.0/22`. |
 | `TRE_ADDRESS_SPACE` | Optional. The address space for the whole TRE environment virtual network where workspaces networks will be created (can include the core network as well). Default value is `10.0.0.0/16` |
@@ -108,6 +109,20 @@ Configure the following **variables** in your github environment:
 | `CUSTOM_DOMAIN` | Optional. Custom domain name to access the Azure TRE portal. See [Custom domain name](../custom-domain.md). |
 | `ENABLE_CMK_ENCRYPTION` | Optional. Default is `false`, if set to `true` customer-managed key encryption will be enabled for all supported resources. |
 
+### Optional: Mirror Docker Hub base images into ACR for CI
+
+To reduce Docker Hub rate-limit exposure in CI, Azure TRE includes a dedicated workflow to mirror selected upstream base images into your ACR and use those mirrors during image builds.
+
+1. Set repository/environment variable `ACR_BASE_IMAGE_PREFIX` to your ACR login server (for example, `myacr.azurecr.io`).
+2. Ensure the following secrets are configured for the mirror workflow:
+- `ACR_NAME`
+- `AZURE_CLIENT_ID`
+- `AZURE_TENANT_ID`
+- `AZURE_SUBSCRIPTION_ID`
+3. Run `/.github/workflows/sync_acr_base_images.yml` manually once to seed mirrored images, then allow the scheduled run to keep them up to date.
+
+The Docker image build workflow (`/.github/workflows/build_docker_images.yml`) automatically uses mirrored base images when `ACR_BASE_IMAGE_PREFIX` is set, and falls back to upstream image sources when it is not set.
+
 ### Configure Authentication Secrets
 
 In a previous [Setup Auth configuration](./setup-auth-entities.md) step authentication configuration was added in `config.yaml` file. Go to this file and add those env vars to your github environment:

docs/tre-admins/setup-instructions/workflows.md

@@ -12,12 +12,12 @@ Before you can run the `deploy_tre.yml` workflow there are some one-time configu
     In some of the steps below, you are asked to configure repository secrets. Follow the [GitHub guide](https://docs.github.com/en/actions/security-guides/encrypted-secrets) on creating repository secrets if you are unfamiliar with this step.
 
 1. Create a service principal for the subscription so that the workflow can provision Azure resources.
-1. Decide on a TRE ID and the location for the Azure resources
-1. Create app registrations for API authentication
-1. Create app registrations and a user for the E2E tests
-1. Create a workspace app registration for setting up workspaces (for the E2E tests)
-1. Configure repository secrets
-1. Deploy the TRE using the workflow
+2. Decide on a TRE ID and the location for the Azure resources
+3. Create app registrations for API authentication
+4. Create app registrations and a user for the E2E tests
+5. Create a workspace app registration for setting up workspaces (for the E2E tests)
+6. Configure repository secrets
+7. Deploy the TRE using the workflow
 
 ### Create a service principal and configure OIDC authentication
 
@@ -33,7 +33,7 @@ Before you can run the `deploy_tre.yml` workflow there are some one-time configu
 
   See [Sign in with Azure CLI](https://docs.microsoft.com/cli/azure/authenticate-azure-cli) for more details.
 
-1. Create a service principal
+2. Create a service principal
 
   A service principal needs to be created to authorize CI/CD workflows to provision resources for the TRE workspaces and workspace services.
 
@@ -46,7 +46,7 @@ Before you can run the `deploy_tre.yml` workflow there are some one-time configu
   !!! caution
       Save the output (especially the `appId` and `tenant`) - you will need it for the next steps
 
-1. Configure federated identity credentials for GitHub Actions OIDC
+3. Configure federated identity credentials for GitHub Actions OIDC
 
   Configure the service principal to trust GitHub Actions OIDC tokens from your repository:
 
@@ -72,7 +72,7 @@ Before you can run the `deploy_tre.yml` workflow there are some one-time configu
 
   See [Configure a federated identity credential on an app](https://learn.microsoft.com/entra/workload-id/workload-identity-federation-create-trust?pivots=identity-wif-apps-methods-azcli) for more details.
 
-1. Configure repository secrets for OIDC authentication
+4. Configure repository secrets for OIDC authentication
 
   Configure the following **secrets** (not variables) in your repository or environment:
 
@@ -147,6 +147,7 @@ Configure variables used in the deployment workflow:
 | <div style="width: 230px">Variable name</div> | Description |
 | ----------- | ----------- |
 | `LOCATION` | The Azure location (region) for all resources. E.g. `westeurope` |
+| `ACR_BASE_IMAGE_PREFIX` | Optional. ACR login server prefix used by image build workflows to pull mirrored base images instead of directly pulling from Docker Hub. Example value: `myacr.azurecr.io`. |
 | `TERRAFORM_STATE_CONTAINER_NAME` | Optional. The name of the blob container to hold the Terraform state. Default value is `tfstate`. |
 | `CORE_ADDRESS_SPACE` | Optional. The address space for the Azure TRE core virtual network. Default value is `10.0.0.0/22`. |
 | `TRE_ADDRESS_SPACE` | Optional. The address space for the whole TRE environment virtual network where workspaces networks will be created (can include the core network as well). Default value is `10.0.0.0/16` |
@@ -159,6 +160,20 @@ Configure variables used in the deployment workflow:
 | `APP_GATEWAY_SKU` | Optional. The SKU of the Application Gateway. Default value is `Standard_v2`. Allowed values [`Standard_v2`, `WAF_v2`] |
 | `ENABLE_CMK_ENCRYPTION` | Optional. Default is `false`, if set to `true` customer-managed key encryption will be enabled for all supported resources. |
 
+### Optional: Mirror Docker Hub base images into ACR for CI
+
+To reduce Docker Hub rate-limit exposure in CI, Azure TRE includes a dedicated workflow to mirror selected upstream base images into your ACR and use those mirrors during image builds.
+
+1. Set repository/environment variable `ACR_BASE_IMAGE_PREFIX` to your ACR login server (for example, `myacr.azurecr.io`).
+2. Ensure the following secrets are configured for the mirror workflow:
+- `ACR_NAME`
+- `AZURE_CLIENT_ID`
+- `AZURE_TENANT_ID`
+- `AZURE_SUBSCRIPTION_ID`
+3. Run `/.github/workflows/sync_acr_base_images.yml` manually once to seed mirrored images, then allow the scheduled run to keep them up to date.
+
+The Docker image build workflow (`/.github/workflows/build_docker_images.yml`) automatically uses mirrored base images when `ACR_BASE_IMAGE_PREFIX` is set, and falls back to upstream image sources when it is not set.
+
 ### Deploy the TRE using the workflow
 
 With all the repository secrets set, you can trigger a workflow run by pushing to develop/main of your fork, or by dispatching the workflow manually.

resource_processor/_version.py

@@ -1 +1 @@
-__version__ = "0.13.4"
+__version__ = "0.13.5"

resource_processor/vmss_porter/Dockerfile

@@ -1,5 +1,6 @@
 # syntax=docker/dockerfile:1
-FROM python:3.12-slim-bookworm
+ARG PYTHON_BASE_IMAGE=python:3.12-slim-bookworm
+FROM ${PYTHON_BASE_IMAGE}
 
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 

templates/shared_services/gitea/docker/Dockerfile

@@ -1,7 +1,8 @@
 ARG GITEA_TAG=1.15
+ARG GITEA_BASE_IMAGE_REPO=gitea/gitea
 ARG CERTIFICATE_URL=https://www.digicert.com/CACerts/BaltimoreCyberTrustRoot.crt.pem
 
-FROM gitea/gitea:${GITEA_TAG}
+FROM ${GITEA_BASE_IMAGE_REPO}:${GITEA_TAG}
 # need to pass args to stage
 ARG CERTIFICATE_URL
 RUN wget -nv -O /usr/local/share/ca-certificates/mysql.crt.pem ${CERTIFICATE_URL} && update-ca-certificates

templates/shared_services/gitea/docker/version.txt

@@ -1 +1 @@
-__version__ = "0.3.12"
+__version__ = "0.3.13"

templates/workspace_services/gitea/docker/Dockerfile

@@ -1,7 +1,8 @@
 ARG GITEA_TAG=1.17.3
+ARG GITEA_BASE_IMAGE_REPO=gitea/gitea
 ARG CERTIFICATE_URL=https://www.digicert.com/CACerts/BaltimoreCyberTrustRoot.crt.pem
 
-FROM gitea/gitea:${GITEA_TAG}
+FROM ${GITEA_BASE_IMAGE_REPO}:${GITEA_TAG}
 # need to pass args to stage
 ARG CERTIFICATE_URL
 RUN wget -q -O /usr/local/share/ca-certificates/mysql.crt.pem ${CERTIFICATE_URL} && update-ca-certificates

templates/workspace_services/gitea/version.txt

@@ -1 +1 @@
-__version__ = "0.6.1"
+__version__ = "0.6.2"

templates/workspace_services/guacamole/guacamole-server/docker/Dockerfile

@@ -1,4 +1,6 @@
-FROM maven:3.9-eclipse-temurin-17-alpine AS client_build
+ARG MAVEN_BASE_IMAGE=maven:3.9-eclipse-temurin-17-alpine
+ARG GUACD_BASE_IMAGE=guacamole/guacd:1.6.0
+FROM ${MAVEN_BASE_IMAGE} AS client_build
 
 COPY ./guacamole-auth-azure/pom.xml /pom.xml
 # cache dependencies in a separate layer
@@ -11,7 +13,7 @@ RUN bash /tmp/maven_package_and_exit_succesfully.sh
 FROM scratch AS test-results
 COPY --from=client_build /target/surefire-reports/* /
 
-FROM guacamole/guacd:1.6.0
+FROM ${GUACD_BASE_IMAGE}
 
 ARG GUACAMOLE_AZURE_VERSION=0.5.0
 

templates/workspace_services/guacamole/guacamole-server/docker/version.txt

@@ -1 +1 @@
-__version__ = "0.10.0"
+__version__ = "0.10.1"