feat(track): clarify master-guard failure output (IN-3714)

src/commands/track/promote_production.yaml

@@ -18,3 +18,8 @@ steps:
         IMAGE_REGISTRY: "<< parameters.image_registry>>"
         COMPONENT_NAMES: "<< parameters.modifiedComponentNames >>"
       command: <<include(scripts/track/promote_production.sh)>>
+
+  - store_artifacts:
+      path: /tmp/promote-production-logs
+      destination: promote-production-logs
+      when: always

src/scripts/track/promote_production.sh

@@ -3,6 +3,37 @@ set -eE
 
 BUCKET="com.voiceflow.ci.assets"
 
+# Full command trace is redirected to an artifact file instead of the console,
+# so the main log stays readable. Published as the "promote-production-logs"
+# CircleCI artifact after the run.
+LOG_DIR="/tmp/promote-production-logs"
+mkdir -p "$LOG_DIR"
+exec {trace_fd}>"${LOG_DIR}/promote-production.trace.log"
+export BASH_XTRACEFD="$trace_fd"
+set -x
+
+CURRENT_PHASE="startup"
+
+on_error() {
+  local exit_code=$?
+  cat <<EOF
+
+================================================================================
+  TAG IMAGES FAILED (exit ${exit_code}) during: ${CURRENT_PHASE}
+================================================================================
+
+  This is a real failure in the image-tagging/promotion step (unlike the
+  master-guard check that runs before it). Full command trace is in the
+  "promote-production-logs" CircleCI artifact:
+
+      ${LOG_DIR}/promote-production.trace.log
+
+================================================================================
+
+EOF
+}
+trap on_error ERR
+
 echo "IMAGE_REGISTRY=${IMAGE_REGISTRY}"
 echo "BUCKET=${BUCKET}"
 
@@ -82,14 +113,23 @@ copy_track() {
   aws s3 cp --no-progress "${MASTER_TRACK}" "s3://${BUCKET}/${PRODUCTION_TRACK}"
 }
 
+echo "==> Fetching master tracks for ${#COMPONENT_NAMES[@]} component(s)..."
+CURRENT_PHASE="fetch master tracks"
 get_master_tracks "${TMP_DIR}" "${COMPONENT_NAMES[@]}"
 
-echo "Adding production tag and updating tracks for..."
-
+echo "==> Tagging images and updating production tracks..."
 for INDEX in "${!COMPONENT_NAMES[@]}"; do
   NAME="${COMPONENT_NAMES[$INDEX]}"
+  echo "  - ${NAME}"
+
+  CURRENT_PHASE="parse tag for ${NAME}"
   TAG="$(parse_tag "$TMP_DIR" "${NAME}")"
 
+  CURRENT_PHASE="tag image ${NAME}:${TAG}"
   add_production_tags "${NAME}" "${TAG}"
+
+  CURRENT_PHASE="copy track for ${NAME}"
   copy_track "${TMP_DIR}" "${NAME}"
 done
+
+echo "==> Done. Promoted ${#COMPONENT_NAMES[@]} component(s) to production."

src/scripts/utils/check_wait_master.sh

@@ -1,12 +1,23 @@
 #!/bin/bash
 
-set -x
-
 # Static
 ORG="gh/voiceflow"
 BRANCH="master"
 REVISION="$CIRCLE_SHA1"
 PROJECT="$CIRCLE_PROJECT_REPONAME"
+APP_ORG="github/${ORG#gh/}"   # gh/voiceflow -> github/voiceflow (app.circleci.com URL segment)
+
+MAX_RETRY=25
+INTERVAL=60
+
+# Full command trace (including the raw CircleCI API payloads) is redirected to
+# an artifact file instead of the console, so the main log stays readable. It is
+# published as the "promote-production-logs" CircleCI artifact after the run.
+LOG_DIR="/tmp/promote-production-logs"
+mkdir -p "$LOG_DIR"
+exec {trace_fd}>"${LOG_DIR}/check-wait-master.trace.log"
+export BASH_XTRACEFD="$trace_fd"
+set -x
 
 list_pipelines() {
   ##
@@ -24,23 +35,17 @@ get_pipeline() {
   ##
   # find master pipeline by long commit hash
   ##
-  local LENGTH
-  local PIPELINES
-  local PAGE_TOKEN
-  local MAX_PAGES
-  local COUNT
-
+  local LENGTH PIPELINES PAGE_TOKEN MAX_PAGES COUNT
   LENGTH=0
   COUNT=0
   MAX_PAGES=10
 
   while [[ "$LENGTH" == 0 && "$((COUNT++))" -lt "$MAX_PAGES" ]]; do
     PIPELINES=$(list_pipelines "${PAGE_TOKEN}")
-    PAGE_TOKEN=$(<<<"$PIPELINES" jq -r '.next_page_token // empty' || echo "")
-
-    LENGTH=$(<<<"$PIPELINES" jq --arg revision "${REVISION}" '.items | map(select(.vcs.revision == $revision and .state != "setup")) | length')
+    PAGE_TOKEN=$(jq -r '.next_page_token // empty' <<<"$PIPELINES" || echo "")
+    LENGTH=$(jq --arg revision "${REVISION}" '.items | map(select(.vcs.revision == $revision and .state != "setup")) | length' <<<"$PIPELINES")
   done
-  <<<"$PIPELINES" jq -r --arg revision "${REVISION}" '.items | map(select(.vcs.revision == $revision and .state != "setup")) | first | .id'
+  jq -r --arg revision "${REVISION}" '.items | map(select(.vcs.revision == $revision and .state != "setup")) | first | .id' <<<"$PIPELINES"
 }
 
 get_workflow() {
@@ -55,33 +60,108 @@ get_workflow() {
     | jq -r '.items | map(select(.name != "setup")) | first'
 }
 
+workflow_url() {
+  ##
+  # build a clickable app.circleci.com URL for the master workflow from the
+  # workflow JSON we already fetched
+  ##
+  local workflow_json pipeline_number workflow_id
+  workflow_json="$1"
+  pipeline_number="$(jq -r '.pipeline_number' <<<"$workflow_json")"
+  workflow_id="$(jq -r '.id' <<<"$workflow_json")"
+  echo "https://app.circleci.com/pipelines/${APP_ORG}/${PROJECT}/${pipeline_number}/workflows/${workflow_id}"
+}
+
+print_blocked_banner() {
+  ##
+  # explain that a non-successful master workflow blocked promotion on purpose
+  ##
+  local status url
+  status="$1"
+  url="$2"
+  cat <<EOF
+
+================================================================================
+  PRODUCTION PROMOTION BLOCKED - THIS IS EXPECTED, NOT A BUG IN THIS JOB
+================================================================================
+
+  This job did its one job correctly: it stopped the promotion because the
+  matching MASTER workflow for this commit is not green. No broken master image
+  was promoted to production.
+
+  >> Do NOT re-run this production job. It will keep failing until master is
+     fixed - the production branch and this job are working as intended.
+
+  >> Look at the MASTER workflow instead. That is where the real failure is:
+
+       Master workflow status : ${status}
+       Master workflow        : ${url}
+
+  Once that master workflow is green, promotion for this commit succeeds
+  automatically on the next production build. Nothing to fix here.
+
+================================================================================
+
+EOF
+}
+
+echo "Looking for the master workflow matching this commit (${REVISION})..."
+
 MASTER_PIPELINE_ID=$(get_pipeline)
 
+if [[ -z "$MASTER_PIPELINE_ID" || "$MASTER_PIPELINE_ID" == "null" ]]; then
+  cat <<EOF
+
+================================================================================
+  PRODUCTION PROMOTION BLOCKED - NO MASTER WORKFLOW FOUND FOR THIS COMMIT
+================================================================================
+
+  Could not find a master-branch pipeline for commit ${REVISION}. Either the
+  master build for this commit has not started yet, or this commit was never on
+  master. Promotion is blocked until a master workflow for this commit exists
+  and succeeds.
+
+  >> Do NOT re-run this production job to force it through. Check master's
+     builds here: https://app.circleci.com/pipelines/${APP_ORG}/${PROJECT}?branch=master
+
+================================================================================
+
+EOF
+  exit 1
+fi
 WORKFLOW=$(get_workflow "$MASTER_PIPELINE_ID")
 
+# Persist the resolved pipeline id and workflow JSON to the artifact for debugging.
+echo "$MASTER_PIPELINE_ID" > "${LOG_DIR}/master-pipeline-id.txt"
+echo "$WORKFLOW" > "${LOG_DIR}/master-workflow.json"
+
+MASTER_URL="$(workflow_url "$WORKFLOW")"
+echo "Found master workflow: ${MASTER_URL}"
+
 COUNT=0
-MAX_RETRY=25
-INTERVAL=60
 while [[ "$((COUNT++))" -lt "$MAX_RETRY" ]]; do
-  case $(<<<"$WORKFLOW" jq -r '.status') in
+  STATUS=$(jq -r '.status' <<<"$WORKFLOW")
+  case "$STATUS" in
     "running")
-      echo "waiting for master workflow to finish"
-      echo "sleep ${INTERVAL}s..."
+      echo "Master workflow is still running. Waiting ${INTERVAL}s before re-checking [attempt ${COUNT}/${MAX_RETRY}]..."
       sleep "$INTERVAL"
       WORKFLOW=$(get_workflow "$MASTER_PIPELINE_ID")
+      echo "$WORKFLOW" > "${LOG_DIR}/master-workflow.json"
       ;;
 
     "success")
-      echo "Found master workflow successfully completed"
+      echo "Master workflow succeeded. Proceeding with production promotion."
       exit 0
       ;;
 
     *)
-      echo "master workflow not successful. exiting..."
+      echo "Master workflow finished with status: ${STATUS} (not a success)."
+      print_blocked_banner "$STATUS" "$MASTER_URL"
       exit 1
       ;;
   esac
 done
 
-echo "Timing out"
+echo "Timed out after ${MAX_RETRY} checks (~$((MAX_RETRY * INTERVAL / 60)) min) waiting for the master workflow to finish."
+print_blocked_banner "timed out (still running or stuck)" "$MASTER_URL"
 exit 1