diff --git a/src/commands/track/promote_production.yaml b/src/commands/track/promote_production.yaml index a92f7f13..b1fc924f 100644 --- a/src/commands/track/promote_production.yaml +++ b/src/commands/track/promote_production.yaml @@ -18,3 +18,8 @@ steps: IMAGE_REGISTRY: "<< parameters.image_registry>>" COMPONENT_NAMES: "<< parameters.modifiedComponentNames >>" command: <> + + - store_artifacts: + path: /tmp/promote-production-logs + destination: promote-production-logs + when: always diff --git a/src/scripts/track/promote_production.sh b/src/scripts/track/promote_production.sh index 85dd009e..112d9040 100644 --- a/src/scripts/track/promote_production.sh +++ b/src/scripts/track/promote_production.sh @@ -3,6 +3,37 @@ set -eE BUCKET="com.voiceflow.ci.assets" +# Full command trace is redirected to an artifact file instead of the console, +# so the main log stays readable. Published as the "promote-production-logs" +# CircleCI artifact after the run. +LOG_DIR="/tmp/promote-production-logs" +mkdir -p "$LOG_DIR" +exec {trace_fd}>"${LOG_DIR}/promote-production.trace.log" +export BASH_XTRACEFD="$trace_fd" +set -x + +CURRENT_PHASE="startup" + +on_error() { + local exit_code=$? + cat < Fetching master tracks for ${#COMPONENT_NAMES[@]} component(s)..." +CURRENT_PHASE="fetch master tracks" get_master_tracks "${TMP_DIR}" "${COMPONENT_NAMES[@]}" -echo "Adding production tag and updating tracks for..." - +echo "==> Tagging images and updating production tracks..." for INDEX in "${!COMPONENT_NAMES[@]}"; do NAME="${COMPONENT_NAMES[$INDEX]}" + echo " - ${NAME}" + + CURRENT_PHASE="parse tag for ${NAME}" TAG="$(parse_tag "$TMP_DIR" "${NAME}")" + CURRENT_PHASE="tag image ${NAME}:${TAG}" add_production_tags "${NAME}" "${TAG}" + + CURRENT_PHASE="copy track for ${NAME}" copy_track "${TMP_DIR}" "${NAME}" done + +echo "==> Done. Promoted ${#COMPONENT_NAMES[@]} component(s) to production." diff --git a/src/scripts/utils/check_wait_master.sh b/src/scripts/utils/check_wait_master.sh index b70ad5e0..57824ddd 100644 --- a/src/scripts/utils/check_wait_master.sh +++ b/src/scripts/utils/check_wait_master.sh @@ -1,12 +1,23 @@ #!/bin/bash -set -x - # Static ORG="gh/voiceflow" BRANCH="master" REVISION="$CIRCLE_SHA1" PROJECT="$CIRCLE_PROJECT_REPONAME" +APP_ORG="github/${ORG#gh/}" # gh/voiceflow -> github/voiceflow (app.circleci.com URL segment) + +MAX_RETRY=25 +INTERVAL=60 + +# Full command trace (including the raw CircleCI API payloads) is redirected to +# an artifact file instead of the console, so the main log stays readable. It is +# published as the "promote-production-logs" CircleCI artifact after the run. +LOG_DIR="/tmp/promote-production-logs" +mkdir -p "$LOG_DIR" +exec {trace_fd}>"${LOG_DIR}/check-wait-master.trace.log" +export BASH_XTRACEFD="$trace_fd" +set -x list_pipelines() { ## @@ -24,23 +35,17 @@ get_pipeline() { ## # find master pipeline by long commit hash ## - local LENGTH - local PIPELINES - local PAGE_TOKEN - local MAX_PAGES - local COUNT - + local LENGTH PIPELINES PAGE_TOKEN MAX_PAGES COUNT LENGTH=0 COUNT=0 MAX_PAGES=10 while [[ "$LENGTH" == 0 && "$((COUNT++))" -lt "$MAX_PAGES" ]]; do PIPELINES=$(list_pipelines "${PAGE_TOKEN}") - PAGE_TOKEN=$(<<<"$PIPELINES" jq -r '.next_page_token // empty' || echo "") - - LENGTH=$(<<<"$PIPELINES" jq --arg revision "${REVISION}" '.items | map(select(.vcs.revision == $revision and .state != "setup")) | length') + PAGE_TOKEN=$(jq -r '.next_page_token // empty' <<<"$PIPELINES" || echo "") + LENGTH=$(jq --arg revision "${REVISION}" '.items | map(select(.vcs.revision == $revision and .state != "setup")) | length' <<<"$PIPELINES") done - <<<"$PIPELINES" jq -r --arg revision "${REVISION}" '.items | map(select(.vcs.revision == $revision and .state != "setup")) | first | .id' + jq -r --arg revision "${REVISION}" '.items | map(select(.vcs.revision == $revision and .state != "setup")) | first | .id' <<<"$PIPELINES" } get_workflow() { @@ -55,33 +60,108 @@ get_workflow() { | jq -r '.items | map(select(.name != "setup")) | first' } +workflow_url() { + ## + # build a clickable app.circleci.com URL for the master workflow from the + # workflow JSON we already fetched + ## + local workflow_json pipeline_number workflow_id + workflow_json="$1" + pipeline_number="$(jq -r '.pipeline_number' <<<"$workflow_json")" + workflow_id="$(jq -r '.id' <<<"$workflow_json")" + echo "https://app.circleci.com/pipelines/${APP_ORG}/${PROJECT}/${pipeline_number}/workflows/${workflow_id}" +} + +print_blocked_banner() { + ## + # explain that a non-successful master workflow blocked promotion on purpose + ## + local status url + status="$1" + url="$2" + cat <> Do NOT re-run this production job. It will keep failing until master is + fixed - the production branch and this job are working as intended. + + >> Look at the MASTER workflow instead. That is where the real failure is: + + Master workflow status : ${status} + Master workflow : ${url} + + Once that master workflow is green, promotion for this commit succeeds + automatically on the next production build. Nothing to fix here. + +================================================================================ + +EOF +} + +echo "Looking for the master workflow matching this commit (${REVISION})..." + MASTER_PIPELINE_ID=$(get_pipeline) +if [[ -z "$MASTER_PIPELINE_ID" || "$MASTER_PIPELINE_ID" == "null" ]]; then + cat <> Do NOT re-run this production job to force it through. Check master's + builds here: https://app.circleci.com/pipelines/${APP_ORG}/${PROJECT}?branch=master + +================================================================================ + +EOF + exit 1 +fi WORKFLOW=$(get_workflow "$MASTER_PIPELINE_ID") +# Persist the resolved pipeline id and workflow JSON to the artifact for debugging. +echo "$MASTER_PIPELINE_ID" > "${LOG_DIR}/master-pipeline-id.txt" +echo "$WORKFLOW" > "${LOG_DIR}/master-workflow.json" + +MASTER_URL="$(workflow_url "$WORKFLOW")" +echo "Found master workflow: ${MASTER_URL}" + COUNT=0 -MAX_RETRY=25 -INTERVAL=60 while [[ "$((COUNT++))" -lt "$MAX_RETRY" ]]; do - case $(<<<"$WORKFLOW" jq -r '.status') in + STATUS=$(jq -r '.status' <<<"$WORKFLOW") + case "$STATUS" in "running") - echo "waiting for master workflow to finish" - echo "sleep ${INTERVAL}s..." + echo "Master workflow is still running. Waiting ${INTERVAL}s before re-checking [attempt ${COUNT}/${MAX_RETRY}]..." sleep "$INTERVAL" WORKFLOW=$(get_workflow "$MASTER_PIPELINE_ID") + echo "$WORKFLOW" > "${LOG_DIR}/master-workflow.json" ;; "success") - echo "Found master workflow successfully completed" + echo "Master workflow succeeded. Proceeding with production promotion." exit 0 ;; *) - echo "master workflow not successful. exiting..." + echo "Master workflow finished with status: ${STATUS} (not a success)." + print_blocked_banner "$STATUS" "$MASTER_URL" exit 1 ;; esac done -echo "Timing out" +echo "Timed out after ${MAX_RETRY} checks (~$((MAX_RETRY * INTERVAL / 60)) min) waiting for the master workflow to finish." +print_blocked_banner "timed out (still running or stuck)" "$MASTER_URL" exit 1