Issue 4940: Add skip deployment flag for PR bot tests

.github/scripts/build.js

@@ -51,6 +51,7 @@ async function getCommandFromComment({ core, context, github }) {
   const commentBody = context.payload.comment.body;
   const commentFirstLine = commentBody.split("\n")[0];
   let command = "none";
+  let skipDeployment = false;
   const trimmedFirstLine = commentFirstLine.trim();
   if (trimmedFirstLine[0] === "/") {
     // only allow actions for users with write access
@@ -83,6 +84,7 @@ async function getCommandFromComment({ core, context, github }) {
           const runTests = await handleTestCommand({ core, github }, parts, "tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink });
           if (runTests) {
             command = "run-tests";
+            skipDeployment = commandHasSkipDeploymentFlag(parts);
           }
           break;
         }
@@ -92,6 +94,7 @@ async function getCommandFromComment({ core, context, github }) {
           const runTests = await handleTestCommand({ core, github }, parts, "extended tests", runId, { number: prNumber, authorUsername: prAuthorUsername, repoOwner, repoName, headSha: prHeadSha, refId: prRefId, details: pr }, { username: commentUsername, link: commentLink });
           if (runTests) {
             command = "run-tests-extended";
+            skipDeployment = commandHasSkipDeploymentFlag(parts);
           }
           break;
         }
@@ -138,6 +141,7 @@ async function getCommandFromComment({ core, context, github }) {
         break;
     }
   }
+  logAndSetOutput(core, "skipDeployment", skipDeployment.toString());
   logAndSetOutput(core, "command", command);
   return command;
 }
@@ -159,19 +163,19 @@ async function handleTestCommand({ core, github }, commandParts, testDescription
   const prAuthorHasWriteAccess = await userHasWriteAccessToRepo({ core, github }, pr.authorUsername, pr.repoOwner, pr.repoName);
   const externalPr = !prAuthorHasWriteAccess;
   if (externalPr) {
-    if (commandParts.length === 1) {
+    const commentSha = getShaFromCommandParts(commandParts);
+    if (!commentSha) {
       const message = `:warning: When using \`${command}\` on external PRs, the SHA of the checked commit must be specified`;
       await addActionComment({ github }, pr.repoOwner, pr.repoName, pr.number, comment.username, comment.link, message);
       return false;
     }
-    const commentSha = commandParts[1];
     if (commentSha.length < 7) {
       const message = `:warning: When specifying a commit SHA it must be at least 7 characters (received \`${commentSha}\`)`;
       await addActionComment({ github }, pr.repoOwner, pr.repoName, pr.number, comment.username, comment.link, message);
       return false;
     }
     if (!pr.headSha.startsWith(commentSha)) {
-      const message = `:warning: The specified SHA \`${commentSha}\` is not the latest commit on the PR. Please validate the latest commit and re-run \`/test\``;
+      const message = `:warning: The specified SHA \`${commentSha}\` is not the latest commit on the PR. Please validate the latest commit and re-run \`${command}\``;
       await addActionComment({ github }, pr.repoOwner, pr.repoName, pr.number, comment.username, comment.link, message);
       return false;
     }
@@ -183,6 +187,14 @@ async function handleTestCommand({ core, github }, commandParts, testDescription
 
 }
 
+function commandHasSkipDeploymentFlag(commandParts) {
+  return commandParts.slice(1).includes("skip_deployment");
+}
+
+function getShaFromCommandParts(commandParts) {
+  return commandParts.slice(1).find(part => part !== "skip_deployment");
+}
+
 async function prContainsNonDocChanges(github, repoOwner, repoName, prNumber) {
   const prFilesResponse = await github.paginate(github.rest.pulls.listFiles, {
     owner: repoOwner,
@@ -246,10 +258,10 @@ async function showHelp({ github }, repoOwner, repoName, prNumber, commentUser,
   const body = `${leadingContent}
 
 You can use the following commands:
-&nbsp;&nbsp;&nbsp;&nbsp;/test - build, deploy and run smoke tests on a PR
-&nbsp;&nbsp;&nbsp;&nbsp;/test-extended - build, deploy and run smoke & extended tests on a PR
-&nbsp;&nbsp;&nbsp;&nbsp;/test-extended-aad - build, deploy and run smoke & extended AAD tests on a PR
-&nbsp;&nbsp;&nbsp;&nbsp;/test-shared-services - test the deployment of shared services on a PR build
+&nbsp;&nbsp;&nbsp;&nbsp;/test [<sha>] [skip_deployment] - build, deploy and run smoke tests on a PR
+&nbsp;&nbsp;&nbsp;&nbsp;/test-extended [<sha>] [skip_deployment] - build, deploy and run smoke & extended tests on a PR
+&nbsp;&nbsp;&nbsp;&nbsp;/test-extended-aad [<sha>] - build, deploy and run smoke & extended AAD tests on a PR
+&nbsp;&nbsp;&nbsp;&nbsp;/test-shared-services [<sha>] - test the deployment of shared services on a PR build
 &nbsp;&nbsp;&nbsp;&nbsp;/test-force-approve - force approval of the PR tests (i.e. skip the deployment checks)
 &nbsp;&nbsp;&nbsp;&nbsp;/test-destroy-env - delete the validation environment for a PR (e.g. to enable testing a deployment from a clean start after previous tests)
 &nbsp;&nbsp;&nbsp;&nbsp;/help - show this help`;

.github/scripts/build.test.js

@@ -125,6 +125,27 @@ describe('getCommandFromComment', () => {
           expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests');
         });
 
+        test(`should set skipDeployment to 'true' when skip_deployment is supplied`, async () => {
+          const context = createCommentContext({
+            username: 'admin',
+            body: '/test skip_deployment',
+            pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
+          });
+          await getCommandFromComment({ core, context, github });
+          expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests');
+          expect(outputFor(mockCoreSetOutput, 'skipDeployment')).toBe('true');
+        });
+
+        test(`should set skipDeployment to 'false' by default`, async () => {
+          const context = createCommentContext({
+            username: 'admin',
+            body: '/test',
+            pullRequestNumber: PR_NUMBER.UPSTREAM_NON_DOCS_CHANGES,
+          });
+          await getCommandFromComment({ core, context, github });
+          expect(outputFor(mockCoreSetOutput, 'skipDeployment')).toBe('false');
+        });
+
         test(`should set nonDocsChanges to 'true'`, async () => {
           const context = createCommentContext({
             username: 'admin',
@@ -331,6 +352,20 @@ describe('getCommandFromComment', () => {
         });
       })
 
+      describe(`for '/test 2345678 skip_deployment' for external PR`, () => {
+        test(`should set command to 'run-tests' and skipDeployment to 'true'`, async () => {
+          const context = createCommentContext({
+            username: 'admin',
+            body: '/test 2345678 skip_deployment',
+            pullRequestNumber: PR_NUMBER.FORK_NON_DOCS_CHANGES,
+            authorUsername: 'non-contributor',
+          });
+          await getCommandFromComment({ core, context, github });
+          expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests');
+          expect(outputFor(mockCoreSetOutput, 'skipDeployment')).toBe('true');
+        });
+      })
+
       describe(`for '/test  2345678' for external PR (i.e. with latest commit SHA specified but extra space after test)`, () => {
         test(`should set command to 'run-tests'`, async () => {
           const context = createCommentContext({
@@ -370,6 +405,16 @@ describe('getCommandFromComment', () => {
           expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-extended');
         });
 
+        test(`should set skipDeployment to 'true' when skip_deployment is supplied`, async () => {
+          const context = createCommentContext({
+            username: 'admin',
+            body: '/test-extended skip_deployment',
+          });
+          await getCommandFromComment({ core, context, github });
+          expect(outputFor(mockCoreSetOutput, 'command')).toBe('run-tests-extended');
+          expect(outputFor(mockCoreSetOutput, 'skipDeployment')).toBe('true');
+        });
+
         test(`should add comment with run link`, async () => {
           const context = createCommentContext({
             username: 'admin',
@@ -491,7 +536,7 @@ describe('getCommandFromComment', () => {
             owner: 'someOwner',
             repo: 'someRepo',
             issue_number: PR_NUMBER.FORK_NON_DOCS_CHANGES,
-            bodyMatcher: /The specified SHA `00000000` is not the latest commit on the PR. Please validate the latest commit and re-run `\/test`/,
+            bodyMatcher: /The specified SHA `00000000` is not the latest commit on the PR. Please validate the latest commit and re-run `\/test-extended`/,
           });
         });
       })

.github/workflows/deploy_tre_reusable.yml

@@ -45,6 +45,11 @@ on: # yamllint disable-line rule:truthy
         type: string
         default: ""
         required: false
+      skipDeployment:
+        description: Skip deployment jobs and run only the requested E2E tests against an existing environment
+        type: boolean
+        default: false
+        required: false
     secrets:
       AAD_TENANT_ID:
         description: ""
@@ -120,6 +125,7 @@ concurrency: "deploy-${{ inputs.ciGitRef }}"
 jobs:
   deploy_management:
     name: Deploy Management
+    if: ${{ !inputs.skipDeployment }}
     runs-on: ubuntu-latest
     permissions:
       id-token: write
@@ -832,6 +838,7 @@ jobs:
 
   e2e_tests_smoke:
     name: "Run E2E Tests (Smoke)"
+    if: ${{ always() && (inputs.skipDeployment || (!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled'))) }}
     runs-on: ubuntu-latest
     environment: ${{ inputs.environmentName }}
     needs: [deploy_shared_services, register_bundles, deploy_ui]
@@ -876,7 +883,7 @@ jobs:
 
   e2e_tests_custom:
     name: "Run E2E Tests"
-    if: ${{ inputs.e2eTestsCustomSelector != '' }}
+    if: ${{ always() && inputs.e2eTestsCustomSelector != '' && (inputs.skipDeployment || (!contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled'))) }}
     runs-on: ubuntu-latest
     environment: ${{ inputs.environmentName }}
     needs:

.github/workflows/pr_comment_bot.yml

@@ -30,8 +30,9 @@ jobs:
       prRef: ${{ steps.check_command.outputs.prRef }}
       prHeadSha: ${{ steps.check_command.outputs.prHeadSha }}
       prRefId: ${{ steps.check_command.outputs.prRefId }}
-      branchRefId: ${{ steps.check_command.outputs.branchRefid }}
+      branchRefId: ${{ steps.check_command.outputs.branchRefId }}
       ciGitRef: ${{ steps.check_command.outputs.ciGitRef }}
+      skipDeployment: ${{ steps.check_command.outputs.skipDeployment }}
     steps:
       # Ensure we have the script file for the github-script action to use
       - name: Checkout
@@ -56,7 +57,8 @@ jobs:
           echo "prHeadSha   : ${{ steps.check_command.outputs.prHeadSha }}"
           echo "prRefId     : ${{ steps.check_command.outputs.prRefId }}"
           echo "ciGitRef    : ${{ steps.check_command.outputs.ciGitRef }}"
-          echo "branchRefId : ${{ steps.check_command.outputs.prRefId }}"
+          echo "branchRefId : ${{ steps.check_command.outputs.branchRefId }}"
+          echo "skipDeploy  : ${{ steps.check_command.outputs.skipDeployment }}"
 
       # If we don't run the actual deploy (see the run_test job below) we won't receive a check-run status,
       # and will have to send it "manually"
@@ -178,6 +180,7 @@ jobs:
       environmentName: CICD
       E2E_TESTS_NUMBER_PROCESSES: 1
       DEVCONTAINER_TAG: ${{ needs.pr_comment.outputs.prRefId }}
+      skipDeployment: ${{ needs.pr_comment.outputs.skipDeployment == 'true' }}
     secrets:
       AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }}
       AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}

docs/tre-developers/github-pr-bot-commands.md

@@ -12,7 +12,7 @@ These commands can only be run when commented by a user who is identified as a r
 
 This command will cause the pr-comment-bot to respond with a comment listing the available commands.
 
-### `/test [<sha>]`
+### `/test [<sha>] [skip_deployment]`
 
 This command runs the build, deploy, and smoke tests for a PR.
 
@@ -21,6 +21,8 @@ For PRs from maintainers (i.e. users with write access to microsoft/AzureTRE), `
 For other PRs, the checks below should be carried out. Once satisfied that the PR is safe to run tests against, you should use `/test <sha>` where `<sha>` is the SHA for the commit that you have verified.
 You can use the full or short form of the SHA, but it must be at least 7 characters (GitHub UI shows 7 characters).
 
+If the PR validation environment is already deployed, add `skip_deployment` to skip the build and deployment jobs and run the smoke tests against the existing environment. For PRs from forks, include both the SHA and `skip_deployment`, for example `/test <sha> skip_deployment`.
+
 **IMPORTANT**
 
 This command works on PRs from forks, and makes the deployment secrets available.
@@ -32,7 +34,7 @@ Check for changes to anything that is run during the build/deploy/test cycle, in
 - modifications to scripts
 - new python packages being installed
 
-### `/test-extended [<sha>]` / `/test-extended-aad [<sha>]`/ `/test-shared-services [<sha>]`
+### `/test-extended [<sha>] [skip_deployment]` / `/test-extended-aad [<sha>]`/ `/test-shared-services [<sha>]`
 
 This command runs the build, deploy, and smoke & extended / shared services tests for a PR.
 
@@ -43,6 +45,8 @@ If a change has been made which would affect any of the core shared services, ma
 For other PRs, the checks below should be carried out. Once satisfied that the PR is safe to run tests against, you should use `/test-extended <sha>` where `<sha>` is the SHA for the commit that you have verified.
 You can use the full or short form of the SHA, but it must be at least 7 characters (GitHub UI shows 7 characters).
 
+If the PR validation environment is already deployed, add `skip_deployment` to `/test-extended` to skip the build and deployment jobs and run the smoke and extended tests against the existing environment. For PRs from forks, include both the SHA and `skip_deployment`, for example `/test-extended <sha> skip_deployment`.
+
 **IMPORTANT**
 
 As with `/test`, this command works on PRs from forks, and makes the deployment secrets available.