Merge pull request #432 from NHSDigital/apm-3869-fix-ecs-clean-up-pipeline

APM-3869 Fix ECS clean up pipeline again

Author: ed-bright

azure/cleanup-ecs-pr-proxies.yml

@@ -4,7 +4,7 @@ trigger: none
 pr: none
 
 schedules:
-  - cron: "0 2 * * *"
+  - cron: "0 */4 * * *" # Every 4 hours
     displayName: Daily PR cleanup
     branches:
       include:
@@ -47,6 +47,13 @@ jobs:
       - bash: |
           tfenv use 0.14.6
         displayName: setup terraform
+      
+      - task: s3-cache-action@1
+        inputs:
+          key: poetry | utils | poetry.lock
+          location: ".venv"
+          debug: true
+        displayName: cache utils pre-requisites 
 
       - bash: |
           make install
@@ -61,5 +68,5 @@ jobs:
           echo "AWS role session has timed out after multiple retries"
           exit -1
         displayName: Trigger failure if role has timed out
-        condition: eq(variables['has_aws_role_timedout'], 'true')
+        condition: eq(variables['should_retry'], 'true')
 

azure/components/cleanup-ecs-pr-proxies-job.yml

@@ -11,20 +11,17 @@ steps:
       - bash: |
           make remove-stale-locks
           export retain_hours=72
-          ANSIBLE_FORCE_COLOR=yes make -C ansible remove-old-ecs-pr-deploys | tee /tmp/output.txt
+          ANSIBLE_FORCE_COLOR=yes make -C ansible remove-old-ecs-pr-deploys
           ERROR_CODE=$?
-          ROLE_TIMEOUT_MSG="The AWS assume role session token is due to expire"
-          if grep -q "$ROLE_TIMEOUT_MSG" /tmp/output.txt ; then
-            echo "stderr for ansible has the error \"$ROLE_TIMEOUT_MSG\""
-            echo "Re-assuming role and re-running step"
-            echo "##vso[task.setvariable variable=has_aws_role_timedout;]true"
+          echo ERROR_CODE - $ERROR_CODE
+
+          if [ $ERROR_CODE -ne 0 ] ; then 
+            echo "\n\nansible has unhandled error, re-trying"
+            echo "##vso[task.setvariable variable=should_retry;]true"
 
-          elif [ $ERROR_CODE -ne 0 ] ; then 
-            echo "\n\nansible has unhandled error, re-raising"
-            exit -1
           else
-            echo "##vso[task.setvariable variable=has_aws_role_timedout;]false"
+            echo "##vso[task.setvariable variable=should_retry;]false"
           fi
 
         displayName: "cleanup older pr deploys"
-        condition: or(eq( ${{ parameters.retry }}, '0'), eq(variables['has_aws_role_timedout'], 'true'))
+        condition: or(eq( ${{ parameters.retry }}, '0'), eq(variables['should_retry'], 'true'))