From 5eff611ff1a424fd718138e346d6707a66a7269c Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Wed, 8 Feb 2023 08:45:53 -0800
Subject: [PATCH 01/27] Add aws-google-auth to requirements.txt

---
 requirements.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/requirements.txt b/requirements.txt
index 69b801796..ef38bfab1 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,6 +1,7 @@
 astroid==2.8.4
 autoflake==1.4
 autopep8==1.6.0
+aws-google-auth
 boto3==1.19.10
 botocore==1.22.10
 certifi==2021.10.8

From d9767f38f4680cc9c89c97ed6754e3162835a0d2 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Wed, 8 Feb 2023 08:46:09 -0800
Subject: [PATCH 02/27] Comment out buggy code until I find a fix

---
 shared/nodes.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/shared/nodes.py b/shared/nodes.py
index 8f16e629c..684ee17fa 100644
--- a/shared/nodes.py
+++ b/shared/nodes.py
@@ -811,12 +811,12 @@ def __init__(self, parent, json_blob):
         for vpc in parent.children:
             if vpc.local_id == json_blob["VpcId"]:
                 self._parent = vpc
-        if self._parent is None:
-            raise Exception(
-                "Could not find parent for Redshift node, was looking for VPC {}".format(
-                    json_blob["VpcId"]
-                )
-            )
+        # if self._parent is None:
+        #     raise Exception(
+        #         "Could not find parent for Redshift node, was looking for VPC {}".format(
+        #             json_blob["VpcId"]
+        #         )
+        #     )
 
         self._local_id = json_blob["ClusterIdentifier"]
         self._arn = json_blob["Endpoint"]["Address"]

From 1c8494d2fc70f3132f2161e441c53c3c67320195 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Wed, 15 Feb 2023 10:49:56 -0800
Subject: [PATCH 03/27] Create entrypoint script for daily Fargate task

---
 Dockerfile | 3 ++-
 entrypoint | 8 ++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)
 create mode 100755 entrypoint

diff --git a/Dockerfile b/Dockerfile
index 25fd522cd..f43d8fc0c 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -14,4 +14,5 @@ RUN apt-get install -y bash
 COPY . /opt/cloudmapper
 RUN pip install -r requirements.txt
 
-RUN bash
+#RUN bash
+ENTRYPOINT "./entrypoint"
diff --git a/entrypoint b/entrypoint
new file mode 100755
index 000000000..0649c7397
--- /dev/null
+++ b/entrypoint
@@ -0,0 +1,8 @@
+#!/bin/bash
+
+export AWS_REGION="us-west-2"
+aws configure --profile prod --region us-west-2 --output json
+
+python cloudmapper.py collect --account prod --regions us-west-2 --clean
+python cloudmapper.py stats --accounts prod --stats_all_resources --no_output_image > prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
+aws s3 cp prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER

From 9e877be908553ea94b3a2984a252499628bf074e Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Wed, 15 Feb 2023 10:50:19 -0800
Subject: [PATCH 04/27] Comment out code that forces AWS_REGION to be us-east-1

---
 commands/collect.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/commands/collect.py b/commands/collect.py
index 073914f4e..321c6862d 100644
--- a/commands/collect.py
+++ b/commands/collect.py
@@ -228,8 +228,8 @@ def collect(arguments):
         default_region = "us-gov-west-1"
     elif "cn-" in default_region:
         default_region = "cn-north-1"
-    else:
-        default_region = "us-east-1"
+    #else:
+    #    default_region = "us-east-1"
 
     regions_filter = None
     if len(arguments.regions_filter) > 0:

From f71f8598f7da86ae89789e5363b68b1d779d2451 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Wed, 22 Feb 2023 10:29:34 -0800
Subject: [PATCH 05/27] Working image / entrypoint to scan prod account

---
 Dockerfile       | 24 +++++++++++++++++++++++-
 entrypoint       | 26 +++++++++++++++++++++++---
 requirements.txt |  2 +-
 3 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index f43d8fc0c..47b1b11c3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,7 @@
-FROM python:3.7-slim as cloudmapper
+#FROM python:3.7-slim as cloudmapper
+#FROM alpine:3.17.2
+#FROM python:3.7-alpine3.17
+FROM python:3.9-slim as cloudmapper
 
 LABEL maintainer="https://github.com/0xdabbad00/"
 LABEL Project="https://github.com/duo-labs/cloudmapper"
@@ -10,6 +13,25 @@ ENV AWS_DEFAULT_REGION=us-east-1
 RUN apt-get update -y
 RUN apt-get install -y build-essential autoconf automake libtool python3-tk jq awscli
 RUN apt-get install -y bash
+# RUN apk update && \
+#     apk upgrade && \
+#     apk add --virtual build-dependencies \
+#     build-base \
+#     gcc \
+#     wget \
+#     git 
+# RUN apk add --no-cache \
+#     alpine-sdk \
+#     autoconf \
+#     automake \
+#     bash \
+#     jq \
+#     libtool \
+#     python3-tkinter \
+#     python3-dev \
+#     py-pip && \
+#     pip install --upgrade pip && \
+#     pip install awscli
 
 COPY . /opt/cloudmapper
 RUN pip install -r requirements.txt
diff --git a/entrypoint b/entrypoint
index 0649c7397..496c58aa2 100755
--- a/entrypoint
+++ b/entrypoint
@@ -1,8 +1,28 @@
 #!/bin/bash
 
+# Create the entire file via echo or create it beforehand?
 export AWS_REGION="us-west-2"
-aws configure --profile prod --region us-west-2 --output json
+export AWS_CONFIG_FILE=aws-config
+echo "[profile prod]" >> aws-config
+echo "region=us-west-2" >> caws-onfig
+echo "output=json" >> aws-config
+echo "credential_source=EcsContainer" >> aws-config
+echo "role_arn=$CLOUDMAPPER_PROD_ROLE_ARN" >> aws-config
+# aws configure \
+#     --profile sre-ops \
+#     --region us-west-2 \
+#     --output json
 
-python cloudmapper.py collect --account prod --regions us-west-2 --clean
-python cloudmapper.py stats --accounts prod --stats_all_resources --no_output_image > prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
+cat aws-config
+
+python cloudmapper.py collect \
+    --account prod \
+    --profile prod \
+    --regions us-west-2 \
+    --clean
+python cloudmapper.py stats \
+    --accounts prod \
+    --stats_all_resources \
+    --no_output_image \
+    > prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
 aws s3 cp prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER
diff --git a/requirements.txt b/requirements.txt
index ef38bfab1..308e7fd8a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,7 @@
 astroid==2.8.4
 autoflake==1.4
 autopep8==1.6.0
-aws-google-auth
+# aws-google-auth
 boto3==1.19.10
 botocore==1.22.10
 certifi==2021.10.8

From 30423c218ee769e00a7847b2bcdb7ca8c3c1a280 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Thu, 23 Feb 2023 10:49:05 -0800
Subject: [PATCH 06/27] Entrypoint overhaul

Create aws config file via echo (may redo later)

Configure Cloudmapper's config.json via commands

Run Cloudmapper on dev and solutions-external accounts in addition to prod

Make commands stay on one line where it makes sense
---
 entrypoint | 52 ++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 36 insertions(+), 16 deletions(-)

diff --git a/entrypoint b/entrypoint
index 496c58aa2..0ff928912 100755
--- a/entrypoint
+++ b/entrypoint
@@ -8,21 +8,41 @@ echo "region=us-west-2" >> caws-onfig
 echo "output=json" >> aws-config
 echo "credential_source=EcsContainer" >> aws-config
 echo "role_arn=$CLOUDMAPPER_PROD_ROLE_ARN" >> aws-config
-# aws configure \
-#     --profile sre-ops \
-#     --region us-west-2 \
-#     --output json
-
-cat aws-config
-
-python cloudmapper.py collect \
-    --account prod \
-    --profile prod \
-    --regions us-west-2 \
-    --clean
-python cloudmapper.py stats \
-    --accounts prod \
-    --stats_all_resources \
-    --no_output_image \
+echo "" >> aws-config
+
+echo "[profile dev]" >> aws-config
+echo "region=us-west-2" >> caws-onfig
+echo "output=json" >> aws-config
+echo "credential_source=EcsContainer" >> aws-config
+echo "role_arn=$CLOUDMAPPER_DEV_ROLE_ARN" >> aws-config
+echo "" >> aws-config
+
+echo "[profile solutions-external]" >> aws-config
+echo "region=us-west-2" >> caws-onfig
+echo "output=json" >> aws-config
+echo "credential_source=EcsContainer" >> aws-config
+echo "role_arn=$CLOUDMAPPER_SOL_EXT_ROLE_ARN" >> aws-config
+echo "" >> aws-config
+
+# Configure config.json for CloudMapper
+python cloudmapper.py configure add-account --config-file config.json --name prod --id $PROD_ID
+python cloudmapper.py configure add-account --config-file config.json --name dev --id $DEV_ID
+python cloudmapper.py configure add-account --config-file config.json --name solutions-external --id $SOL_EXT_ID
+
+# Run CloudMapper
+python cloudmapper.py collect --account prod --profile prod --regions us-west-2 --clean
+python cloudmapper.py stats --accounts prod --stats_all_resources --no_output_image \
     > prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
+
+python cloudmapper.py collect --account dev --profile dev --regions us-west-2 --clean
+python cloudmapper.py stats --accounts dev --stats_all_resources --no_output_image \
+    > dev-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
+
+python cloudmapper.py collect --account solutions-external --profile solutions-external --regions us-west-2 --clean
+python cloudmapper.py stats --accounts solutions-external --stats_all_resources --no_output_image \
+    > solutions-external-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt
+
+# Output stats results to S3
 aws s3 cp prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER
+aws s3 cp dev-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER
+aws s3 cp solutions-external-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER
\ No newline at end of file

From 13c89b3ae300bdcf18f323ffa6aadef2f6ebff2b Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Wed, 5 Apr 2023 10:17:37 -0700
Subject: [PATCH 07/27] Allow SQS list_queues pagination via MaxResults param

Avoid error that occurs when a parameter is an int
---
 collect_commands.yaml | 3 +++
 commands/collect.py   | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/collect_commands.yaml b/collect_commands.yaml
index d3b150c74..9ffb2d372 100644
--- a/collect_commands.yaml
+++ b/collect_commands.yaml
@@ -175,6 +175,9 @@
   Request: describe-cluster-subnet-groups
 - Service: sqs
   Request: list-queues
+  Parameters:
+  - Name: MaxResults
+    Value: 1000
 - Service: sqs
   Request: get-queue-attributes
   Parameters:
diff --git a/commands/collect.py b/commands/collect.py
index 321c6862d..03710cd27 100644
--- a/commands/collect.py
+++ b/commands/collect.py
@@ -364,7 +364,7 @@ def collect(arguments):
                     parameters[parameter["Name"]] = parameter["Value"]
 
                     # Look for any dynamic values (ones that jq parse a file)
-                    if "|" in parameter["Value"]:
+                    if not isinstance(parameter["Value"], int) and "|" in parameter["Value"]:
                         dynamic_parameter = parameter["Name"]
 
             if runner.get("Custom_collection", False):

From 244c87b4fdb94538fa34c776e7119d238e7c4f24 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Thu, 6 Apr 2023 10:52:25 -0700
Subject: [PATCH 08/27] Add optional profile flag to parse_arguments method

---
 shared/common.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/shared/common.py b/shared/common.py
index f10338361..e9797fbeb 100644
--- a/shared/common.py
+++ b/shared/common.py
@@ -229,6 +229,13 @@ def parse_arguments(arguments, parser=None):
         required=False,
         type=str,
     )
+    parser.add_argument(
+        "--profile",
+        help="AWS CLI profile to use",
+        default=None,
+        required=False,
+        type=str,
+    )
     args = parser.parse_args(arguments)
 
     global LOG_LEVEL

From e99af3fe9d628288ecb1c5eebf59c9d2f3e3a390 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Thu, 6 Apr 2023 10:53:38 -0700
Subject: [PATCH 09/27] Allow tracking of private_commands/

---
 .gitignore | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index ac50e35b2..5c4e1a7d8 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,7 +13,6 @@ tmp/
 trusted_ips.png
 resource_stats.png
 data/
-private_commands/
 output/
 .vscode/
 Pipfile.lock

From 639cc2695bf8421e9476d7d7014870ad05ad24ed Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Thu, 6 Apr 2023 10:54:17 -0700
Subject: [PATCH 10/27] WIP for sqs-determine-unused

Can fetch a metric for one queue, which it learns about by parsing the saved JSON files
---
 private_commands/sqs-determine-unused.py | 110 +++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 private_commands/sqs-determine-unused.py

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
new file mode 100644
index 000000000..1c0c55795
--- /dev/null
+++ b/private_commands/sqs-determine-unused.py
@@ -0,0 +1,110 @@
+import os
+import json
+import boto3
+import logging
+import argparse
+
+from shared.nodes import Account, Region
+from shared.common import query_aws, get_regions, parse_arguments, get_parameter_file
+from datetime import datetime, date, timedelta
+from shared.common import get_account, custom_serializer
+from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError
+
+def make_directory(path):
+    try:
+        os.mkdir(path)
+    except OSError:
+        # Already exists
+        pass
+
+def snakecase(s):
+    return s.replace("-", "_")
+
+def get_sqs_queue_metrics(arguments, accounts, config):
+    logging.getLogger("botocore").setLevel(logging.WARN)
+    outputfile = "account-data/prod/us-west-2/sqs-list-queues.json"
+
+    default_region = os.environ.get("AWS_REGION", "us-east-1")
+    # regions_filter = None
+    # if len(arguments.regions_filter) > 0:
+    #     regions_filter = arguments.regions_filter.lower().split(",")
+    #     # Force include of default region -- seems to be required
+    #     if default_region not in regions_filter:
+    #         regions_filter.append(default_region)
+
+    session_data = {"region_name": default_region}
+
+    if arguments.profile:
+        session_data["profile_name"] = arguments.profile
+
+    session = boto3.Session(**session_data)
+
+    sts = session.client("sts")
+    try:
+        sts.get_caller_identity()
+        logging.debug("Using AWS account: {}".format(sts.get_caller_identity()["Account"]))
+    except ClientError as e:
+        if "InvalidClientTokenId" in str(e):
+            print(
+                "ERROR: sts.get_caller_identity failed with InvalidClientTokenId. Likely cause is no AWS credentials are set.",
+                flush=True,
+            )
+            exit(-1)
+        else:
+            print(
+                "ERROR: Unknown exception when trying to call sts.get_caller_identity: {}".format(
+                    e
+                ),
+                flush=True,
+            )
+            exit(-1)
+
+    client = session.client('cloudwatch')
+    parameters = {
+        'MetricDataQueries': [{
+            'Id': 'id_0',
+            'MetricStat': {
+                'Metric': {
+                    'Namespace': 'AWS/SQS',
+                    'MetricName': 'NumberOfMessagesReceived',
+                    'Dimensions': [
+                        {
+                            'Name': 'QueueName',
+                            #'Value': 'hsl-activities-migration'
+                        },
+                    ],
+                },
+                'Period': 86400,
+                'Stat': 'Sum'
+            },
+            'ReturnData': True,
+        }],
+        'StartTime': datetime.utcnow() - timedelta(weeks=4),
+        'EndTime': datetime.utcnow(),
+    }
+
+    #response = client.get_metric_data(**parameters)
+    #print('Metric Total: ', sum(response['MetricDataResults'][0]['Values']))
+
+    for account in accounts:
+        for region_json in get_regions(Account(None, account)):
+            region = Region(Account(None, account), region_json)
+            print(f"Region: {region.name}")
+            saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region)
+            #print(saved_sqs_list.keys())
+            if 'QueueUrls' in saved_sqs_list:
+                print(saved_sqs_list['QueueUrls'][0])
+
+                saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", saved_sqs_list['QueueUrls'][0])
+                print(saved_sqs_details)
+                queue_name = saved_sqs_details['Attributes']['QueueArn'].split(':')[-1]
+                print(f"Queue Name: {queue_name}")
+
+                parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
+                response = client.get_metric_data(**parameters)
+                print('Metric Total: ', sum(response['MetricDataResults'][0]['Values']))
+
+def run(arguments):
+
+    args, accounts, config = parse_arguments(arguments)
+    get_sqs_queue_metrics(args, accounts, config)

From 39220b3a0c30a48cc0a883ff6ace4cb7ce7ac5c7 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Fri, 7 Apr 2023 10:09:06 -0700
Subject: [PATCH 11/27] sqs-determine-unused now fetches metrics for one queue
 and saves the results back to disk

---
 private_commands/sqs-determine-unused.py | 39 +++++++++++++++++++-----
 1 file changed, 32 insertions(+), 7 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index 1c0c55795..acbeae5eb 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -8,6 +8,7 @@
 from shared.common import query_aws, get_regions, parse_arguments, get_parameter_file
 from datetime import datetime, date, timedelta
 from shared.common import get_account, custom_serializer
+from commands.collect import get_filename_from_parameter
 from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError
 
 def make_directory(path):
@@ -86,23 +87,47 @@ def get_sqs_queue_metrics(arguments, accounts, config):
     #response = client.get_metric_data(**parameters)
     #print('Metric Total: ', sum(response['MetricDataResults'][0]['Values']))
 
+    # parameter file is loaded into saved_sqs_list
+    sqs_dynamic_param = "QueueUrl"
+
     for account in accounts:
         for region_json in get_regions(Account(None, account)):
             region = Region(Account(None, account), region_json)
             print(f"Region: {region.name}")
             saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region)
-            #print(saved_sqs_list.keys())
-            if 'QueueUrls' in saved_sqs_list:
-                print(saved_sqs_list['QueueUrls'][0])
+            sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
+                arguments.profile, region.name, "sqs", "get-queue-attributes"
+            )
 
-                saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", saved_sqs_list['QueueUrls'][0])
+            if 'QueueUrls' in saved_sqs_list:
+                #print(saved_sqs_list['QueueUrls'][0])
+                queue_url = saved_sqs_list['QueueUrls'][0]
+                saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", queue_url)
                 print(saved_sqs_details)
+
+                queue_arn = saved_sqs_details['Attributes']['QueueArn']
                 queue_name = saved_sqs_details['Attributes']['QueueArn'].split(':')[-1]
                 print(f"Queue Name: {queue_name}")
 
-                parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
-                response = client.get_metric_data(**parameters)
-                print('Metric Total: ', sum(response['MetricDataResults'][0]['Values']))
+                if "MessagesReceivedInLastMonth" not in saved_sqs_details['Attributes']:
+                    parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
+                    response = client.get_metric_data(**parameters)
+                    messages_received = sum(response['MetricDataResults'][0]['Values'])
+                    print('Messages Received in Last Month: ', messages_received)
+                    saved_sqs_details['Attributes']['MessagesReceivedInLastMonth'] = messages_received
+
+                    # This is the filename we want to save the data to
+                    #print(get_filename_from_parameter(queue_url))
+                    filename = get_filename_from_parameter(queue_url)
+                    outputfile = "{}/{}".format(sqs_attributes_filepath, filename)
+                    print(f"Output File: {outputfile}")
+                    if saved_sqs_details is not None:
+                        with open(outputfile, "w+") as f:
+                            f.write(
+                                json.dumps(saved_sqs_details, indent=4, sort_keys=True, default=custom_serializer)
+                            )
+
+
 
 def run(arguments):
 

From d757f2ae4c446c6d57762c4c0bad5329486df3d7 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Fri, 7 Apr 2023 10:33:37 -0700
Subject: [PATCH 12/27] Clean up code, overwrite attributes file every time,
 fetch queue tags

---
 private_commands/sqs-determine-unused.py | 67 +++++++++++-------------
 1 file changed, 30 insertions(+), 37 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index acbeae5eb..b21d385da 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -1,15 +1,14 @@
-import os
-import json
+import argparse
 import boto3
+import json
 import logging
-import argparse
+import os
 
-from shared.nodes import Account, Region
-from shared.common import query_aws, get_regions, parse_arguments, get_parameter_file
-from datetime import datetime, date, timedelta
-from shared.common import get_account, custom_serializer
-from commands.collect import get_filename_from_parameter
 from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError
+from commands.collect import get_filename_from_parameter
+from datetime import datetime, date, timedelta
+from shared.common import query_aws, get_regions, parse_arguments, get_parameter_file, custom_serializer
+from shared.nodes import Account, Region
 
 def make_directory(path):
     try:
@@ -23,7 +22,7 @@ def snakecase(s):
 
 def get_sqs_queue_metrics(arguments, accounts, config):
     logging.getLogger("botocore").setLevel(logging.WARN)
-    outputfile = "account-data/prod/us-west-2/sqs-list-queues.json"
+    sqs_queue_list_file = "account-data/prod/us-west-2/sqs-list-queues.json"
 
     default_region = os.environ.get("AWS_REGION", "us-east-1")
     # regions_filter = None
@@ -60,6 +59,7 @@ def get_sqs_queue_metrics(arguments, accounts, config):
             )
             exit(-1)
 
+    sqs = session.client('sqs')
     client = session.client('cloudwatch')
     parameters = {
         'MetricDataQueries': [{
@@ -70,8 +70,7 @@ def get_sqs_queue_metrics(arguments, accounts, config):
                     'MetricName': 'NumberOfMessagesReceived',
                     'Dimensions': [
                         {
-                            'Name': 'QueueName',
-                            #'Value': 'hsl-activities-migration'
+                            'Name': 'QueueName'
                         },
                     ],
                 },
@@ -84,48 +83,42 @@ def get_sqs_queue_metrics(arguments, accounts, config):
         'EndTime': datetime.utcnow(),
     }
 
-    #response = client.get_metric_data(**parameters)
-    #print('Metric Total: ', sum(response['MetricDataResults'][0]['Values']))
-
-    # parameter file is loaded into saved_sqs_list
-    sqs_dynamic_param = "QueueUrl"
-
     for account in accounts:
         for region_json in get_regions(Account(None, account)):
             region = Region(Account(None, account), region_json)
             print(f"Region: {region.name}")
+
             saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region)
             sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
                 arguments.profile, region.name, "sqs", "get-queue-attributes"
             )
 
             if 'QueueUrls' in saved_sqs_list:
-                #print(saved_sqs_list['QueueUrls'][0])
-                queue_url = saved_sqs_list['QueueUrls'][0]
+                queue_url = saved_sqs_list['QueueUrls'][1]
                 saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", queue_url)
-                print(saved_sqs_details)
 
                 queue_arn = saved_sqs_details['Attributes']['QueueArn']
                 queue_name = saved_sqs_details['Attributes']['QueueArn'].split(':')[-1]
                 print(f"Queue Name: {queue_name}")
 
-                if "MessagesReceivedInLastMonth" not in saved_sqs_details['Attributes']:
-                    parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
-                    response = client.get_metric_data(**parameters)
-                    messages_received = sum(response['MetricDataResults'][0]['Values'])
-                    print('Messages Received in Last Month: ', messages_received)
-                    saved_sqs_details['Attributes']['MessagesReceivedInLastMonth'] = messages_received
-
-                    # This is the filename we want to save the data to
-                    #print(get_filename_from_parameter(queue_url))
-                    filename = get_filename_from_parameter(queue_url)
-                    outputfile = "{}/{}".format(sqs_attributes_filepath, filename)
-                    print(f"Output File: {outputfile}")
-                    if saved_sqs_details is not None:
-                        with open(outputfile, "w+") as f:
-                            f.write(
-                                json.dumps(saved_sqs_details, indent=4, sort_keys=True, default=custom_serializer)
-                            )
+                parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
+                response = client.get_metric_data(**parameters)
+                messages_received = sum(response['MetricDataResults'][0]['Values'])
+                print('Messages Received in Last Month: ', messages_received)
+                saved_sqs_details['Attributes']['MessagesReceivedInLastMonth'] = messages_received
+
+                queue_tags = sqs.list_queue_tags(QueueUrl=queue_url)
+                if 'Tags' in queue_tags:
+                    saved_sqs_details['Attributes']['Tags'] = queue_tags['Tags']
+
+                filename = get_filename_from_parameter(queue_url)
+                sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename)
+                print(f"Output File: {sqs_attributes_file}")
+                if saved_sqs_details is not None:
+                    with open(sqs_attributes_file, "w+") as f:
+                        f.write(
+                            json.dumps(saved_sqs_details, indent=4, sort_keys=True, default=custom_serializer)
+                        )
 
 
 

From 37d07843fe92a80292246491cf84d38c21a6484d Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Fri, 7 Apr 2023 11:01:02 -0700
Subject: [PATCH 13/27] Remove unused elements

---
 private_commands/sqs-determine-unused.py | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index b21d385da..cb6907193 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -1,12 +1,11 @@
-import argparse
 import boto3
 import json
 import logging
 import os
 
-from botocore.exceptions import ClientError, EndpointConnectionError, NoCredentialsError
+from botocore.exceptions import ClientError
 from commands.collect import get_filename_from_parameter
-from datetime import datetime, date, timedelta
+from datetime import datetime, timedelta
 from shared.common import query_aws, get_regions, parse_arguments, get_parameter_file, custom_serializer
 from shared.nodes import Account, Region
 
@@ -20,9 +19,8 @@ def make_directory(path):
 def snakecase(s):
     return s.replace("-", "_")
 
-def get_sqs_queue_metrics(arguments, accounts, config):
+def get_sqs_queue_metrics_and_tags(arguments, accounts, config):
     logging.getLogger("botocore").setLevel(logging.WARN)
-    sqs_queue_list_file = "account-data/prod/us-west-2/sqs-list-queues.json"
 
     default_region = os.environ.get("AWS_REGION", "us-east-1")
     # regions_filter = None
@@ -84,6 +82,7 @@ def get_sqs_queue_metrics(arguments, accounts, config):
     }
 
     for account in accounts:
+        # get_regions reads the file at account-data/{profile}/describe-regions.json
         for region_json in get_regions(Account(None, account)):
             region = Region(Account(None, account), region_json)
             print(f"Region: {region.name}")
@@ -97,7 +96,6 @@ def get_sqs_queue_metrics(arguments, accounts, config):
                 queue_url = saved_sqs_list['QueueUrls'][1]
                 saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", queue_url)
 
-                queue_arn = saved_sqs_details['Attributes']['QueueArn']
                 queue_name = saved_sqs_details['Attributes']['QueueArn'].split(':')[-1]
                 print(f"Queue Name: {queue_name}")
 
@@ -125,4 +123,4 @@ def get_sqs_queue_metrics(arguments, accounts, config):
 def run(arguments):
 
     args, accounts, config = parse_arguments(arguments)
-    get_sqs_queue_metrics(args, accounts, config)
+    get_sqs_queue_metrics_and_tags(args, accounts, config)

From da6ec373dc16882c45d9cee569ec9f4b952ab473 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Fri, 7 Apr 2023 11:25:51 -0700
Subject: [PATCH 14/27] Refactor body of code into smaller functions

Invert QueueUrls check so it looks nicer / don't have to tab
---
 private_commands/sqs-determine-unused.py | 85 ++++++++++++++++--------
 1 file changed, 58 insertions(+), 27 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index cb6907193..99a73ce65 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -16,9 +16,55 @@ def make_directory(path):
         # Already exists
         pass
 
+
 def snakecase(s):
     return s.replace("-", "_")
 
+
+def fetch_queue_metrics(client, request_parameters, queue_name, queue_attributes_obj):
+    request_parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
+    response = client.get_metric_data(**request_parameters)
+    queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth'] = sum(response['MetricDataResults'][0]['Values'])
+
+    return queue_attributes_obj
+
+
+def fetch_queue_tags(client, queue_url, queue_attributes_obj):
+    queue_tags = client.list_queue_tags(QueueUrl=queue_url)
+    if 'Tags' in queue_tags:
+        queue_attributes_obj['Attributes']['Tags'] = queue_tags['Tags']
+
+    return queue_attributes_obj
+
+
+def set_unused_flag(queue_attributes_obj):
+    creation_time = int(queue_attributes_obj['Attributes']['CreatedTimestamp'])
+    created_recently = datetime.utcnow() - timedelta(weeks=4) < datetime.fromtimestamp(creation_time)
+
+    if int(queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth']) == 0 and not created_recently:
+        queue_attributes_obj['Attributes']['Unused'] = True
+    else:
+        queue_attributes_obj['Attributes']['Unused'] = False
+
+    return queue_attributes_obj
+
+
+def save_sqs_attributes_file(profile_name, region_name, queue_url, queue_attributes_obj):
+    sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
+        profile_name, region_name, "sqs", "get-queue-attributes"
+    )
+    filename = get_filename_from_parameter(queue_url)
+    sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename)
+    print(f"Output File: {sqs_attributes_file}")
+    if queue_attributes_obj is not None:
+        with open(sqs_attributes_file, "w+") as f:
+            f.write(
+                json.dumps(queue_attributes_obj, indent=4, sort_keys=True, default=custom_serializer)
+            )
+
+    return
+
+
 def get_sqs_queue_metrics_and_tags(arguments, accounts, config):
     logging.getLogger("botocore").setLevel(logging.WARN)
 
@@ -59,7 +105,7 @@ def get_sqs_queue_metrics_and_tags(arguments, accounts, config):
 
     sqs = session.client('sqs')
     client = session.client('cloudwatch')
-    parameters = {
+    base_parameters = {
         'MetricDataQueries': [{
             'Id': 'id_0',
             'MetricStat': {
@@ -88,36 +134,21 @@ def get_sqs_queue_metrics_and_tags(arguments, accounts, config):
             print(f"Region: {region.name}")
 
             saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region)
-            sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
-                arguments.profile, region.name, "sqs", "get-queue-attributes"
-            )
-
-            if 'QueueUrls' in saved_sqs_list:
-                queue_url = saved_sqs_list['QueueUrls'][1]
-                saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", queue_url)
-
-                queue_name = saved_sqs_details['Attributes']['QueueArn'].split(':')[-1]
-                print(f"Queue Name: {queue_name}")
 
-                parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
-                response = client.get_metric_data(**parameters)
-                messages_received = sum(response['MetricDataResults'][0]['Values'])
-                print('Messages Received in Last Month: ', messages_received)
-                saved_sqs_details['Attributes']['MessagesReceivedInLastMonth'] = messages_received
+            if 'QueueUrls' not in saved_sqs_list:
+                continue
 
-                queue_tags = sqs.list_queue_tags(QueueUrl=queue_url)
-                if 'Tags' in queue_tags:
-                    saved_sqs_details['Attributes']['Tags'] = queue_tags['Tags']
+            queue_url = saved_sqs_list['QueueUrls'][1]
+            saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", queue_url)
+            queue_name = saved_sqs_details['Attributes']['QueueArn'].split(':')[-1]
+            #print(f"Queue Name: {queue_name}")
 
-                filename = get_filename_from_parameter(queue_url)
-                sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename)
-                print(f"Output File: {sqs_attributes_file}")
-                if saved_sqs_details is not None:
-                    with open(sqs_attributes_file, "w+") as f:
-                        f.write(
-                            json.dumps(saved_sqs_details, indent=4, sort_keys=True, default=custom_serializer)
-                        )
+            saved_sqs_details = fetch_queue_metrics(client, base_parameters, queue_name, saved_sqs_details)
+            saved_sqs_details = fetch_queue_tags(sqs, queue_url, saved_sqs_details)
+            saved_sqs_details = set_unused_flag(saved_sqs_details)
+            save_sqs_attributes_file(arguments.profile, region.name, queue_url, saved_sqs_details)
 
+    return
 
 
 def run(arguments):

From 2654357b6144fda6d4e005224190e39a1d6fe33f Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Fri, 7 Apr 2023 13:24:27 -0700
Subject: [PATCH 15/27] Add type hinting

---
 private_commands/sqs-determine-unused.py | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index 99a73ce65..4973e6de9 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -9,7 +9,7 @@
 from shared.common import query_aws, get_regions, parse_arguments, get_parameter_file, custom_serializer
 from shared.nodes import Account, Region
 
-def make_directory(path):
+def make_directory(path: str) -> None:
     try:
         os.mkdir(path)
     except OSError:
@@ -17,11 +17,11 @@ def make_directory(path):
         pass
 
 
-def snakecase(s):
+def snakecase(s: str) -> str:
     return s.replace("-", "_")
 
 
-def fetch_queue_metrics(client, request_parameters, queue_name, queue_attributes_obj):
+def fetch_queue_metrics(client: boto3.Session.client, request_parameters: dict, queue_name: str, queue_attributes_obj: dict) -> dict:
     request_parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
     response = client.get_metric_data(**request_parameters)
     queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth'] = sum(response['MetricDataResults'][0]['Values'])
@@ -29,7 +29,7 @@ def fetch_queue_metrics(client, request_parameters, queue_name, queue_attributes
     return queue_attributes_obj
 
 
-def fetch_queue_tags(client, queue_url, queue_attributes_obj):
+def fetch_queue_tags(client: boto3.Session.client, queue_url: str, queue_attributes_obj: dict) -> dict:
     queue_tags = client.list_queue_tags(QueueUrl=queue_url)
     if 'Tags' in queue_tags:
         queue_attributes_obj['Attributes']['Tags'] = queue_tags['Tags']
@@ -37,7 +37,7 @@ def fetch_queue_tags(client, queue_url, queue_attributes_obj):
     return queue_attributes_obj
 
 
-def set_unused_flag(queue_attributes_obj):
+def set_unused_flag(queue_attributes_obj: dict) -> dict:
     creation_time = int(queue_attributes_obj['Attributes']['CreatedTimestamp'])
     created_recently = datetime.utcnow() - timedelta(weeks=4) < datetime.fromtimestamp(creation_time)
 
@@ -49,7 +49,7 @@ def set_unused_flag(queue_attributes_obj):
     return queue_attributes_obj
 
 
-def save_sqs_attributes_file(profile_name, region_name, queue_url, queue_attributes_obj):
+def save_sqs_attributes_file(profile_name: str, region_name: str, queue_url: str, queue_attributes_obj: dict) -> None:
     sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
         profile_name, region_name, "sqs", "get-queue-attributes"
     )
@@ -65,7 +65,7 @@ def save_sqs_attributes_file(profile_name, region_name, queue_url, queue_attribu
     return
 
 
-def get_sqs_queue_metrics_and_tags(arguments, accounts, config):
+def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict) -> None:
     logging.getLogger("botocore").setLevel(logging.WARN)
 
     default_region = os.environ.get("AWS_REGION", "us-east-1")
@@ -138,7 +138,8 @@ def get_sqs_queue_metrics_and_tags(arguments, accounts, config):
             if 'QueueUrls' not in saved_sqs_list:
                 continue
 
-            queue_url = saved_sqs_list['QueueUrls'][1]
+            print(f"Queue Count: {len(saved_sqs_list['QueueUrls'])}")
+            queue_url = saved_sqs_list['QueueUrls'][2]
             saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", queue_url)
             queue_name = saved_sqs_details['Attributes']['QueueArn'].split(':')[-1]
             #print(f"Queue Name: {queue_name}")
@@ -151,7 +152,7 @@ def get_sqs_queue_metrics_and_tags(arguments, accounts, config):
     return
 
 
-def run(arguments):
+def run(arguments: list) -> None:
 
     args, accounts, config = parse_arguments(arguments)
     get_sqs_queue_metrics_and_tags(args, accounts, config)

From 2501b77b4c5fc4d3a052e882016c19afabab5c2c Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 10 Apr 2023 10:51:18 -0700
Subject: [PATCH 16/27] Make all methods loop

This successfully parsed all queues
---
 private_commands/sqs-determine-unused.py | 135 +++++++++++++++--------
 1 file changed, 88 insertions(+), 47 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index 4973e6de9..b2aeb1c01 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -2,6 +2,8 @@
 import json
 import logging
 import os
+import re
+import copy
 
 from botocore.exceptions import ClientError
 from commands.collect import get_filename_from_parameter
@@ -21,46 +23,78 @@ def snakecase(s: str) -> str:
     return s.replace("-", "_")
 
 
-def fetch_queue_metrics(client: boto3.Session.client, request_parameters: dict, queue_name: str, queue_attributes_obj: dict) -> dict:
-    request_parameters['MetricDataQueries'][0]['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
-    response = client.get_metric_data(**request_parameters)
-    queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth'] = sum(response['MetricDataResults'][0]['Values'])
+def fetch_queue_metrics(client: boto3.Session.client, base_parameters: dict, inner_base_parameters: dict, queue_names: list, queue_attributes_objs: list) -> list:
+    print(f'Fetching metrics for {len(queue_names)} queues')
+    if len(base_parameters['MetricDataQueries']) > 0:
+        base_parameters['MetricDataQueries'] = []
+    full_params = base_parameters.copy()
 
-    return queue_attributes_obj
+    for i, queue_name in enumerate(queue_names):
+        single_sub_params = inner_base_parameters.copy()
+        single_sub_params['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
+        single_sub_params['Id'] = f'queue{str(i)}'
+        full_params['MetricDataQueries'].append(single_sub_params)
 
+    response = client.get_metric_data(**full_params)
+    metric_results = [(int(re.sub('[a-z]', '', x['Id'])), sum(x['Values'])) for x in response['MetricDataResults']]
 
-def fetch_queue_tags(client: boto3.Session.client, queue_url: str, queue_attributes_obj: dict) -> dict:
-    queue_tags = client.list_queue_tags(QueueUrl=queue_url)
-    if 'Tags' in queue_tags:
-        queue_attributes_obj['Attributes']['Tags'] = queue_tags['Tags']
+    # paranoid about things being out of order, so sort by the id
+    metric_results.sort(key = lambda x: x[0])
+    metric_results = [x[1] for x in metric_results]
 
-    return queue_attributes_obj
+    if len(metric_results) != len(queue_names) or len(metric_results) != len(queue_attributes_objs):
+        print("Error: Number of metric results don't match queue names or attributes")
+        exit(-1)
 
+    for i, queue_attributes_obj in enumerate(queue_attributes_objs):
+        queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth'] = metric_results[i]
 
-def set_unused_flag(queue_attributes_obj: dict) -> dict:
-    creation_time = int(queue_attributes_obj['Attributes']['CreatedTimestamp'])
-    created_recently = datetime.utcnow() - timedelta(weeks=4) < datetime.fromtimestamp(creation_time)
+    return queue_attributes_objs
 
-    if int(queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth']) == 0 and not created_recently:
-        queue_attributes_obj['Attributes']['Unused'] = True
-    else:
-        queue_attributes_obj['Attributes']['Unused'] = False
 
-    return queue_attributes_obj
+def fetch_queue_tags(client: boto3.Session.client, queue_urls: list, queue_attributes_objs: list) -> list:
+    print('Fetching tags')
+    for i in range(len(queue_urls)):
+        queue_tags = client.list_queue_tags(QueueUrl=queue_urls[i])
+        if 'Tags' in queue_tags:
+            queue_attributes_objs[i]['Attributes']['Tags'] = queue_tags['Tags']
 
+    return queue_attributes_objs
 
-def save_sqs_attributes_file(profile_name: str, region_name: str, queue_url: str, queue_attributes_obj: dict) -> None:
-    sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
-        profile_name, region_name, "sqs", "get-queue-attributes"
-    )
-    filename = get_filename_from_parameter(queue_url)
-    sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename)
-    print(f"Output File: {sqs_attributes_file}")
-    if queue_attributes_obj is not None:
-        with open(sqs_attributes_file, "w+") as f:
-            f.write(
-                json.dumps(queue_attributes_obj, indent=4, sort_keys=True, default=custom_serializer)
-            )
+
+def set_unused_flag(queue_attributes_objs: list) -> list:
+    print('Setting the "Unused" flags')
+    for queue_attributes_obj in queue_attributes_objs:
+        if 'CreatedTimestamp' not in queue_attributes_obj['Attributes']:
+            print("Error: CreatedTimestamp not found in queue attributes")
+            continue
+        creation_time = int(queue_attributes_obj['Attributes']['CreatedTimestamp'])
+        created_recently = datetime.utcnow() - timedelta(weeks=4) < datetime.fromtimestamp(creation_time)
+
+        if int(queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth']) == 0 and not created_recently:
+            queue_attributes_obj['Attributes']['Unused'] = True
+        else:
+            queue_attributes_obj['Attributes']['Unused'] = False
+
+    return queue_attributes_objs
+
+
+def save_sqs_attributes_file(profile_name: str, region_name: str, queue_urls: list, queue_attributes_objs: list) -> None:
+    print('Saving SQS attributes files')
+    for queue_url, queue_attributes_obj in zip(queue_urls, queue_attributes_objs):
+        sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
+            profile_name, region_name, "sqs", "get-queue-attributes"
+        )
+        filename = get_filename_from_parameter(queue_url)
+        sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename)
+        if not os.path.exists(sqs_attributes_file):
+            print(f"File does not exist: {sqs_attributes_file}")
+            continue
+        if queue_attributes_obj is not None:
+            with open(sqs_attributes_file, "w+") as f:
+                f.write(
+                    json.dumps(queue_attributes_obj, indent=4, sort_keys=True, default=custom_serializer)
+                )
 
     return
 
@@ -106,8 +140,12 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
     sqs = session.client('sqs')
     client = session.client('cloudwatch')
     base_parameters = {
-        'MetricDataQueries': [{
-            'Id': 'id_0',
+        'MetricDataQueries': [],
+        'StartTime': datetime.utcnow() - timedelta(weeks=4),
+        'EndTime': datetime.utcnow(),
+    }
+    inner_base_params = {
+            #'Id': 'id_0',
             'MetricStat': {
                 'Metric': {
                     'Namespace': 'AWS/SQS',
@@ -122,32 +160,35 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
                 'Stat': 'Sum'
             },
             'ReturnData': True,
-        }],
-        'StartTime': datetime.utcnow() - timedelta(weeks=4),
-        'EndTime': datetime.utcnow(),
-    }
+        }
 
     for account in accounts:
         # get_regions reads the file at account-data/{profile}/describe-regions.json
         for region_json in get_regions(Account(None, account)):
             region = Region(Account(None, account), region_json)
-            print(f"Region: {region.name}")
+            print(f"Processing SQS Queues for Region {region.name}")
 
             saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region)
 
             if 'QueueUrls' not in saved_sqs_list:
+                print(f"No SQS queues found in {region.name}")
                 continue
 
-            print(f"Queue Count: {len(saved_sqs_list['QueueUrls'])}")
-            queue_url = saved_sqs_list['QueueUrls'][2]
-            saved_sqs_details = get_parameter_file(region, "sqs", "get-queue-attributes", queue_url)
-            queue_name = saved_sqs_details['Attributes']['QueueArn'].split(':')[-1]
-            #print(f"Queue Name: {queue_name}")
-
-            saved_sqs_details = fetch_queue_metrics(client, base_parameters, queue_name, saved_sqs_details)
-            saved_sqs_details = fetch_queue_tags(sqs, queue_url, saved_sqs_details)
-            saved_sqs_details = set_unused_flag(saved_sqs_details)
-            save_sqs_attributes_file(arguments.profile, region.name, queue_url, saved_sqs_details)
+            print(f"Found {len(saved_sqs_list['QueueUrls'])} SQS queues in {region.name}")
+
+            for i in range(len(saved_sqs_list['QueueUrls']) // 500 + 1):
+                end_idx = (i+1)*500
+                if end_idx > len(saved_sqs_list['QueueUrls']):
+                    end_idx = len(saved_sqs_list['QueueUrls'])
+                queue_urls = saved_sqs_list['QueueUrls'][i*500:end_idx]
+                print(f"Processing URLs [{i*500}, {end_idx})")
+                queue_details = [get_parameter_file(region, "sqs", "get-queue-attributes", queue_url) for queue_url in queue_urls]
+                queue_names = [queue_detail['Attributes']['QueueArn'].split(':')[-1] for queue_detail in queue_details]
+
+                queue_details = fetch_queue_metrics(client, base_parameters, inner_base_params, queue_names, queue_details)
+                queue_details = fetch_queue_tags(sqs, queue_urls, queue_details)
+                queue_details = set_unused_flag(queue_details)
+                save_sqs_attributes_file(arguments.profile, region.name, queue_urls, queue_details)
 
     return
 

From 968008f83388fc48d0ea25864353a5339ddf969f Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 10 Apr 2023 13:29:32 -0700
Subject: [PATCH 17/27] Add dry_run flag to parse_arguments

---
 shared/common.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/shared/common.py b/shared/common.py
index e9797fbeb..9b7bae0d1 100644
--- a/shared/common.py
+++ b/shared/common.py
@@ -236,6 +236,13 @@ def parse_arguments(arguments, parser=None):
         required=False,
         type=str,
     )
+    parser.add_argument(
+        "--dry_run",
+        help="Do not run AWS commands",
+        default=False,
+        required=False,
+        type=bool
+    )
     args = parser.parse_args(arguments)
 
     global LOG_LEVEL

From 934a3a619bb301a6abf0ad92c28f8724a75d6ace Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 10 Apr 2023 13:29:54 -0700
Subject: [PATCH 18/27] Correct import statements

---
 private_commands/sqs-determine-unused.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index b2aeb1c01..ad8ce6548 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -8,8 +8,9 @@
 from botocore.exceptions import ClientError
 from commands.collect import get_filename_from_parameter
 from datetime import datetime, timedelta
-from shared.common import query_aws, get_regions, parse_arguments, get_parameter_file, custom_serializer
+from shared.common import get_regions, parse_arguments, custom_serializer
 from shared.nodes import Account, Region
+from shared.query import query_aws, get_parameter_file
 
 def make_directory(path: str) -> None:
     try:

From 6f5caeeadb3a0f16cb1783cd028d632ae0c536b0 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 10 Apr 2023 13:30:13 -0700
Subject: [PATCH 19/27] Add sqs-delete-unused, works

---
 private_commands/sqs-delete-unused.py | 110 ++++++++++++++++++++++++++
 1 file changed, 110 insertions(+)
 create mode 100644 private_commands/sqs-delete-unused.py

diff --git a/private_commands/sqs-delete-unused.py b/private_commands/sqs-delete-unused.py
new file mode 100644
index 000000000..f53714936
--- /dev/null
+++ b/private_commands/sqs-delete-unused.py
@@ -0,0 +1,110 @@
+import boto3
+import json
+import logging
+import os
+import re
+import copy
+
+from botocore.exceptions import ClientError
+from commands.collect import get_filename_from_parameter
+from datetime import datetime, timedelta
+from shared.common import get_regions, parse_arguments, custom_serializer
+from shared.nodes import Account, Region
+from shared.query import query_aws, get_parameter_file
+
+def filter_for_unused_queues(queue_urls: list, queue_attributes_objs: list) -> list:
+    unused_queues = [queue_url for queue_url, queue_attributes_obj in zip(queue_urls, queue_attributes_objs) \
+                     if queue_attributes_obj['Attributes']['Unused'] and \
+                     ('Tags' not in queue_attributes_obj['Attributes'] or 'EngTeam' not in queue_attributes_obj['Attributes']['Tags'])]
+    print(f'Filtered {len(queue_urls)} queues down to {len(unused_queues)} unused queues')
+
+    return unused_queues
+
+
+def delete_queues(client: boto3.Session.client, queue_urls: list) -> None:
+    print(f'Deleting {len(queue_urls)} unused queues')
+    for queue_url in queue_urls:
+        try:
+            client.delete_queue(QueueUrl=queue_url)
+        except ClientError as e:
+            print(f'Error deleting queue {queue_url}: {e}')
+    
+    return
+
+
+def clean_up_deleted_queue_files(unused_queues: list, saved_sqs_list: dict, account_name: str, profile_name: str, region_name: str) -> None:
+    print(f'Cleaning up {len(unused_queues)} deleted queue files')
+    original_queue_list_len = len(saved_sqs_list['QueueUrls'])
+    sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format(
+        profile_name, region_name, "sqs", "get-queue-attributes"
+    )
+
+    for queue_url in unused_queues:
+        saved_sqs_list['QueueUrls'].remove(queue_url)
+        filename = get_filename_from_parameter(queue_url)
+        sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename)
+        if os.path.exists(sqs_attributes_file):
+            os.remove(sqs_attributes_file)
+
+    print(f'Reduced queue list from {original_queue_list_len} to {len(saved_sqs_list["QueueUrls"])}')
+    sqs_queue_list_file = "account-data/{}/{}/{}.json".format(account_name, region_name, "sqs-list-queues")
+    with open(sqs_queue_list_file, "w+") as f:
+        f.write(
+            json.dumps(saved_sqs_list, indent=4, sort_keys=True, default=custom_serializer)
+        )
+
+    return
+
+
+def run(arguments: list) -> None:
+    args, accounts, config = parse_arguments(arguments)
+    logging.getLogger("botocore").setLevel(logging.WARN)
+
+    default_region = os.environ.get("AWS_REGION", "us-east-1")
+
+    session_data = {"region_name": default_region}
+
+    if args.profile:
+        session_data["profile_name"] = args.profile
+
+    session = boto3.Session(**session_data)
+
+    sts = session.client("sts")
+    try:
+        sts.get_caller_identity()
+        logging.debug("Using AWS account: {}".format(sts.get_caller_identity()["Account"]))
+    except ClientError as e:
+        if "InvalidClientTokenId" in str(e):
+            print(
+                "ERROR: sts.get_caller_identity failed with InvalidClientTokenId. Likely cause is no AWS credentials are set.",
+                flush=True,
+            )
+            exit(-1)
+        else:
+            print(
+                "ERROR: Unknown exception when trying to call sts.get_caller_identity: {}".format(
+                    e
+                ),
+                flush=True,
+            )
+            exit(-1)
+
+    sqs = session.client('sqs')
+    for account in accounts:
+        # get_regions reads the file at account-data/{profile}/describe-regions.json
+        for region_json in get_regions(Account(None, account)):
+            region = Region(Account(None, account), region_json)
+            print(f"Processing SQS Queues for Region {region.name}")
+
+            saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region)
+
+            if 'QueueUrls' not in saved_sqs_list:
+                print(f"No SQS queues found in {region.name}")
+                continue
+
+            queue_urls = saved_sqs_list['QueueUrls']
+            queue_details = [get_parameter_file(region, "sqs", "get-queue-attributes", queue_url) for queue_url in queue_urls]
+            unused_queues = filter_for_unused_queues(queue_urls, queue_details)
+            if not args.dry_run:
+                delete_queues(sqs, unused_queues)
+                clean_up_deleted_queue_files(unused_queues, saved_sqs_list, account['name'], args.profile, region.name)

From 8508b33f45cc6fd5269848b3217025f45a243c2d Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Thu, 13 Apr 2023 16:06:41 -0700
Subject: [PATCH 20/27] Enable list of metrics to be used

Check queue names for ec2 instance IDs

Fix multi-region behavior
---
 private_commands/sqs-determine-unused.py | 93 +++++++++++++++++-------
 1 file changed, 65 insertions(+), 28 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index ad8ce6548..2d03442e1 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -3,7 +3,6 @@
 import logging
 import os
 import re
-import copy
 
 from botocore.exceptions import ClientError
 from commands.collect import get_filename_from_parameter
@@ -24,31 +23,56 @@ def snakecase(s: str) -> str:
     return s.replace("-", "_")
 
 
-def fetch_queue_metrics(client: boto3.Session.client, base_parameters: dict, inner_base_parameters: dict, queue_names: list, queue_attributes_objs: list) -> list:
+def fetch_queue_metrics(client: boto3.Session.client, base_parameters: dict, inner_base_parameters: dict, metrics_to_fetch: list, queue_names: list, queue_attributes_objs: list) -> list:
     print(f'Fetching metrics for {len(queue_names)} queues')
     if len(base_parameters['MetricDataQueries']) > 0:
         base_parameters['MetricDataQueries'] = []
     full_params = base_parameters.copy()
+    print(f'Using Metrics: {metrics_to_fetch}')
 
     for i, queue_name in enumerate(queue_names):
-        single_sub_params = inner_base_parameters.copy()
-        single_sub_params['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
-        single_sub_params['Id'] = f'queue{str(i)}'
-        full_params['MetricDataQueries'].append(single_sub_params)
+        for metric in metrics_to_fetch:
+            single_sub_params = inner_base_parameters.copy()
+            single_sub_params['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
+            single_sub_params['MetricStat']['Metric']['MetricName'] = metric
+            single_sub_params['Id'] = f'queue_{str(i)}_{metric}'
+            full_params['MetricDataQueries'].append(single_sub_params)
 
     response = client.get_metric_data(**full_params)
-    metric_results = [(int(re.sub('[a-z]', '', x['Id'])), sum(x['Values'])) for x in response['MetricDataResults']]
+    metric_results = [(int(x['Id'].split('_')[1]), x['Id'].split('_')[-1], sum(x['Values'])) for x in response['MetricDataResults']]
 
-    # paranoid about things being out of order, so sort by the id
-    metric_results.sort(key = lambda x: x[0])
-    metric_results = [x[1] for x in metric_results]
-
-    if len(metric_results) != len(queue_names) or len(metric_results) != len(queue_attributes_objs):
+    if len(metric_results) != len(queue_names)*len(metrics_to_fetch) or \
+        len(metric_results) != len(queue_attributes_objs)*len(metrics_to_fetch):
         print("Error: Number of metric results don't match queue names or attributes")
         exit(-1)
 
     for i, queue_attributes_obj in enumerate(queue_attributes_objs):
-        queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth'] = metric_results[i]
+        for j, metric in enumerate(metrics_to_fetch):
+            result_idx = len(metrics_to_fetch)*i + j
+            if metric != metric_results[result_idx][1]:
+                print(f"Error: Metric name {metric_results[result_idx][1]} doesn't match expected value {metric}")
+                exit(-1)
+            queue_attributes_obj['Attributes'][metric] = metric_results[result_idx][2]
+
+    return queue_attributes_objs
+
+
+def fetch_ec2_dependencies(client: boto3.Session.client, queue_names: list, queue_attributes_objs: list) -> list:
+    print('Fetching EC2 dependencies')
+    ec2_id_regex = 'i-[a-f0-9]{8}(?:[a-f0-9]{9})?'
+
+    for i, queue_name in enumerate(queue_names):
+        instance_id = re.findall(ec2_id_regex, queue_name)
+        if instance_id:
+            try:
+                ec2_response = client.describe_instance_status(InstanceIds=instance_id, IncludeAllInstances=True)
+                print(ec2_response)
+                queue_attributes_objs[i]['Attributes']['HasEC2Dependency'] = True
+            except ClientError as e:
+                if e.response['Error']['Code'] == 'InvalidInstanceID.NotFound':
+                    queue_attributes_objs[i]['Attributes']['HasEC2Dependency'] = False
+                else:
+                    raise e
 
     return queue_attributes_objs
 
@@ -63,7 +87,7 @@ def fetch_queue_tags(client: boto3.Session.client, queue_urls: list, queue_attri
     return queue_attributes_objs
 
 
-def set_unused_flag(queue_attributes_objs: list) -> list:
+def set_unused_flag(queue_attributes_objs: list, metrics: list) -> list:
     print('Setting the "Unused" flags')
     for queue_attributes_obj in queue_attributes_objs:
         if 'CreatedTimestamp' not in queue_attributes_obj['Attributes']:
@@ -72,10 +96,12 @@ def set_unused_flag(queue_attributes_objs: list) -> list:
         creation_time = int(queue_attributes_obj['Attributes']['CreatedTimestamp'])
         created_recently = datetime.utcnow() - timedelta(weeks=4) < datetime.fromtimestamp(creation_time)
 
-        if int(queue_attributes_obj['Attributes']['MessagesReceivedInLastMonth']) == 0 and not created_recently:
-            queue_attributes_obj['Attributes']['Unused'] = True
-        else:
-            queue_attributes_obj['Attributes']['Unused'] = False
+        unused_list = [int(queue_attributes_obj['Attributes'][metric]) == 0 for metric in metrics]
+        unused_list.append(not created_recently)
+        if 'HasEC2Dependency' in queue_attributes_obj['Attributes']:
+            unused_list.append(not queue_attributes_obj['Attributes']['HasEC2Dependency'])
+
+        queue_attributes_obj['Attributes']['Unused'] = all(unused_list)
 
     return queue_attributes_objs
 
@@ -105,7 +131,7 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
 
     default_region = os.environ.get("AWS_REGION", "us-east-1")
     # regions_filter = None
-    # if len(arguments.regions_filter) > 0:
+    # if len(args.regions_filter) > 0:
     #     regions_filter = arguments.regions_filter.lower().split(",")
     #     # Force include of default region -- seems to be required
     #     if default_region not in regions_filter:
@@ -138,8 +164,7 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
             )
             exit(-1)
 
-    sqs = session.client('sqs')
-    client = session.client('cloudwatch')
+    max_metric_queries = 500
     base_parameters = {
         'MetricDataQueries': [],
         'StartTime': datetime.utcnow() - timedelta(weeks=4),
@@ -150,7 +175,7 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
             'MetricStat': {
                 'Metric': {
                     'Namespace': 'AWS/SQS',
-                    'MetricName': 'NumberOfMessagesReceived',
+                    #'MetricName': 'NumberOfMessagesReceived',
                     'Dimensions': [
                         {
                             'Name': 'QueueName'
@@ -162,11 +187,20 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
             },
             'ReturnData': True,
         }
+    metrics = [
+        'NumberOfEmptyReceives',
+        'NumberOfMessagesSent',
+        'NumberOfMessagesReceived',
+        'NumberOfMessagesDeleted'
+    ]
 
     for account in accounts:
         # get_regions reads the file at account-data/{profile}/describe-regions.json
         for region_json in get_regions(Account(None, account)):
             region = Region(Account(None, account), region_json)
+            sqs = session.client("sqs", region_name=region.name)
+            cloudwatch = session.client('cloudwatch', region_name=region.name)
+            ec2 = session.client('ec2', region_name=region.name)
             print(f"Processing SQS Queues for Region {region.name}")
 
             saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region)
@@ -177,18 +211,21 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
 
             print(f"Found {len(saved_sqs_list['QueueUrls'])} SQS queues in {region.name}")
 
-            for i in range(len(saved_sqs_list['QueueUrls']) // 500 + 1):
-                end_idx = (i+1)*500
+            for i in range(len(saved_sqs_list['QueueUrls']) * len(metrics) // max_metric_queries + 1):
+
+                start_idx = i*max_metric_queries//len(metrics)
+                end_idx = (i+1)*max_metric_queries//len(metrics)
                 if end_idx > len(saved_sqs_list['QueueUrls']):
                     end_idx = len(saved_sqs_list['QueueUrls'])
-                queue_urls = saved_sqs_list['QueueUrls'][i*500:end_idx]
-                print(f"Processing URLs [{i*500}, {end_idx})")
+                print(f"Processing URLs [{start_idx}, {end_idx})")
+                queue_urls = saved_sqs_list['QueueUrls'][start_idx:end_idx]
                 queue_details = [get_parameter_file(region, "sqs", "get-queue-attributes", queue_url) for queue_url in queue_urls]
                 queue_names = [queue_detail['Attributes']['QueueArn'].split(':')[-1] for queue_detail in queue_details]
 
-                queue_details = fetch_queue_metrics(client, base_parameters, inner_base_params, queue_names, queue_details)
+                queue_details = fetch_queue_metrics(cloudwatch, base_parameters, inner_base_params, metrics, queue_names, queue_details)
                 queue_details = fetch_queue_tags(sqs, queue_urls, queue_details)
-                queue_details = set_unused_flag(queue_details)
+                queue_details = fetch_ec2_dependencies(ec2, queue_urls, queue_details)
+                queue_details = set_unused_flag(queue_details, metrics)
                 save_sqs_attributes_file(arguments.profile, region.name, queue_urls, queue_details)
 
     return

From d226d4356eef09b286c9fea94439fdf6c3e02c6a Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Thu, 13 Apr 2023 16:07:08 -0700
Subject: [PATCH 21/27] Make dry_run print the queues that will be deleted

---
 private_commands/sqs-delete-unused.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/private_commands/sqs-delete-unused.py b/private_commands/sqs-delete-unused.py
index f53714936..b4f377044 100644
--- a/private_commands/sqs-delete-unused.py
+++ b/private_commands/sqs-delete-unused.py
@@ -108,3 +108,6 @@ def run(arguments: list) -> None:
             if not args.dry_run:
                 delete_queues(sqs, unused_queues)
                 clean_up_deleted_queue_files(unused_queues, saved_sqs_list, account['name'], args.profile, region.name)
+            else:
+                print("This operation would delete the following SQS queues:")
+                print(unused_queues)

From 70f864c698d6b8316c73e07ada69203423fc4fc8 Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 17 Apr 2023 16:08:05 -0700
Subject: [PATCH 22/27] Don't filter out EngTeam-tagged queues anymore

---
 private_commands/sqs-delete-unused.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/private_commands/sqs-delete-unused.py b/private_commands/sqs-delete-unused.py
index b4f377044..b8d48d2da 100644
--- a/private_commands/sqs-delete-unused.py
+++ b/private_commands/sqs-delete-unused.py
@@ -14,8 +14,8 @@
 
 def filter_for_unused_queues(queue_urls: list, queue_attributes_objs: list) -> list:
     unused_queues = [queue_url for queue_url, queue_attributes_obj in zip(queue_urls, queue_attributes_objs) \
-                     if queue_attributes_obj['Attributes']['Unused'] and \
-                     ('Tags' not in queue_attributes_obj['Attributes'] or 'EngTeam' not in queue_attributes_obj['Attributes']['Tags'])]
+                     if queue_attributes_obj['Attributes']['Unused']] #and \
+                     #('Tags' not in queue_attributes_obj['Attributes'] or 'EngTeam' not in queue_attributes_obj['Attributes']['Tags'])]
     print(f'Filtered {len(queue_urls)} queues down to {len(unused_queues)} unused queues')
 
     return unused_queues

From d403abc7bb61046519c778ebc30152ac9fbac54d Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 17 Apr 2023 16:08:21 -0700
Subject: [PATCH 23/27] Print more useful information when using dry_run

---
 private_commands/sqs-delete-unused.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/private_commands/sqs-delete-unused.py b/private_commands/sqs-delete-unused.py
index b8d48d2da..9c91b53d9 100644
--- a/private_commands/sqs-delete-unused.py
+++ b/private_commands/sqs-delete-unused.py
@@ -109,5 +109,9 @@ def run(arguments: list) -> None:
                 delete_queues(sqs, unused_queues)
                 clean_up_deleted_queue_files(unused_queues, saved_sqs_list, account['name'], args.profile, region.name)
             else:
-                print("This operation would delete the following SQS queues:")
-                print(unused_queues)
+                print(f"This operation would delete the following {len(unused_queues)} SQS queues:")
+                print(f"This included {len([queue for queue in unused_queues if 'celery-pidbox' in queue])} celery-pidbox queues.")
+                print(f"Non-celery-pidbox queues are:")
+                for queue in unused_queues:
+                    if 'celery-pidbox' not in queue:
+                        print(queue)

From b17f109e8f4c64e166c2aa900f1da7776ec7560a Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 17 Apr 2023 16:10:00 -0700
Subject: [PATCH 24/27] Expand window from 4 to 13 weeks

Use all 9 SQS metrics
---
 private_commands/sqs-determine-unused.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index 2d03442e1..3f4466d1f 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -94,7 +94,7 @@ def set_unused_flag(queue_attributes_objs: list, metrics: list) -> list:
             print("Error: CreatedTimestamp not found in queue attributes")
             continue
         creation_time = int(queue_attributes_obj['Attributes']['CreatedTimestamp'])
-        created_recently = datetime.utcnow() - timedelta(weeks=4) < datetime.fromtimestamp(creation_time)
+        created_recently = datetime.utcnow() - timedelta(weeks=13) < datetime.fromtimestamp(creation_time)
 
         unused_list = [int(queue_attributes_obj['Attributes'][metric]) == 0 for metric in metrics]
         unused_list.append(not created_recently)
@@ -167,7 +167,7 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
     max_metric_queries = 500
     base_parameters = {
         'MetricDataQueries': [],
-        'StartTime': datetime.utcnow() - timedelta(weeks=4),
+        'StartTime': datetime.utcnow() - timedelta(weeks=13),
         'EndTime': datetime.utcnow(),
     }
     inner_base_params = {
@@ -188,10 +188,15 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
             'ReturnData': True,
         }
     metrics = [
+        'ApproximateAgeOfOldestMessage',
+        'ApproximateNumberOfMessagesDelayed',
+        'ApproximateNumberOfMessagesNotVisible',
+        'ApproximateNumberOfMessagesVisible',
         'NumberOfEmptyReceives',
-        'NumberOfMessagesSent',
+        'NumberOfMessagesDeleted',
         'NumberOfMessagesReceived',
-        'NumberOfMessagesDeleted'
+        'NumberOfMessagesSent',
+        'SentMessageSize'
     ]
 
     for account in accounts:

From 5f067d894573ce684b30ae24294c4c67250d1bfa Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 17 Apr 2023 16:10:28 -0700
Subject: [PATCH 25/27] Fix bug with shallow dictionary copying

---
 private_commands/sqs-determine-unused.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index 3f4466d1f..2eef6c2c2 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -1,4 +1,5 @@
 import boto3
+import copy
 import json
 import logging
 import os
@@ -32,7 +33,7 @@ def fetch_queue_metrics(client: boto3.Session.client, base_parameters: dict, inn
 
     for i, queue_name in enumerate(queue_names):
         for metric in metrics_to_fetch:
-            single_sub_params = inner_base_parameters.copy()
+            single_sub_params = copy.deepcopy(inner_base_parameters)
             single_sub_params['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name
             single_sub_params['MetricStat']['Metric']['MetricName'] = metric
             single_sub_params['Id'] = f'queue_{str(i)}_{metric}'

From e7b5061e4298a5764b01d805cdda84107a0eb5ae Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Mon, 17 Apr 2023 16:10:45 -0700
Subject: [PATCH 26/27] Fix bug with rounding issue in indexing

---
 private_commands/sqs-determine-unused.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py
index 2eef6c2c2..eebf3bc2e 100644
--- a/private_commands/sqs-determine-unused.py
+++ b/private_commands/sqs-determine-unused.py
@@ -219,12 +219,15 @@ def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict
 
             for i in range(len(saved_sqs_list['QueueUrls']) * len(metrics) // max_metric_queries + 1):
 
-                start_idx = i*max_metric_queries//len(metrics)
-                end_idx = (i+1)*max_metric_queries//len(metrics)
+                start_idx = i*(max_metric_queries//len(metrics))
+                end_idx = (i+1)*(max_metric_queries//len(metrics))
+                if start_idx < 0:
+                    start_idx = 0
                 if end_idx > len(saved_sqs_list['QueueUrls']):
                     end_idx = len(saved_sqs_list['QueueUrls'])
                 print(f"Processing URLs [{start_idx}, {end_idx})")
                 queue_urls = saved_sqs_list['QueueUrls'][start_idx:end_idx]
+
                 queue_details = [get_parameter_file(region, "sqs", "get-queue-attributes", queue_url) for queue_url in queue_urls]
                 queue_names = [queue_detail['Attributes']['QueueArn'].split(':')[-1] for queue_detail in queue_details]
 

From 0a68214fa338059d2eec8d5f5f946dfef6f369bd Mon Sep 17 00:00:00 2001
From: Sebastian Alvis <salvis@hearsaycorp.com>
Date: Thu, 6 Jul 2023 15:03:40 -0700
Subject: [PATCH 27/27] Use separate Dockerfile for local work

---
 Dockerfile-local | 40 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 40 insertions(+)
 create mode 100644 Dockerfile-local

diff --git a/Dockerfile-local b/Dockerfile-local
new file mode 100644
index 000000000..2abb52aa3
--- /dev/null
+++ b/Dockerfile-local
@@ -0,0 +1,40 @@
+FROM python:3.9-slim as cloudmapper
+
+LABEL maintainer="https://github.com/0xdabbad00/"
+LABEL Project="https://github.com/duo-labs/cloudmapper"
+
+EXPOSE 8000
+WORKDIR /opt/cloudmapper
+ENV AWS_DEFAULT_REGION=us-east-1 
+
+RUN apt-get update -y
+RUN apt-get install -y build-essential autoconf automake libtool python3-tk jq groff less
+RUN apt-get install -y bash curl unzip
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip 
+RUN ./aws/install
+# RUN apk update && \
+#     apk upgrade && \
+#     apk add --virtual build-dependencies \
+#     build-base \
+#     gcc \
+#     wget \
+#     git 
+# RUN apk add --no-cache \
+#     alpine-sdk \
+#     autoconf \
+#     automake \
+#     bash \
+#     jq \
+#     libtool \
+#     python3-tkinter \
+#     python3-dev \
+#     py-pip && \
+#     pip install --upgrade pip && \
+#     pip install awscli
+
+COPY . /opt/cloudmapper
+RUN pip install -r requirements.txt
+
+RUN bash
+#ENTRYPOINT "./entrypoint"