diff --git a/.gitignore b/.gitignore index ac50e35b2..5c4e1a7d8 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,6 @@ tmp/ trusted_ips.png resource_stats.png data/ -private_commands/ output/ .vscode/ Pipfile.lock diff --git a/Dockerfile b/Dockerfile index 25fd522cd..47b1b11c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,7 @@ -FROM python:3.7-slim as cloudmapper +#FROM python:3.7-slim as cloudmapper +#FROM alpine:3.17.2 +#FROM python:3.7-alpine3.17 +FROM python:3.9-slim as cloudmapper LABEL maintainer="https://github.com/0xdabbad00/" LABEL Project="https://github.com/duo-labs/cloudmapper" @@ -10,8 +13,28 @@ ENV AWS_DEFAULT_REGION=us-east-1 RUN apt-get update -y RUN apt-get install -y build-essential autoconf automake libtool python3-tk jq awscli RUN apt-get install -y bash +# RUN apk update && \ +# apk upgrade && \ +# apk add --virtual build-dependencies \ +# build-base \ +# gcc \ +# wget \ +# git +# RUN apk add --no-cache \ +# alpine-sdk \ +# autoconf \ +# automake \ +# bash \ +# jq \ +# libtool \ +# python3-tkinter \ +# python3-dev \ +# py-pip && \ +# pip install --upgrade pip && \ +# pip install awscli COPY . /opt/cloudmapper RUN pip install -r requirements.txt -RUN bash +#RUN bash +ENTRYPOINT "./entrypoint" diff --git a/Dockerfile-local b/Dockerfile-local new file mode 100644 index 000000000..2abb52aa3 --- /dev/null +++ b/Dockerfile-local @@ -0,0 +1,40 @@ +FROM python:3.9-slim as cloudmapper + +LABEL maintainer="https://github.com/0xdabbad00/" +LABEL Project="https://github.com/duo-labs/cloudmapper" + +EXPOSE 8000 +WORKDIR /opt/cloudmapper +ENV AWS_DEFAULT_REGION=us-east-1 + +RUN apt-get update -y +RUN apt-get install -y build-essential autoconf automake libtool python3-tk jq groff less +RUN apt-get install -y bash curl unzip +RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \ + unzip awscliv2.zip +RUN ./aws/install +# RUN apk update && \ +# apk upgrade && \ +# apk add --virtual build-dependencies \ +# build-base \ +# gcc \ +# wget \ +# git +# RUN apk add --no-cache \ +# alpine-sdk \ +# autoconf \ +# automake \ +# bash \ +# jq \ +# libtool \ +# python3-tkinter \ +# python3-dev \ +# py-pip && \ +# pip install --upgrade pip && \ +# pip install awscli + +COPY . /opt/cloudmapper +RUN pip install -r requirements.txt + +RUN bash +#ENTRYPOINT "./entrypoint" diff --git a/collect_commands.yaml b/collect_commands.yaml index d3b150c74..9ffb2d372 100644 --- a/collect_commands.yaml +++ b/collect_commands.yaml @@ -175,6 +175,9 @@ Request: describe-cluster-subnet-groups - Service: sqs Request: list-queues + Parameters: + - Name: MaxResults + Value: 1000 - Service: sqs Request: get-queue-attributes Parameters: diff --git a/commands/collect.py b/commands/collect.py index 073914f4e..03710cd27 100644 --- a/commands/collect.py +++ b/commands/collect.py @@ -228,8 +228,8 @@ def collect(arguments): default_region = "us-gov-west-1" elif "cn-" in default_region: default_region = "cn-north-1" - else: - default_region = "us-east-1" + #else: + # default_region = "us-east-1" regions_filter = None if len(arguments.regions_filter) > 0: @@ -364,7 +364,7 @@ def collect(arguments): parameters[parameter["Name"]] = parameter["Value"] # Look for any dynamic values (ones that jq parse a file) - if "|" in parameter["Value"]: + if not isinstance(parameter["Value"], int) and "|" in parameter["Value"]: dynamic_parameter = parameter["Name"] if runner.get("Custom_collection", False): diff --git a/entrypoint b/entrypoint new file mode 100755 index 000000000..0ff928912 --- /dev/null +++ b/entrypoint @@ -0,0 +1,48 @@ +#!/bin/bash + +# Create the entire file via echo or create it beforehand? +export AWS_REGION="us-west-2" +export AWS_CONFIG_FILE=aws-config +echo "[profile prod]" >> aws-config +echo "region=us-west-2" >> caws-onfig +echo "output=json" >> aws-config +echo "credential_source=EcsContainer" >> aws-config +echo "role_arn=$CLOUDMAPPER_PROD_ROLE_ARN" >> aws-config +echo "" >> aws-config + +echo "[profile dev]" >> aws-config +echo "region=us-west-2" >> caws-onfig +echo "output=json" >> aws-config +echo "credential_source=EcsContainer" >> aws-config +echo "role_arn=$CLOUDMAPPER_DEV_ROLE_ARN" >> aws-config +echo "" >> aws-config + +echo "[profile solutions-external]" >> aws-config +echo "region=us-west-2" >> caws-onfig +echo "output=json" >> aws-config +echo "credential_source=EcsContainer" >> aws-config +echo "role_arn=$CLOUDMAPPER_SOL_EXT_ROLE_ARN" >> aws-config +echo "" >> aws-config + +# Configure config.json for CloudMapper +python cloudmapper.py configure add-account --config-file config.json --name prod --id $PROD_ID +python cloudmapper.py configure add-account --config-file config.json --name dev --id $DEV_ID +python cloudmapper.py configure add-account --config-file config.json --name solutions-external --id $SOL_EXT_ID + +# Run CloudMapper +python cloudmapper.py collect --account prod --profile prod --regions us-west-2 --clean +python cloudmapper.py stats --accounts prod --stats_all_resources --no_output_image \ + > prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt + +python cloudmapper.py collect --account dev --profile dev --regions us-west-2 --clean +python cloudmapper.py stats --accounts dev --stats_all_resources --no_output_image \ + > dev-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt + +python cloudmapper.py collect --account solutions-external --profile solutions-external --regions us-west-2 --clean +python cloudmapper.py stats --accounts solutions-external --stats_all_resources --no_output_image \ + > solutions-external-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt + +# Output stats results to S3 +aws s3 cp prod-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER +aws s3 cp dev-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER +aws s3 cp solutions-external-stats-$( printf '%(%Y%m%d)T\n' -1 ).txt s3://$S3_BUCKET/$S3_BUCKET_CLOUDMAPPER_LOG_FOLDER \ No newline at end of file diff --git a/private_commands/sqs-delete-unused.py b/private_commands/sqs-delete-unused.py new file mode 100644 index 000000000..9c91b53d9 --- /dev/null +++ b/private_commands/sqs-delete-unused.py @@ -0,0 +1,117 @@ +import boto3 +import json +import logging +import os +import re +import copy + +from botocore.exceptions import ClientError +from commands.collect import get_filename_from_parameter +from datetime import datetime, timedelta +from shared.common import get_regions, parse_arguments, custom_serializer +from shared.nodes import Account, Region +from shared.query import query_aws, get_parameter_file + +def filter_for_unused_queues(queue_urls: list, queue_attributes_objs: list) -> list: + unused_queues = [queue_url for queue_url, queue_attributes_obj in zip(queue_urls, queue_attributes_objs) \ + if queue_attributes_obj['Attributes']['Unused']] #and \ + #('Tags' not in queue_attributes_obj['Attributes'] or 'EngTeam' not in queue_attributes_obj['Attributes']['Tags'])] + print(f'Filtered {len(queue_urls)} queues down to {len(unused_queues)} unused queues') + + return unused_queues + + +def delete_queues(client: boto3.Session.client, queue_urls: list) -> None: + print(f'Deleting {len(queue_urls)} unused queues') + for queue_url in queue_urls: + try: + client.delete_queue(QueueUrl=queue_url) + except ClientError as e: + print(f'Error deleting queue {queue_url}: {e}') + + return + + +def clean_up_deleted_queue_files(unused_queues: list, saved_sqs_list: dict, account_name: str, profile_name: str, region_name: str) -> None: + print(f'Cleaning up {len(unused_queues)} deleted queue files') + original_queue_list_len = len(saved_sqs_list['QueueUrls']) + sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format( + profile_name, region_name, "sqs", "get-queue-attributes" + ) + + for queue_url in unused_queues: + saved_sqs_list['QueueUrls'].remove(queue_url) + filename = get_filename_from_parameter(queue_url) + sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename) + if os.path.exists(sqs_attributes_file): + os.remove(sqs_attributes_file) + + print(f'Reduced queue list from {original_queue_list_len} to {len(saved_sqs_list["QueueUrls"])}') + sqs_queue_list_file = "account-data/{}/{}/{}.json".format(account_name, region_name, "sqs-list-queues") + with open(sqs_queue_list_file, "w+") as f: + f.write( + json.dumps(saved_sqs_list, indent=4, sort_keys=True, default=custom_serializer) + ) + + return + + +def run(arguments: list) -> None: + args, accounts, config = parse_arguments(arguments) + logging.getLogger("botocore").setLevel(logging.WARN) + + default_region = os.environ.get("AWS_REGION", "us-east-1") + + session_data = {"region_name": default_region} + + if args.profile: + session_data["profile_name"] = args.profile + + session = boto3.Session(**session_data) + + sts = session.client("sts") + try: + sts.get_caller_identity() + logging.debug("Using AWS account: {}".format(sts.get_caller_identity()["Account"])) + except ClientError as e: + if "InvalidClientTokenId" in str(e): + print( + "ERROR: sts.get_caller_identity failed with InvalidClientTokenId. Likely cause is no AWS credentials are set.", + flush=True, + ) + exit(-1) + else: + print( + "ERROR: Unknown exception when trying to call sts.get_caller_identity: {}".format( + e + ), + flush=True, + ) + exit(-1) + + sqs = session.client('sqs') + for account in accounts: + # get_regions reads the file at account-data/{profile}/describe-regions.json + for region_json in get_regions(Account(None, account)): + region = Region(Account(None, account), region_json) + print(f"Processing SQS Queues for Region {region.name}") + + saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region) + + if 'QueueUrls' not in saved_sqs_list: + print(f"No SQS queues found in {region.name}") + continue + + queue_urls = saved_sqs_list['QueueUrls'] + queue_details = [get_parameter_file(region, "sqs", "get-queue-attributes", queue_url) for queue_url in queue_urls] + unused_queues = filter_for_unused_queues(queue_urls, queue_details) + if not args.dry_run: + delete_queues(sqs, unused_queues) + clean_up_deleted_queue_files(unused_queues, saved_sqs_list, account['name'], args.profile, region.name) + else: + print(f"This operation would delete the following {len(unused_queues)} SQS queues:") + print(f"This included {len([queue for queue in unused_queues if 'celery-pidbox' in queue])} celery-pidbox queues.") + print(f"Non-celery-pidbox queues are:") + for queue in unused_queues: + if 'celery-pidbox' not in queue: + print(queue) diff --git a/private_commands/sqs-determine-unused.py b/private_commands/sqs-determine-unused.py new file mode 100644 index 000000000..eebf3bc2e --- /dev/null +++ b/private_commands/sqs-determine-unused.py @@ -0,0 +1,246 @@ +import boto3 +import copy +import json +import logging +import os +import re + +from botocore.exceptions import ClientError +from commands.collect import get_filename_from_parameter +from datetime import datetime, timedelta +from shared.common import get_regions, parse_arguments, custom_serializer +from shared.nodes import Account, Region +from shared.query import query_aws, get_parameter_file + +def make_directory(path: str) -> None: + try: + os.mkdir(path) + except OSError: + # Already exists + pass + + +def snakecase(s: str) -> str: + return s.replace("-", "_") + + +def fetch_queue_metrics(client: boto3.Session.client, base_parameters: dict, inner_base_parameters: dict, metrics_to_fetch: list, queue_names: list, queue_attributes_objs: list) -> list: + print(f'Fetching metrics for {len(queue_names)} queues') + if len(base_parameters['MetricDataQueries']) > 0: + base_parameters['MetricDataQueries'] = [] + full_params = base_parameters.copy() + print(f'Using Metrics: {metrics_to_fetch}') + + for i, queue_name in enumerate(queue_names): + for metric in metrics_to_fetch: + single_sub_params = copy.deepcopy(inner_base_parameters) + single_sub_params['MetricStat']['Metric']['Dimensions'][0]['Value'] = queue_name + single_sub_params['MetricStat']['Metric']['MetricName'] = metric + single_sub_params['Id'] = f'queue_{str(i)}_{metric}' + full_params['MetricDataQueries'].append(single_sub_params) + + response = client.get_metric_data(**full_params) + metric_results = [(int(x['Id'].split('_')[1]), x['Id'].split('_')[-1], sum(x['Values'])) for x in response['MetricDataResults']] + + if len(metric_results) != len(queue_names)*len(metrics_to_fetch) or \ + len(metric_results) != len(queue_attributes_objs)*len(metrics_to_fetch): + print("Error: Number of metric results don't match queue names or attributes") + exit(-1) + + for i, queue_attributes_obj in enumerate(queue_attributes_objs): + for j, metric in enumerate(metrics_to_fetch): + result_idx = len(metrics_to_fetch)*i + j + if metric != metric_results[result_idx][1]: + print(f"Error: Metric name {metric_results[result_idx][1]} doesn't match expected value {metric}") + exit(-1) + queue_attributes_obj['Attributes'][metric] = metric_results[result_idx][2] + + return queue_attributes_objs + + +def fetch_ec2_dependencies(client: boto3.Session.client, queue_names: list, queue_attributes_objs: list) -> list: + print('Fetching EC2 dependencies') + ec2_id_regex = 'i-[a-f0-9]{8}(?:[a-f0-9]{9})?' + + for i, queue_name in enumerate(queue_names): + instance_id = re.findall(ec2_id_regex, queue_name) + if instance_id: + try: + ec2_response = client.describe_instance_status(InstanceIds=instance_id, IncludeAllInstances=True) + print(ec2_response) + queue_attributes_objs[i]['Attributes']['HasEC2Dependency'] = True + except ClientError as e: + if e.response['Error']['Code'] == 'InvalidInstanceID.NotFound': + queue_attributes_objs[i]['Attributes']['HasEC2Dependency'] = False + else: + raise e + + return queue_attributes_objs + + +def fetch_queue_tags(client: boto3.Session.client, queue_urls: list, queue_attributes_objs: list) -> list: + print('Fetching tags') + for i in range(len(queue_urls)): + queue_tags = client.list_queue_tags(QueueUrl=queue_urls[i]) + if 'Tags' in queue_tags: + queue_attributes_objs[i]['Attributes']['Tags'] = queue_tags['Tags'] + + return queue_attributes_objs + + +def set_unused_flag(queue_attributes_objs: list, metrics: list) -> list: + print('Setting the "Unused" flags') + for queue_attributes_obj in queue_attributes_objs: + if 'CreatedTimestamp' not in queue_attributes_obj['Attributes']: + print("Error: CreatedTimestamp not found in queue attributes") + continue + creation_time = int(queue_attributes_obj['Attributes']['CreatedTimestamp']) + created_recently = datetime.utcnow() - timedelta(weeks=13) < datetime.fromtimestamp(creation_time) + + unused_list = [int(queue_attributes_obj['Attributes'][metric]) == 0 for metric in metrics] + unused_list.append(not created_recently) + if 'HasEC2Dependency' in queue_attributes_obj['Attributes']: + unused_list.append(not queue_attributes_obj['Attributes']['HasEC2Dependency']) + + queue_attributes_obj['Attributes']['Unused'] = all(unused_list) + + return queue_attributes_objs + + +def save_sqs_attributes_file(profile_name: str, region_name: str, queue_urls: list, queue_attributes_objs: list) -> None: + print('Saving SQS attributes files') + for queue_url, queue_attributes_obj in zip(queue_urls, queue_attributes_objs): + sqs_attributes_filepath = "account-data/{}/{}/{}-{}".format( + profile_name, region_name, "sqs", "get-queue-attributes" + ) + filename = get_filename_from_parameter(queue_url) + sqs_attributes_file = "{}/{}".format(sqs_attributes_filepath, filename) + if not os.path.exists(sqs_attributes_file): + print(f"File does not exist: {sqs_attributes_file}") + continue + if queue_attributes_obj is not None: + with open(sqs_attributes_file, "w+") as f: + f.write( + json.dumps(queue_attributes_obj, indent=4, sort_keys=True, default=custom_serializer) + ) + + return + + +def get_sqs_queue_metrics_and_tags(arguments: dict, accounts: list, config: dict) -> None: + logging.getLogger("botocore").setLevel(logging.WARN) + + default_region = os.environ.get("AWS_REGION", "us-east-1") + # regions_filter = None + # if len(args.regions_filter) > 0: + # regions_filter = arguments.regions_filter.lower().split(",") + # # Force include of default region -- seems to be required + # if default_region not in regions_filter: + # regions_filter.append(default_region) + + session_data = {"region_name": default_region} + + if arguments.profile: + session_data["profile_name"] = arguments.profile + + session = boto3.Session(**session_data) + + sts = session.client("sts") + try: + sts.get_caller_identity() + logging.debug("Using AWS account: {}".format(sts.get_caller_identity()["Account"])) + except ClientError as e: + if "InvalidClientTokenId" in str(e): + print( + "ERROR: sts.get_caller_identity failed with InvalidClientTokenId. Likely cause is no AWS credentials are set.", + flush=True, + ) + exit(-1) + else: + print( + "ERROR: Unknown exception when trying to call sts.get_caller_identity: {}".format( + e + ), + flush=True, + ) + exit(-1) + + max_metric_queries = 500 + base_parameters = { + 'MetricDataQueries': [], + 'StartTime': datetime.utcnow() - timedelta(weeks=13), + 'EndTime': datetime.utcnow(), + } + inner_base_params = { + #'Id': 'id_0', + 'MetricStat': { + 'Metric': { + 'Namespace': 'AWS/SQS', + #'MetricName': 'NumberOfMessagesReceived', + 'Dimensions': [ + { + 'Name': 'QueueName' + }, + ], + }, + 'Period': 86400, + 'Stat': 'Sum' + }, + 'ReturnData': True, + } + metrics = [ + 'ApproximateAgeOfOldestMessage', + 'ApproximateNumberOfMessagesDelayed', + 'ApproximateNumberOfMessagesNotVisible', + 'ApproximateNumberOfMessagesVisible', + 'NumberOfEmptyReceives', + 'NumberOfMessagesDeleted', + 'NumberOfMessagesReceived', + 'NumberOfMessagesSent', + 'SentMessageSize' + ] + + for account in accounts: + # get_regions reads the file at account-data/{profile}/describe-regions.json + for region_json in get_regions(Account(None, account)): + region = Region(Account(None, account), region_json) + sqs = session.client("sqs", region_name=region.name) + cloudwatch = session.client('cloudwatch', region_name=region.name) + ec2 = session.client('ec2', region_name=region.name) + print(f"Processing SQS Queues for Region {region.name}") + + saved_sqs_list = query_aws(region.account, "sqs-list-queues", region=region) + + if 'QueueUrls' not in saved_sqs_list: + print(f"No SQS queues found in {region.name}") + continue + + print(f"Found {len(saved_sqs_list['QueueUrls'])} SQS queues in {region.name}") + + for i in range(len(saved_sqs_list['QueueUrls']) * len(metrics) // max_metric_queries + 1): + + start_idx = i*(max_metric_queries//len(metrics)) + end_idx = (i+1)*(max_metric_queries//len(metrics)) + if start_idx < 0: + start_idx = 0 + if end_idx > len(saved_sqs_list['QueueUrls']): + end_idx = len(saved_sqs_list['QueueUrls']) + print(f"Processing URLs [{start_idx}, {end_idx})") + queue_urls = saved_sqs_list['QueueUrls'][start_idx:end_idx] + + queue_details = [get_parameter_file(region, "sqs", "get-queue-attributes", queue_url) for queue_url in queue_urls] + queue_names = [queue_detail['Attributes']['QueueArn'].split(':')[-1] for queue_detail in queue_details] + + queue_details = fetch_queue_metrics(cloudwatch, base_parameters, inner_base_params, metrics, queue_names, queue_details) + queue_details = fetch_queue_tags(sqs, queue_urls, queue_details) + queue_details = fetch_ec2_dependencies(ec2, queue_urls, queue_details) + queue_details = set_unused_flag(queue_details, metrics) + save_sqs_attributes_file(arguments.profile, region.name, queue_urls, queue_details) + + return + + +def run(arguments: list) -> None: + + args, accounts, config = parse_arguments(arguments) + get_sqs_queue_metrics_and_tags(args, accounts, config) diff --git a/requirements.txt b/requirements.txt index 69b801796..308e7fd8a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ astroid==2.8.4 autoflake==1.4 autopep8==1.6.0 +# aws-google-auth boto3==1.19.10 botocore==1.22.10 certifi==2021.10.8 diff --git a/shared/common.py b/shared/common.py index f10338361..9b7bae0d1 100644 --- a/shared/common.py +++ b/shared/common.py @@ -229,6 +229,20 @@ def parse_arguments(arguments, parser=None): required=False, type=str, ) + parser.add_argument( + "--profile", + help="AWS CLI profile to use", + default=None, + required=False, + type=str, + ) + parser.add_argument( + "--dry_run", + help="Do not run AWS commands", + default=False, + required=False, + type=bool + ) args = parser.parse_args(arguments) global LOG_LEVEL diff --git a/shared/nodes.py b/shared/nodes.py index 8f16e629c..684ee17fa 100644 --- a/shared/nodes.py +++ b/shared/nodes.py @@ -811,12 +811,12 @@ def __init__(self, parent, json_blob): for vpc in parent.children: if vpc.local_id == json_blob["VpcId"]: self._parent = vpc - if self._parent is None: - raise Exception( - "Could not find parent for Redshift node, was looking for VPC {}".format( - json_blob["VpcId"] - ) - ) + # if self._parent is None: + # raise Exception( + # "Could not find parent for Redshift node, was looking for VPC {}".format( + # json_blob["VpcId"] + # ) + # ) self._local_id = json_blob["ClusterIdentifier"] self._arn = json_blob["Endpoint"]["Address"]